diff --git a/.github/workflows/docker-testing-matrix-manual-install.yml b/.github/workflows/docker-testing-matrix-manual-install.yml new file mode 100644 index 000000000..31fca7abe --- /dev/null +++ b/.github/workflows/docker-testing-matrix-manual-install.yml @@ -0,0 +1,46 @@ +# This is a basic workflow to help you get started with Actions + +name: Docker Testing Matrix for manual installation script + +# Controls when the action will run. +on: + schedule: + # Run once a week, in this case on Sundays + - cron: "0 0 * * 0" + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +# A workflow run is made up of one or more jobs that can run sequentially or in parallel +jobs: + # This workflow contains a single job called "build" + check: + # The type of runner that the job will run on + runs-on: ubuntu-latest + strategy: + fail-fast: false # Run all combos, don't skip on failures + matrix: + # Here we will use different system images to test steps of installing Mephisto. + # For now, it is just Ubuntu. + system-version: ["ubuntu:22.04", "ubuntu:24.04"] + + # Steps represent a sequence of tasks that will be executed as part of the job + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + + # Runs a single command using the runners shell + - name: Build docker image - ${{ matrix.system-version }} + run: docker build -f ./docker/dockerfiles/Dockerfile.ubuntu-24.04 -t mephisto_manual --build-arg BASE_IMAGE=${{ matrix.system-version }} . + + - name: Print out version numbers + run: docker run mephisto_manual bash -c 'python --version && node --version && npm --version' + + # Runs a set of commands using the runners shell + - name: Check that Mephisto was installed correctly + run: | + docker run mephisto_manual + + - name: Run command that removes and rebuilds all React apps related to the FormComposer + run: | + docker run mephisto_manual bash -c 'mephisto scripts form_composer rebuild_all_apps' diff --git a/Dockerfile b/Dockerfile index 95e36dcc8..542ff92bc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,15 +11,14 @@ FROM $BASE_IMAGE RUN apt update RUN apt install keychain curl -y -COPY . /mephisto -RUN mkdir ~/.mephisto - # Create the main Mephisto data directory +COPY . /mephisto RUN mkdir -p /mephisto/data # Write the mephisto config file manually for now to avoid prompt. # For bash-style string $ expansion for newlines, # we need to switch the shell to bash: +RUN mkdir ~/.mephisto SHELL ["/bin/bash", "-c"] RUN echo $'core: \n main_data_directory: /mephisto/data' >> ~/.mephisto/config.yml diff --git a/Dockerfile-frontend b/Dockerfile-frontend deleted file mode 100644 index 41379ca0b..000000000 --- a/Dockerfile-frontend +++ /dev/null @@ -1,34 +0,0 @@ -# Using the -slim version below for minimal size. You may want to -# remove -slim, or switch to -alpine if encountering issues -ARG BASE_TAG=python3.9-nodejs15-slim -ARG BASE_IMAGE=nikolaik/python-nodejs:$BASE_TAG - -FROM $BASE_IMAGE - -# Storybook will be launched on this port: -EXPOSE 6006 - -COPY . /mephisto -RUN mkdir ~/.mephisto - -WORKDIR /mephisto - -# The `open` package needs to be unplugged from yarn 2 ONLY for Linux -# See more details here: https://github.com/yarnpkg/berry/issues/856 -RUN yarn unplug -AR open - -RUN yarn install -RUN yarn build-all - - -# Write the mephisto config file manually for now to avoid prompt. -# For bash-style string $ expansion for newlines, -# we need to switch the shell to bash: -SHELL ["/bin/bash", "-c"] -RUN echo $'core: \n main_data_directory: /mephisto/data' >> ~/.mephisto/config.yml - -# Uncomment if you'd like to install the Mephisto CLI as well: -# RUN cd /mephisto && ls && pip install -e . -# RUN mephisto check # Run mephisto check so a mock requester gets created - -CMD yarn sb diff --git a/docker/dockerfiles/Dockerfile.ubuntu-24.04 b/docker/dockerfiles/Dockerfile.ubuntu-24.04 new file mode 100644 index 000000000..8e450765b --- /dev/null +++ b/docker/dockerfiles/Dockerfile.ubuntu-24.04 @@ -0,0 +1,60 @@ +# Latest LTS Ubuntu +ARG BASE_IMAGE=ubuntu:24.04 +FROM $BASE_IMAGE + +# [FOR DOCKERFILE ONLY] Skip interactive prompt questions, like selecting your geographic area +ENV DEBIAN_FRONTEND noninteractive +ENV DOCKER_CLI_HINTS false + +# 1. --- Setup environment --- + +# Install system requirements +RUN apt update -y +RUN apt upgrade -y +RUN apt install software-properties-common keychain curl -y +RUN add-apt-repository ppa:deadsnakes/ppa -y +RUN apt update -y + +# Install Python env (Python3.9 and pip), and set default Python version +RUN apt install wget python3.9 python3.9-dev python3.9-distutils -y +RUN wget https://bootstrap.pypa.io/get-pip.py +RUN python3.9 get-pip.py +RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 + +# Install JS env (Nodejs and Yarn) +RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +RUN export NVM_DIR="$HOME/.nvm" \ + && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ + && [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" \ + && nvm install 16 \ + && ln -s $(which node) /usr/bin/node \ + && ln -s $(which npm) /usr/bin/npm + +RUN npm install -g yarn + +# 2. --- Setup Mephisto --- + +# Define path to the repo location +ENV MEPHISTO_REPO_PATH /mephisto + +# [FOR DOCKERFILE ONLY] Copy repo into Docker container +COPY . $MEPHISTO_REPO_PATH + +# Upgrade pip so we can use the `pyproject.toml` without raising an error +RUN pip install --upgrade pip +# Install Python requirements +# [FOR DOCKERFILE ONLY] `--ignore-installed` - some libs can be preinstall in Docker-system +# Use `cd /mephisto && pip install -e .` in your local environment +RUN cd /mephisto && pip install --ignore-installed -e . + +# Setup Mephisto's directories +RUN mkdir -p ~/.mephisto +# Create Mephisto's data directory +RUN mkdir -p $MEPHISTO_REPO_PATH/data +# Change Mephisto data directory from the default `data/` in repo root to `~/mephisto-data/data/` +RUN mephisto config core.main_data_directory $MEPHISTO_REPO_PATH/data + +# Assert that everything has been set up correctly +RUN mephisto check + +CMD mephisto check diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md b/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md index f9cb7cc14..2ff2e7c31 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/manual_installation.md @@ -7,55 +7,94 @@ sidebar_position: 4 --- -# Manual installation Mephisto +# Manually installing Mephisto -_(THIS FILE IS WORK IN PROGRESS)_ +We strongly recommend [running Mephisto with Docker](/docs/guides/how_to_use/efficiency_organization/docker/) as we've shown in all included task examples. -First, clone this repo to your local system. +If you do need to build it up from scratch (e.g. on a remote server), +follow this provided installation manual. -Mephisto requires >= Python 3.8 and >= npm v6. +> NOTE: Currently our instructions include only Ubuntu 24.04 LTS with Python 3.9 and Nodejs 16. +> In the future we will include instructions and support for other systems as well. -### Installation +## Ubuntu 24.04 LTS -You can install Mephisto in a few ways (Docker being the safest choice): +Note that the following steps have already been coded up in +[Dockerfile for Ubuntu 24.04 LTS](https://github.com/facebookresearch/Mephisto/blob/main/docker/dockerfiles/Dockerfile.ubuntu-24.04). +This Dockerfile is not the base Mephisto image, +but rather an example for testing and experimenting. -- **Using docker:** see [Running Mephisto with Docker](/docs/guides/how_to_use/efficiency_organization/docker/) -- **Using pip:** run this in the root repo directory - ```bash - $ pip install -e . - ``` -- **Using [poetry](https://github.com/python-poetry/poetry):** run this in the root repo directory - ```bash - # install poetry - $ curl -sSL https://install.python-poetry.org | python3 - - # from the root dir, install Mephisto: - $ poetry install - ``` +### 1. Clone Mephisto repository -### Setup +Download Mephisto code to `` directory in your local system. -Now that you have Mephisto installed, you should have access to the `mephisto` CLI tool. _If using Docker, you will need to SSH into the Docker container first._ +```shell +cd +git clone git@github.com:facebookresearch/Mephisto.git +``` + + +### 2. Prepare system + +#### Install system requirements + +```shell +apt update -y +apt upgrade -y +apt install software-properties-common keychain curl -y +add-apt-repository ppa:deadsnakes/ppa -y +apt update -y +``` + +#### Install Python env (Python3.9 and pip), and set default Python version + +```shell +apt install wget python3.9 python3.9-dev python3.9-distutils -y +wget https://bootstrap.pypa.io/get-pip.py +python3.9 get-pip.py +update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1 +``` -- We can use this CLI tool to change data directory (where the results of your crowdsourcing tasks will be stored). Its default location is `data` inside the repo root; and here we will set it to `~/mephisto-data/data` directory: - ```bash - $ mkdir ~/mephisto-data - $ mkdir ~/mephisto-data/data - $ mephisto config core.main_data_directory ~/mephisto-data/data - ``` -- We can check that everything has been set up correctly: -```bash -$ mephisto check -Mephisto seems to be set up correctly. +#### Install JS env (Nodejs and Yarn) + +```shell +curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash +export NVM_DIR="$HOME/.nvm" \ + && [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" \ + && [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" \ + && nvm install 16 \ + && ln -s $(which node) /usr/bin/node \ + && ln -s $(which npm) /usr/bin/npm +npm install -g yarn ``` -Note that registering a sandbox user will not create a new entry in your `~/.aws/credentials` file if it's for the same account as your production user, as sandbox and prod on AWS use the same access keys. -#### Task parameters +### 3. Setup Mephisto -After registering a requester, you can use `mephisto.provider.requester_name=my_mturk_user` or `mephisto.provider.requester_name=my_mturk_user_sandbox` respectively to launch a task on Mturk. +#### Install Python requirements + +```shell +pip install --upgrade pip +cd && pip install -e . +``` + +#### # Setup Mephisto's directories + +```shell +mkdir -p ~/.mephisto +mkdir -p /data +mephisto config core.main_data_directory /data +``` + +#### Assert that everything has been set up correctly + +```shell +mephisto check +``` --- ## Let's get running! -Now that you have your environment set up, you're ready for [Running your first task.](/docs/guides/tutorials/first_task/) +Now that you have your environment set up, you're ready for +[Running your first task](/docs/guides/tutorials/first_task/).