cypress-end-to-end-tests.yml

name: cypress-end-to-end-tests
on:
  pull_request:
    paths-ignore:
      - "**.md"
  push:
    branches: [main]
    paths-ignore:
      - "**.md"
env:
  HYDRA_FULL_ERROR: 1
jobs:
  # This job is made to setup path filtering, learn more about it here: https://github.com/facebookresearch/Mephisto/pull/857
  changes:
    runs-on: ubuntu-latest
    # Set job outputs to values from filters step below
    outputs:
      simple_static_task: ${{ steps.filter.outputs.simple_static_task }}
      static_react_task: ${{ steps.filter.outputs.static_react_task }}
      static_react_task_with_worker_opinion: ${{ steps.filter.outputs.static_react_task_with_worker_opinion }}
      elementary_remote_procedure: ${{ steps.filter.outputs.elementary_remote_procedure }}
      interactive_image_generation: ${{ steps.filter.outputs.interactive_image_generation }}
      mnist: ${{ steps.filter.outputs.mnist }}
      toxicity_detection: ${{ steps.filter.outputs.toxicity_detection }}
      abstractions: ${{ steps.filter.outputs.abstractions }}
      data_model: ${{ steps.filter.outputs.data_model }}
      operations: ${{ steps.filter.outputs.operations }}
      tools: ${{ steps.filter.outputs.tools }}
      mephisto-core: ${{ steps.filter.outputs.mephisto-core }}
      mephisto-addons: ${{ steps.filter.outputs.mephisto-addons }}
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 2
      # Workaround to avoid Post Use step failures. See: https://github.com/actions/setup-node/issues/317
      - run: mkdir -p /home/runner/work/Mephisto/Mephisto/.yarn/cache
        continue-on-error: true
      - uses: actions/setup-node@v4
        with:
          cache: yarn
          node-version: 20.18.1
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            video_annotator_demo:
              - 'examples/video_annotator_demo/**'
            form_composer_demo:
              - 'examples/form_composer_demo/**'
            simple_static_task:
              - 'examples/simple_static_task/**'
            static_react_task:
              - 'examples/static_react_task/**'
            static_react_task_with_worker_opinion:
              - 'examples/static_react_task_with_worker_opinion/**'
            elementary_remote_procedure:
              - 'examples/remote_procedure/elementary_remote_procedure/**'
            interactive_image_generation:
              - 'examples/remote_procedure/interactive_image_generation/**'
            mnist:
              - 'examples/remote_procedure/mnist/**'
            toxicity_detection:
              - 'examples/remote_procedure/toxicity_detection/**'
            abstractions:
              - 'mephisto/abstractions/**'
            data_model:
              - 'mephisto/data_model/**'
            operations:
              - 'mephisto/operations/**'
            tools:
              - 'mephisto/tools/**'
            mephisto-core:
              - 'packages/mephisto-core/src/**'
            mephisto-addons:
              - 'packages/mephisto-addons/src/**'

  #  TODO: Add tests and enable
  #  form_composer_demo:
  #    needs: changes
  #    if: ${{ (needs.changes.outputs.form_composer_demo == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
  #    runs-on: ubuntu-latest
  #    steps:
  #      - name: 🔀 Checking out repo
  #        uses: actions/checkout@v2
  #
  #      - name: 🐍 Installing python
  #        uses: actions/setup-python@v5
  #        with:
  #          python-version: 3.9
  #
  #      - name: 🪨 Setup Node
  #        uses: actions/setup-node@v4
  #        with:
  #          node-version: 20.18.1
  #
  #      - name: 🤖 Install Mephisto
  #        run: pip install -e .
  #
  #      - name: 🖋 Create data directory
  #        run: mkdir -p ~/mephisto/data
  #
  #      - name: 📂 Set the data directory
  #        run: mephisto config core.main_data_directory ~/mephisto/data
  #
  #      - name: 📦 Setting up mephisto-core package
  #        run: |
  #          cd packages/mephisto-core
  #          yarn install
  #          yarn build
  #          npm link
  #
  #      - name: 📦 Setting up mephisto-addons package
  #        run: |
  #          cd packages/mephisto-addons
  #          yarn install
  #          yarn build
  #          npm link
  #
  #      - name: ⌛️ Running cypress tests
  #        uses: cypress-io/github-action@v6.7.1
  #        with:
  #          build: npm i -D cypress@11
  #          install: false
  #          browser: chrome
  #          project: ./examples/form_composer_demo/webapp
  #          config-file: ./cypress.config.js
  #          start: python examples/form_composer_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
  #          wait-on: "http://localhost:3000/?worker_id=x&=1"
  #          headless: true
  #
  #  video_annotator_demo:
  #    needs: changes
  #    if: ${{ (needs.changes.outputs.video_annotator_demo == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
  #    runs-on: ubuntu-latest
  #    steps:
  #      - name: 🔀 Checking out repo
  #        uses: actions/checkout@v2
  #
  #      - name: 🐍 Installing python
  #        uses: actions/setup-python@v5
  #        with:
  #          python-version: 3.9
  #
  #      - name: 🪨 Setup Node
  #        uses: actions/setup-node@v4
  #        with:
  #          node-version: 20.18.1
  #
  #      - name: 🤖 Install Mephisto
  #        run: pip install -e .
  #
  #      - name: 🖋 Create data directory
  #        run: mkdir -p ~/mephisto/data
  #
  #      - name: 📂 Set the data directory
  #        run: mephisto config core.main_data_directory ~/mephisto/data
  #
  #      - name: 📦 Setting up mephisto-core package
  #        run: |
  #          cd packages/mephisto-core
  #          yarn install
  #          yarn build
  #          npm link
  #
  #      - name: 📦 Setting up mephisto-addons package
  #        run: |
  #          cd packages/mephisto-addons
  #          yarn install
  #          yarn build
  #          npm link
  #
  #      - name: ⌛️ Running cypress tests
  #        uses: cypress-io/github-action@v6.7.1
  #        with:
  #          build: npm i -D cypress@11
  #          install: false
  #          browser: chrome
  #          project: ./examples/video_annotator_demo/webapp
  #          config-file: ./cypress.config.js
  #          start: python examples/video_annotator_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
  #          wait-on: "http://localhost:3000/?worker_id=x&id=1"
  #          headless: true

  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881
  simple_static_task:
    needs: changes
    if: ${{ (needs.changes.outputs.simple_static_task == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: 🪨 Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 20.18.1

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 🚚 Create In-House provider
        run: mephisto register inhouse

      - name: 📦 Setting up mephisto-core package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-core
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./mephisto/abstractions/blueprints/static_html_task/source
          config-file: ./cypress.config.js
          start: python ./examples/simple_static_task/run_task__local__inhouse.py
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          headless: true

  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795
  static_react_task:
    needs: changes
    if: ${{ (needs.changes.outputs.static_react_task == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: 🪨 Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 20.18.1

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 🚚 Create In-House provider
        run: mephisto register inhouse

      - name: 📦 Setting up mephisto-core package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-core
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/static_react_task/webapp
          config-file: ./cypress.config.js
          start: python examples/static_react_task/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          headless: true

  # Learn more about the remote_procedure_tests here: https://github.com/facebookresearch/Mephisto/pull/800
  remote_procedure_elementary_remote_procedure:
    needs: changes
    if: ${{ (needs.changes.outputs.elementary_remote_procedure == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: 🪨 Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 20.18.1

      - name: 🤖 Install Mephisto
        run: pip install -e .

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 🚚 Create In-House provider
        run: mephisto register inhouse

      - name: 📦 Setting up mephisto-core package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-core
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/remote_procedure/elementary_remote_procedure/webapp
          config-file: ./cypress.config.js
          start: python examples/remote_procedure/elementary_remote_procedure/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          headless: true

  #  TODO: Add tests and enable
  #  remote_procedure_interactive_image_generation:
  #    needs: changes
  #    if: ${{ (needs.changes.outputs.interactive_image_generation == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
  #    runs-on: ubuntu-latest
  #    steps:
  #      - name: 🔀 Checking out repo
  #        uses: actions/checkout@v2
  #
  #      - name: 🐍 Installing python
  #        uses: actions/setup-python@v5
  #        with:
  #          python-version: 3.9
  #
  #      - name: 🪨 Setup Node
  #        uses: actions/setup-node@v4
  #        with:
  #          node-version: 20.18.1
  #
  #      - name: 🤖 Install Mephisto
  #        run: pip install -e .
  #
  #      - name: 🖋 Create data directory
  #        run: mkdir -p ~/mephisto/data
  #
  #      - name: 📂 Set the data directory
  #        run: mephisto config core.main_data_directory ~/mephisto/data
  #
  #      - name: 📦 Setting up mephisto-core package
  #        run: |
  #          cd packages/mephisto-core
  #          yarn install
  #          yarn build
  #          npm link
  #
  #      - name: 📦 Setting up mephisto-addons package
  #        run: |
  #          cd packages/mephisto-addons
  #          yarn install
  #          yarn build
  #          npm link
  #
  #      - name: ⌛️ Running cypress tests
  #        uses: cypress-io/github-action@v6.7.1
  #        with:
  #          build: npm i -D cypress@11
  #          install: false
  #          browser: chrome
  #          project: ./examples/remote_procedure/interactive_image_generation/webapp
  #          config-file: ./cypress.config.js
  #          start: python examples/remote_procedure/interactive_image_generation/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
  #          wait-on: "http://localhost:3000/?worker_id=x&id=1"
  #          headless: true

  remote_procedure_mnist:
    needs: changes
    if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: 🪨 Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 20.18.1

      - name: 🤖 Install Mephisto
        run: |
          pip install -e .
          pip install torch pillow numpy

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 🚚 Create In-House provider
        run: mephisto register inhouse

      - name: 📦 Setting up mephisto-core package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-core
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/remote_procedure/mnist/webapp
          config-file: ./cypress.config.js
          start: python examples/remote_procedure/mnist/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          headless: true

  remote_procedure_toxicity_detection:
    needs: changes
    if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: 🪨 Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 20.18.1

      - name: 🤖 Install Mephisto
        run: |
          pip install -e .
          pip install detoxify

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 🚚 Create In-House provider
        run: mephisto register inhouse

      - name: 📦 Setting up mephisto-core package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-core
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/remote_procedure/toxicity_detection/webapp
          config-file: ./cypress.config.js
          start: python examples/remote_procedure/toxicity_detection/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          headless: true

  # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/833
  static_react_task_with_worker_opinion:
    needs: changes
    if: ${{ (needs.changes.outputs.static_react_task_with_worker_opinion == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.mephisto-addons == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}}
    runs-on: ubuntu-latest
    steps:
      - name: 🔀 Checking out repo
        uses: actions/checkout@v2

      - name: 🐍 Installing python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: 🪨 Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: 20.18.1

      - name: 🤖 Install Mephisto
        run: |
          pip install -e .
          yes | pip uninstall detoxify

      - name: 🖋 Create data directory
        run: mkdir -p ~/mephisto/data

      - name: 📂 Set the data directory
        run: mephisto config core.main_data_directory ~/mephisto/data

      - name: 🚚 Create In-House provider
        run: mephisto register inhouse

      - name: 📦 Setting up mephisto-core package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-core
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: 📦 Setting up mephisto-addons package
        env:
          YARN_ENABLE_IMMUTABLE_INSTALLS: false # prevents error YN0028 during build
        run: |
          cd packages/mephisto-addons
          yarn cache clean --all
          yarn install
          yarn build
          npm link

      - name: ⌛️ Running pre-submission cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          browser: chrome
          project: ./examples/static_react_task_with_worker_opinion/webapp
          config-file: ./cypress.config.js
          spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/pre_submission_tests/*
          start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_mephisto_task.sh
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          headless: true

      - name: 🔪 Killing the web server
        run: |
          kill -INT $(lsof -i -P -n | grep LISTEN | grep python | awk '{print $2}')
          kill -INT $(lsof -i -P -n | grep LISTEN | grep node | awk '{print $2}')
          echo "Web server was killed"

      - name: 🥛 Expiring units
        run: |
          cd mephisto/scripts/local_db/gh_actions
          python expire_all_units.py

      # TODO: Fix tests for WorkerOpinion widget
      # - name: 📚 Accepting the first submitted tip, accepting the second submitted tip, and rejecting the last submitted tip
      #   run: |
      #     cd mephisto/scripts/local_db
      #     python review_tips_for_task.py << EOF
      #     react-static-task-with-tips
      #     a
      #     5
      #     The tip is very informative
      #     a
      #     0
      #     r
      #     EOF
      #
      # - name: 🔥 Removing the second accepted tip
      #   run: |
      #     cd mephisto/scripts/local_db
      #     python remove_accepted_tip.py << EOF
      #     react-static-task-with-tips
      #     n
      #     y
      #     EOF
      #
      # - name: 📖 Reviewing the accepted feedback
      #   run: |
      #     cd mephisto/scripts/local_db
      #     python review_feedback_for_task.py << EOF
      #     react-static-task-with-tips
      #     0
      #     n
      #     u
      #     y
      #     y
      #     EOF
      #     python review_feedback_for_task.py << EOF
      #     react-static-task-with-tips
      #     1
      #     n
      #     u
      #     y
      #     n
      #     EOF

      - name: ⌛️ Running post-submission cypress tests
        uses: cypress-io/github-action@v6.7.1
        with:
          build: npm i -D cypress@11
          install: false
          project: ./examples/static_react_task_with_worker_opinion/webapp
          config-file: cypress.config.js
          spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/post_submission_tests/*
          start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true
          wait-on: "http://localhost:3000/?worker_id=x&id=1"
          browser: chrome
          headless: true