From 3147809fa952c75a7429cc0915a68ee140e81ded Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Fri, 15 Nov 2024 17:15:43 -0500 Subject: [PATCH] Port existing Task examples to use In-House provider --- .../workflows/cypress-end-to-end-tests.yml | 58 ++++++++------ .gitignore | 2 - CONTRIBUTING.md | 10 +-- docker/docker-compose.dev.vscode.yml | 2 +- .../how_to_contribute/frontend_development.md | 2 +- .../efficiency_organization/docker.md | 4 +- .../reusing_configs.md | 6 +- .../how_to_use/form_composer/embedding.md | 2 +- .../prolific/eligibility_requirements.md | 2 +- .../guides/how_to_use/review_app/overview.md | 7 +- .../how_to_use/task_creation/task_run.md | 29 +++---- .../how_to_use/video_annotator/embedding.md | 2 +- .../how_to_use/worker_quality/using_golds.md | 6 +- .../worker_quality/using_onboarding.mdx | 8 +- .../worker_quality/using_screen_units.mdx | 10 +-- docs/web/docs/guides/quickstart.md | 4 +- .../web/docs/guides/tutorials/custom_react.md | 10 +-- docs/web/docs/guides/tutorials/first_task.md | 40 +++++----- .../docs/guides/tutorials/worker_controls.md | 10 +-- docs/web/docs/guides/tutorials/workflows.md | 6 +- .../mephisto/abstractions/blueprints.html | 4 +- examples/README.md | 64 +++++++-------- examples/form_composer_demo/README.md | 21 +++-- ... dynamic_example__ec2__mturk_sandbox.yaml} | 0 ...ml => dynamic_example__ec2__prolific.yaml} | 0 ...l => dynamic_example__local__inhouse.yaml} | 2 +- ...xample_presigned_urls__ec2__prolific.yaml} | 0 ...ouse.yaml => example__local__inhouse.yaml} | 0 .../conf/example_local_mock.yaml | 26 ------ ...ample_with_gold_unit__local__inhouse.yaml} | 2 +- ...mple_with_onboarding__local__inhouse.yaml} | 2 +- ...ample_with_screening__local__inhouse.yaml} | 2 +- examples/form_composer_demo/run_task.py | 25 ------ .../run_task__local__inhouse.py | 2 +- ...> run_task_dynamic__ec2__mturk_sandbox.py} | 2 +- ....py => run_task_dynamic__ec2__prolific.py} | 2 +- ...py => run_task_dynamic__local__inhouse.py} | 2 +- ..._dynamic_presigned_urls__ec2__prolific.py} | 2 +- ...un_task_with_gold_unit__local__inhouse.py} | 2 +- ...n_task_with_onboarding__local__inhouse.py} | 2 +- ...un_task_with_screening__local__inhouse.py} | 2 +- ...sk_with_worker_opinion__local__inhouse.py} | 2 +- examples/parlai_chat_task_demo/README.md | 21 ++--- .../custom_input_and_messages/README.md | 4 +- .../conf/{base.yaml => _base.yaml} | 2 +- ...l => custom_prebuilt__local__inhouse.yaml} | 3 +- ...aml => custom_simple__local__inhouse.yaml} | 3 +- .../hydra_configs/conf/example.yaml | 18 ----- .../conf/example__local__inhouse.yaml | 4 +- ...> onboarding_example__local__inhouse.yaml} | 3 +- examples/parlai_chat_task_demo/run_task.py | 64 --------------- ...mock.yaml => example__local__inhouse.yaml} | 4 +- ...ample_with_screening__local__inhouse.yaml} | 4 +- ...un_task.py => run_task__local__inhouse.py} | 2 +- .../mnist/webapp/cypress.config.js | 2 +- .../cypress/e2e/remote_procedure_mnist.cy.js | 4 +- examples/simple_static_task/README.md | 10 ++- ....yaml => example__ec2__mturk_sandbox.yaml} | 0 ...lific.yaml => example__ec2__prolific.yaml} | 0 ...mock.yaml => example__local__inhouse.yaml} | 2 +- ...mple_with_onboarding__local__inhouse.yaml} | 2 +- ...box.py => run_task__ec2__mturk_sandbox.py} | 2 +- ...prolific.py => run_task__ec2__prolific.py} | 2 +- ...un_task.py => run_task__local__inhouse.py} | 2 +- ...n_task_with_onboarding__local__inhouse.py} | 2 +- examples/static_react_task/README.md | 67 +++++++++++----- ...mock.yaml => example__local__inhouse.yaml} | 2 +- ...mple_with_onboarding__local__inhouse.yaml} | 2 +- ...ample_with_screening__local__inhouse.yaml} | 2 +- ...un_task.py => run_task__local__inhouse.py} | 2 +- .../webapp/cypress.config.js | 3 +- .../cypress/e2e/static_react_task.cy.js | 13 +-- .../README.md | 2 +- ...mock.yaml => example__local__inhouse.yaml} | 2 +- ...mple_with_onboarding__local__inhouse.yaml} | 2 +- ...un_task.py => run_task__local__inhouse.py} | 2 +- .../webapp/cypress.config.js | 2 +- examples/video_annotator_demo/README.md | 2 +- ...l => dynamic_example__local__inhouse.yaml} | 4 +- ...ouse.yaml => example__local__inhouse.yaml} | 0 .../conf/example_local_mock.yaml | 26 ------ examples/video_annotator_demo/run_task.py | 25 ------ .../run_task__local__inhouse.py | 2 +- ...py => run_task_dynamic__local__inhouse.py} | 2 +- mephisto/README.md | 9 ++- mephisto/abstractions/blueprints/README.md | 80 ++++++++++++++----- .../blueprints/static_html_task/README.md | 48 ++++++++--- .../static_html_task/source/cypress.config.js | 2 +- .../cypress/e2e/simple_static_task.cy.js | 5 +- 89 files changed, 390 insertions(+), 462 deletions(-) rename examples/form_composer_demo/hydra_configs/conf/{dynamic_example_ec2_mturk_sandbox.yaml => dynamic_example__ec2__mturk_sandbox.yaml} (100%) rename examples/form_composer_demo/hydra_configs/conf/{dynamic_example_ec2_prolific.yaml => dynamic_example__ec2__prolific.yaml} (100%) rename examples/form_composer_demo/hydra_configs/conf/{dynamic_example_local_mock.yaml => dynamic_example__local__inhouse.yaml} (96%) rename examples/form_composer_demo/hydra_configs/conf/{dynamic_presigned_urls_example_ec2_prolific.yaml => dynamic_example_presigned_urls__ec2__prolific.yaml} (100%) rename examples/form_composer_demo/hydra_configs/conf/{example_local_inhouse.yaml => example__local__inhouse.yaml} (100%) delete mode 100644 examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml rename examples/form_composer_demo/hydra_configs/conf/{example_local_mock_with_gold_unit.yaml => example_with_gold_unit__local__inhouse.yaml} (97%) rename examples/form_composer_demo/hydra_configs/conf/{example_local_mock_with_oboarding.yaml => example_with_onboarding__local__inhouse.yaml} (96%) rename examples/form_composer_demo/hydra_configs/conf/{example_local_mock_with_screening.yaml => example_with_screening__local__inhouse.yaml} (97%) delete mode 100644 examples/form_composer_demo/run_task.py rename examples/form_composer_demo/{run_task_dynamic_ec2_mturk_sandbox.py => run_task_dynamic__ec2__mturk_sandbox.py} (98%) rename examples/form_composer_demo/{run_task_dynamic_ec2_prolific.py => run_task_dynamic__ec2__prolific.py} (97%) rename examples/form_composer_demo/{run_task_dynamic.py => run_task_dynamic__local__inhouse.py} (97%) rename examples/form_composer_demo/{run_task_dynamic_presigned_urls_ec2_prolific.py => run_task_dynamic_presigned_urls__ec2__prolific.py} (97%) rename examples/form_composer_demo/{run_task_with_gold_unit.py => run_task_with_gold_unit__local__inhouse.py} (96%) rename examples/form_composer_demo/{run_task_with_onboarding.py => run_task_with_onboarding__local__inhouse.py} (94%) rename examples/form_composer_demo/{run_task_with_screening.py => run_task_with_screening__local__inhouse.py} (96%) rename examples/form_composer_demo/{run_task_with_worker_opinion.py => run_task_with_worker_opinion__local__inhouse.py} (93%) rename examples/parlai_chat_task_demo/hydra_configs/conf/{base.yaml => _base.yaml} (94%) rename examples/parlai_chat_task_demo/hydra_configs/conf/{custom_prebuilt.yaml => custom_prebuilt__local__inhouse.yaml} (98%) rename examples/parlai_chat_task_demo/hydra_configs/conf/{custom_simple.yaml => custom_simple__local__inhouse.yaml} (98%) delete mode 100644 examples/parlai_chat_task_demo/hydra_configs/conf/example.yaml rename examples/parlai_chat_task_demo/hydra_configs/conf/{onboarding_example.yaml => onboarding_example__local__inhouse.yaml} (98%) delete mode 100644 examples/parlai_chat_task_demo/run_task.py rename examples/remote_procedure/mnist/hydra_configs/conf/{example_local_mock.yaml => example__local__inhouse.yaml} (93%) rename examples/remote_procedure/mnist/hydra_configs/conf/{screening_example_local_mock.yaml => example_with_screening__local__inhouse.yaml} (94%) rename examples/remote_procedure/mnist/{run_task.py => run_task__local__inhouse.py} (98%) rename examples/simple_static_task/hydra_configs/conf/{example_ec2_mturk_sandbox.yaml => example__ec2__mturk_sandbox.yaml} (100%) rename examples/simple_static_task/hydra_configs/conf/{example_ec2_prolific.yaml => example__ec2__prolific.yaml} (100%) rename examples/simple_static_task/hydra_configs/conf/{example_local_mock.yaml => example__local__inhouse.yaml} (96%) rename examples/simple_static_task/hydra_configs/conf/{onboarding_example_local_mock.yaml => example_with_onboarding__local__inhouse.yaml} (96%) rename examples/simple_static_task/{run_task_ec2_mturk_sandbox.py => run_task__ec2__mturk_sandbox.py} (94%) rename examples/simple_static_task/{run_task_ec2_prolific.py => run_task__ec2__prolific.py} (95%) rename examples/simple_static_task/{run_task.py => run_task__local__inhouse.py} (88%) rename examples/simple_static_task/{run_task_with_onboarding.py => run_task_with_onboarding__local__inhouse.py} (92%) rename examples/static_react_task/hydra_configs/conf/{example_local_mock.yaml => example__local__inhouse.yaml} (96%) rename examples/static_react_task/hydra_configs/conf/{onboarding_example_local_mock.yaml => example_with_onboarding__local__inhouse.yaml} (96%) rename examples/static_react_task/hydra_configs/conf/{screening_example_local_mock.yaml => example_with_screening__local__inhouse.yaml} (97%) rename examples/static_react_task/{run_task.py => run_task__local__inhouse.py} (97%) rename examples/static_react_task_with_worker_opinion/hydra_configs/conf/{example_local_mock.yaml => example__local__inhouse.yaml} (96%) rename examples/static_react_task_with_worker_opinion/hydra_configs/conf/{onboarding_example_local_mock.yaml => example_with_onboarding__local__inhouse.yaml} (96%) rename examples/static_react_task_with_worker_opinion/{run_task.py => run_task__local__inhouse.py} (95%) rename examples/video_annotator_demo/hydra_configs/conf/{dynamic_example_local_mock.yaml => dynamic_example__local__inhouse.yaml} (94%) rename examples/video_annotator_demo/hydra_configs/conf/{example_local_inhouse.yaml => example__local__inhouse.yaml} (100%) delete mode 100644 examples/video_annotator_demo/hydra_configs/conf/example_local_mock.yaml delete mode 100644 examples/video_annotator_demo/run_task.py rename examples/video_annotator_demo/{run_task_dynamic.py => run_task_dynamic__local__inhouse.py} (97%) diff --git a/.github/workflows/cypress-end-to-end-tests.yml b/.github/workflows/cypress-end-to-end-tests.yml index 56c04ec8c..0a162d963 100644 --- a/.github/workflows/cypress-end-to-end-tests.yml +++ b/.github/workflows/cypress-end-to-end-tests.yml @@ -124,8 +124,8 @@ jobs: # browser: chrome # project: ./examples/form_composer_demo/webapp # config-file: ./cypress.config.js - # start: python examples/form_composer_demo/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - # wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + # start: python examples/form_composer_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + # wait-on: "http://localhost:3000/?worker_id=x&=1" # headless: true # # video_annotator_demo: @@ -177,8 +177,8 @@ jobs: # browser: chrome # project: ./examples/video_annotator_demo/webapp # config-file: ./cypress.config.js - # start: python examples/video_annotator_demo/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - # wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + # start: python examples/video_annotator_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + # wait-on: "http://localhost:3000/?worker_id=x&id=1" # headless: true # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881 @@ -209,6 +209,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -224,12 +227,12 @@ jobs: browser: chrome project: ./mephisto/abstractions/blueprints/static_html_task/source config-file: ./cypress.config.js - start: python ./examples/simple_static_task/run_task.py - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python ./examples/simple_static_task/run_task__local__inhouse.py + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795 - static-react-task: + static_react_task: needs: changes if: ${{ (needs.changes.outputs.static_react_task == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} runs-on: ubuntu-latest @@ -256,6 +259,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -271,8 +277,8 @@ jobs: browser: chrome project: ./examples/static_react_task/webapp config-file: ./cypress.config.js - start: python examples/static_react_task/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/static_react_task/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # Learn more about the remote_procedure_tests here: https://github.com/facebookresearch/Mephisto/pull/800 @@ -303,7 +309,7 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data - - name: 🖋 Create Inhouse provider + - name: 🚚 Create Inhouse provider run: mephisto register inhouse - name: 📦 Setting up mephisto-core package @@ -322,7 +328,7 @@ jobs: project: ./examples/remote_procedure/elementary_remote_procedure/webapp config-file: ./cypress.config.js start: python examples/remote_procedure/elementary_remote_procedure/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # TODO: Add tests and enable @@ -375,13 +381,13 @@ jobs: # browser: chrome # project: ./examples/remote_procedure/interactive_image_generation/webapp # config-file: ./cypress.config.js - # start: python examples/remote_procedure/interactive_image_generation/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - # wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + # start: python examples/remote_procedure/interactive_image_generation/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + # wait-on: "http://localhost:3000/?worker_id=x&id=1" # headless: true remote_procedure_mnist: needs: changes - if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-core == 'true') }} + if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} runs-on: ubuntu-latest steps: - name: 🔀 Checking out repo @@ -408,6 +414,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -423,13 +432,13 @@ jobs: browser: chrome project: ./examples/remote_procedure/mnist/webapp config-file: ./cypress.config.js - start: python examples/remote_procedure/mnist/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/remote_procedure/mnist/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true remote_procedure_toxicity_detection: needs: changes - if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-core == 'true') }} + if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} runs-on: ubuntu-latest steps: - name: 🔀 Checking out repo @@ -456,7 +465,7 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data - - name: 🖋 Create Inhouse provider + - name: 🚚 Create Inhouse provider run: mephisto register inhouse - name: 📦 Setting up mephisto-core package @@ -475,7 +484,7 @@ jobs: project: ./examples/remote_procedure/toxicity_detection/webapp config-file: ./cypress.config.js start: python examples/remote_procedure/toxicity_detection/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/833 @@ -507,6 +516,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -530,8 +542,8 @@ jobs: project: ./examples/static_react_task_with_worker_opinion/webapp config-file: ./cypress.config.js spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/pre_submission_tests/* - start: python examples/static_react_task_with_worker_opinion/run_task.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_mephisto_task.sh + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true - name: 🔪 Killing the web server @@ -604,7 +616,7 @@ jobs: project: ./examples/static_react_task_with_worker_opinion/webapp config-file: cypress.config.js spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/post_submission_tests/* - start: python examples/static_react_task_with_worker_opinion/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true + wait-on: "http://localhost:3000/?worker_id=x&id=1" browser: chrome headless: true diff --git a/.gitignore b/.gitignore index 35d4a14d2..8c61f2496 100644 --- a/.gitignore +++ b/.gitignore @@ -28,8 +28,6 @@ mephisto/scripts/metrics/* # Examples examples/simple_static_task/hydra_configs/conf/* !examples/simple_static_task/hydra_configs/conf/example*.yaml -!examples/simple_static_task/hydra_configs/conf/prolific_example.yaml -!examples/simple_static_task/hydra_configs/conf/onboarding_example*.yaml examples/**/build/* examples/form_composer_demo/preview/*_preview.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f8945d72..5b9b5c7e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,10 +23,10 @@ We actively welcome your pull requests. 6. If you haven't already, complete the Contributor License Agreement ("CLA"). ## Cypress Testing -For cypress testing the base url is: http://localhost:3000/?worker_id=x&assignment_id=1 +For cypress testing the base url is: http://localhost:3000/?worker_id=x&id=1 ### Running end-to-end tests on a task: -1. Run the task by running python run_task.py in the appropriate task folder +1. Run the task by running python `run_task__local__inhouse.py` in the appropriate task folder 2. In a separate terminal window go into the webapp directory and run `npm run test` 3. This should open a cypress app 4. It is advised to test in Chrome(Chrome, Electron, and Firefox are all the options) as this browser works well with Cypress. @@ -40,7 +40,7 @@ Suppose you ran the toxicity detection task and then closed it. This would use a If you then ran the mnist task, for example, then assignmentId=3 and assignmentId=4 would be used. While this task is running you can choose to run cypress tests in a different terminal window by going into the webapp folder and running `npm run test`. -These tests will fail because the base url of http://localhost:3000/?worker_id=x&assignment_id=1 is not associated with the mnist task, it is associated with the toxicity detection task. The correct react-elements will not show up. +These tests will fail because the base url of http://localhost:3000/?worker_id=x&id=1 is not associated with the mnist task, it is associated with the toxicity detection task. The correct react-elements will not show up. There is a way to fix this: * You can change the base url(found in the cypress.config.js file in the webapp folder) to the current url that you are on. @@ -51,7 +51,7 @@ If you are modifying either the `mephisto-core` or `mephisto-addons` packages yo The easiest way to do this is to run a task by doing: ```bash -python run_task.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true +python run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true ``` Setting `mephisto.task.force_rebuild=true` runs `npm build` before running your task. By default the task is only rebuilt if a file is changed in the webapp, not if a linked package is changed. @@ -60,7 +60,7 @@ Setting `mephisto.task.post_install_script=link_mephisto_task.sh` runs the `link Alternatively, these values can be set in the task's hydra_configs/conf yaml file if you want to forgo typing the above and just type ```bash -python run_task.py +python run_task__local__inhouse.py ``` instead. diff --git a/docker/docker-compose.dev.vscode.yml b/docker/docker-compose.dev.vscode.yml index 46896bfd8..92834cded 100644 --- a/docker/docker-compose.dev.vscode.yml +++ b/docker/docker-compose.dev.vscode.yml @@ -29,6 +29,6 @@ services: /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 - /mephisto/examples/simple_static_task/run_task.py + /mephisto/examples/simple_static_task/run_task__local__inhouse.py ", ] diff --git a/docs/web/docs/guides/how_to_contribute/frontend_development.md b/docs/web/docs/guides/how_to_contribute/frontend_development.md index e6da5aaa7..66cd85080 100644 --- a/docs/web/docs/guides/how_to_contribute/frontend_development.md +++ b/docs/web/docs/guides/how_to_contribute/frontend_development.md @@ -23,7 +23,7 @@ To setup your local codebase to auto-lint and avoid lint test failures for your This repo uses cypress to conduct frontend end-to-end tests. Tasks in the examples folder have cypress tests. To run the tests for a task: -* Launch the task using `python run_task.py`. +* Launch the task using `python run_task__local__inhouse.py`. * Open cypress by running `npm run test` in the tasks' webapp folder. * Choose the Chrome browser to run the tests (it is the most consistent). * Click one of the specs to run its tests. diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md b/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md index bf3a04c96..97fa0d326 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md @@ -22,7 +22,7 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` ## Customizing Docker settings @@ -50,5 +50,5 @@ docker-compose -f docker/docker-compose.local.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md b/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md index 473718301..c69e9d4dd 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md @@ -13,7 +13,7 @@ As you begin launching many Mephisto tasks, you may find that there are some spe Setting up profiles is pretty easy, and makes it so that you don't have to be writing architect and crowdprovider args on every launch: ``` -python run_task.py mephisto/architect=heroku mephisto/provider=mturk_sandbox mephisto.provider.requester_name=MY_REQUESTER +python run_task__local__inhouse.py mephisto/architect=heroku mephisto/provider=mturk_sandbox mephisto.provider.requester_name=MY_REQUESTER ``` Instead you can move these common configurations into a file in your `~/.mephisto/hydra_configs/profile` dir. @@ -61,9 +61,9 @@ mephisto: Then augmenting your launch configs is as easy as doing: ``` -python run_task.py +profile=local_testing +python run_task__local__inhouse.py +profile=local_testing ... -python run_task.py +profile=live_launch +python run_task__local__inhouse.py +profile=live_launch ``` Using `profile` can be an effective way to simplify the configuration for your most common workflows. diff --git a/docs/web/docs/guides/how_to_use/form_composer/embedding.md b/docs/web/docs/guides/how_to_use/form_composer/embedding.md index 98a613eb8..4990df5c4 100644 --- a/docs/web/docs/guides/how_to_use/form_composer/embedding.md +++ b/docs/web/docs/guides/how_to_use/form_composer/embedding.md @@ -11,7 +11,7 @@ sidebar_position: 4 A few tips if you wish to embed FormComposer in your custom application: - To extrapolate form config (and generate the `task_data.json` file), call the extrapolator function `mephisto.generators.generators_utils.config_validation.task_data_config.create_extrapolated_config` - - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/form_composer_demo/run_task_dynamic.py) module + - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/form_composer_demo/run_task_dynamic__local__inhouse.py) module - To use code insertions: - for custom validators: - Point `WEBAPP__GENERATOR__CUSTOM_VALIDATORS` backend env variable to the location of `custom_validators.js` module (before building all webapp applications) diff --git a/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md b/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md index ba9ef3707..9da372d8b 100644 --- a/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md +++ b/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md @@ -33,7 +33,7 @@ You can specify Prolific qualifications via Task config file, or directly in Tas ### Usage in shared state -Shared state can handle Prolific-supported qualifications. Example in `run_task.py`: +Shared state can handle Prolific-supported qualifications. Example in `run_task__ec2__prolific.py`: ```python shared_state.prolific_specific_qualifications = [ diff --git a/docs/web/docs/guides/how_to_use/review_app/overview.md b/docs/web/docs/guides/how_to_use/review_app/overview.md index e75f70ce0..99dbd8c3a 100644 --- a/docs/web/docs/guides/how_to_use/review_app/overview.md +++ b/docs/web/docs/guides/how_to_use/review_app/overview.md @@ -8,9 +8,12 @@ sidebar_position: 1 # Overview -Generally, to view/export the data, you could write a Python script using the Mephisto [`DataBrowser` class](https://github.com/facebookresearch/Mephisto/blob/main/mephisto/tools/data_browser.py) to access the submitted data. +Generally, to view/export the data, you could write a Python script using the Mephisto +[`DataBrowser` class](https://github.com/facebookresearch/Mephisto/blob/main/mephisto/tools/data_browser.py) to access the submitted data. -For example, for the `html-static-task-example` task such a script already exists in the task folder, called [`examine_results.py`](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/examine_results.py). (This file uses the Mephisto `DataBrowser` class through the helpers in `mephisto.tools.examine_utils`.) +For example, for the `html-static-task-example` task such a script already exists in the task folder, +called [`examine_results.py`](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/examine_results.py). +(This file uses the Mephisto `DataBrowser` class through the helpers in `mephisto.tools.examine_utils`.) Another example is TaskReview app, an application with convenient rich UI. diff --git a/docs/web/docs/guides/how_to_use/task_creation/task_run.md b/docs/web/docs/guides/how_to_use/task_creation/task_run.md index 52eb4e63a..a2489441b 100644 --- a/docs/web/docs/guides/how_to_use/task_creation/task_run.md +++ b/docs/web/docs/guides/how_to_use/task_creation/task_run.md @@ -9,7 +9,8 @@ sidebar_position: 1 # How task run works -Let's understand basic components of the task launch, such as configs and the `run_task.py` script. This will help with customization of tash launch behaviors. +Let's understand basic components of the task launch, such as configs and the `run_task__local__inhouse.py` script. +This will help with customization of tash launch behaviors. ### 3.1 Config registration @@ -17,7 +18,7 @@ Mephisto wires up to configuration using standard Hydra syntax, but with both `y Here's the config we've set up for this example: ```python -# examples/form_composer_demo/run_task.py +# examples/form_composer_demo/run_task__local__inhouse.py import os from omegaconf import DictConfig @@ -27,7 +28,7 @@ from mephisto.tools.scripts import build_custom_bundle from mephisto.tools.scripts import task_script -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: ``` @@ -36,7 +37,7 @@ This is all you really *need* to launch a Mephisto task! The `@task_script` deco Of course, there's quite a bit of 'magic' happening underneath the hood thanks to the script utilities. This version is explicit to show where you may add customization, and re-ordered for understanding: ```python -# modified examples/form_composer_demo/run_task.py +# modified examples/form_composer_demo/run_task__local__inhouse.py import os from dataclasses import dataclass from typing import Any @@ -49,7 +50,7 @@ from mephisto.tools.scripts import build_custom_bundle from mephisto.tools.scripts import task_script @dataclass -class MyTaskConfig(build_default_task_config('example_local_mock')): +class MyTaskConfig(build_default_task_config('example__local__inhouse')): custom_args: Any = 4 @task_script(config=MyTaskConfig) @@ -57,20 +58,20 @@ def main(operator: Operator, cfg: DictConfig) -> None: ``` In this snippet, we do a few things: -1. We set up the default [`conf`](https://hydra.cc/docs/tutorials/basic/your_first_app/config_file/) file to be `example_local_mock`, +1. We set up the default [`conf`](https://hydra.cc/docs/tutorials/basic/your_first_app/config_file/) file to be `example__local__inhouse`, using `build_default_task_config`, which returns a `TaskConfig` that we can extend. 2. We extend the returned `TaskConfig` with `MyTaskConfig`, which allows us to specify custom arguments. 3. We decorate the main, noting that the correct config is `MyTaskConfig`. Note that the `default_config_file` version of this simply takes care of the above steps inline in the decorator. -With all the above, we're able to just make edits to `example_local_mock.yaml` or make other configs in the `conf/` directory and route to them directly. +With all the above, we're able to just make edits to `example__local__inhouse.yaml` or make other configs in the `conf/` directory and route to them directly. ### 3.2 Invoking Mephisto Mephisto itself is actually invoked just a little later: ```python -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: # Build packages _build_custom_bundles(cfg) @@ -89,17 +90,17 @@ To ensure we're not frozen, the operator takes in a `log_rate` in seconds to pri Again we can look back at the `example_local_mock.yaml` file to see this setup: ```yaml -# examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +# examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse ``` -These ensure that, when not provided other arguments, we launch this task locally using a `LocalArchitect` and `MockProvider`. -With these defaults, this and other example tasks are run using a "local" architect, and a "mock" requester without arguments. +These ensure that, when not provided other arguments, we launch this task locally using a `LocalArchitect` and `InhouseProvider`. +With these defaults, this and other example tasks are run using a "local" architect, and a "inhouse" requester without arguments. The "local" architect is reponsible for running a server on your local machine to host the task, -and the "mock" requester lets *you* simulate a worker without using an external crowd-provider platform such as Prolific or MTurk to launch the task. +and the "inhouse" requester lets *you* simulate a worker without using an external crowd-provider platform such as Prolific or MTurk to launch the task. ### 3.4 `Unit` creation explained @@ -108,7 +109,7 @@ It's useful to understand how this happens. Taking a look at the config and data: ```yaml -# examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +# examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml #@package _global_ defaults: diff --git a/docs/web/docs/guides/how_to_use/video_annotator/embedding.md b/docs/web/docs/guides/how_to_use/video_annotator/embedding.md index 893409896..77b69f89c 100644 --- a/docs/web/docs/guides/how_to_use/video_annotator/embedding.md +++ b/docs/web/docs/guides/how_to_use/video_annotator/embedding.md @@ -11,7 +11,7 @@ sidebar_position: 4 A few tips if you wish to embed VideoAnnotator in your custom application: - To extrapolate annotator config (and generate the `task_data.json` file), call the extrapolator function `mephisto.generators.generators_utils.config_validation.task_data_config.create_extrapolated_config` - - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/video_annotator_demo/run_task_dynamic.py) module + - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/video_annotator_demo/run_task_dynamic__local__inhouse.py) module - To use code insertions: - for custom validators: - Point `WEBAPP__GENERATOR__CUSTOM_VALIDATORS` backend env variable to the location of `custom_validators.js` module (before building all webapp applications) diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md b/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md index 6f64d3606..47fa4eba7 100644 --- a/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md +++ b/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md @@ -61,7 +61,7 @@ You can run an example project to try gold units for yourself. docker-compose -f docker/docker-compose.dev.yml up docker exec -it mephisto_dc bash cd /mephisto/examples/form_composer_demo -python ./run_task_with_gold_unit.py +python ./run_task_with_gold_unit__local__inhouse.py ``` The first unit that you will see will be the gold one. @@ -78,8 +78,8 @@ To get past these example gold units, provide these predefined values: For an in-depth look at code underlying this example, you can read these Python files in `examples/form_composer_demo` directory: -- `run_task_with_gold_unit.py` - script to configure and launch this Task -- `hydra_configs/conf/example_local_mock_with_gold_unit.yaml` - YAML configuration for this Task +- `run_task_with_gold_unit__local__inhouse.py` - script to configure and launch this Task +- `hydra_configs/conf/example_with_gold_unit__local__inhouse.yaml` - YAML configuration for this Task - `data/simple/gold_units/gold_units_data.json` - configuration for form that will be used specifically for gold units - `data/simple/gold_units/gold_units_validation.py` - logic of validating worker's output in gold unit form diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx b/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx index 76d263546..503b799fc 100644 --- a/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx +++ b/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx @@ -37,7 +37,7 @@ refer to [using screening units](../using_screen_units), as Mephisto doesn't pay ### Things to note in the showcase: -- The `static_react_task` example is ran with the `onboarding_example` configuration enabled to ensure that onboarding page will be shown. +- The `static_react_task` example is ran with the `example_with_onboarding__local__inhouse` configuration enabled to ensure that onboarding page will be shown. - Worker "x" clicks the "Get Blocked" button and this doesn't allow the worker to progress - Worker "y" clicks the "Move To Main Task" button and this allows the worker to go to the main task. @@ -59,7 +59,7 @@ def handle_onboarding(onboarding_data): return True return False -@task_script(default_config_file="example.yaml") +@task_script(default_config_file="example__local__inhouse.yaml") def main(operator: Operator, cfg: DictConfig) -> None: ... shared_state = SharedStaticTaskState( @@ -72,9 +72,9 @@ def main(operator: Operator, cfg: DictConfig) -> None: ... ``` -### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/run_task.py) +### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/run_task__local__inhouse.py) -### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/hydra_configs/conf/onboarding_example.yaml) +### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml) Unlike Screening and Gold units, Onboarding expects that you set up a custom frontend compared to your main task. You want to provide workers with an in-depth exploration of your task up-front diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx b/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx index 833f25ad9..c0990c3a3 100644 --- a/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx +++ b/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx @@ -33,7 +33,7 @@ or on specific 'test' data you believe it's easier to validate on. ### Things to note in the showcase: -- The `remote_procedure/mnist` example is ran with the `screening_example` configuration enabled to ensure that screening units are generated. +- The `remote_procedure/mnist` example is ran with the `example_with_screening__local__inhouse` configuration enabled to ensure that screening units are generated. - When a worker goes to an assignment for the first time they see a screening unit. - Drawing a "3" gives the worker the passing qualification - Drawing any number other than "3" gives the worker the blocked qualification @@ -64,7 +64,7 @@ to register the required qualifications and the screening validation function. A shortened version of the run script for the video above looks like: ```python -# run_task.py +# run_task__local__inhouse.py def my_screening_unit_generator(): """ @@ -85,7 +85,7 @@ def validate_screening_unit(unit: Unit): return True return False -@task_script(default_config_file="example.yaml") +@task_script(default_config_file="example__local__inhouse.yaml") def main(operator: Operator, cfg: DictConfig) -> None: is_using_screening_units = cfg.mephisto.blueprint["use_screening_task"] tasks: List[Dict[str, Any]] = [{"isScreeningUnit": False}] * cfg.num_tasks @@ -108,9 +108,9 @@ def main(operator: Operator, cfg: DictConfig) -> None: ... ``` -### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/run_task.py) +### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/run_task__local__inhouse.py) -### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/hydra_configs/conf/screening_example.yaml) +### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/hydra_configs/conf/example_with_screening__local__inhouse.yaml) ### Simple example of `ScreeningComponent` diff --git a/docs/web/docs/guides/quickstart.md b/docs/web/docs/guides/quickstart.md index 117fb21dc..1e140389f 100644 --- a/docs/web/docs/guides/quickstart.md +++ b/docs/web/docs/guides/quickstart.md @@ -24,9 +24,9 @@ sidebar_position: 1 --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` -5. After the script finishes building, in your console output you will see URLs like this: `localhost:3000/?worker_id=x&assignment_id=1`. Each URL represents a unit of your Task. Copy them one-by-one into your browser, changing port from `3000` to `3001`. +5. After the script finishes building, in your console output you will see URLs like this: `localhost:3000/?worker_id=x&id=1`. Each URL represents a unit of your Task. Copy them one-by-one into your browser, changing port from `3000` to `3001`. 6. After completing all units, you will see your task automatically shut down 7. Now we are ready to review Task results. Run command: ```shell diff --git a/docs/web/docs/guides/tutorials/custom_react.md b/docs/web/docs/guides/tutorials/custom_react.md index b7451f1ca..98a58280b 100644 --- a/docs/web/docs/guides/tutorials/custom_react.md +++ b/docs/web/docs/guides/tutorials/custom_react.md @@ -26,7 +26,7 @@ Now that we're here, we should also set up your config file defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: task_source: ${task_dir}/webapp/build/bundle.js @@ -50,13 +50,13 @@ From the current directory, you should be able to execute the run script and get You can update the `task_name` and `link_task_source` values in your config and run the task like below ```bash -python run_task.py +python run_task__local__inhouse.py ``` or you can set them when you run the task: ```bash -python run_task.py mephisto.task.task_name=custom-react-tutorial-iterating mephisto.blueprint.link_task_source=true +python run_task__local__inhouse.py mephisto.task.task_name=custom-react-tutorial-iterating mephisto.blueprint.link_task_source=true ``` This will launch a simple task where an annotator is supposed to note a sentence as being good or bad. Clicking a button auto-submits the task. In the next sections we'll add other content. @@ -123,7 +123,7 @@ static_task_data=[ At this point you can run the task again. ```bash -python run_task.py mephisto.task.task_name=custom-react-tutorial-iterating +python run_task__local__inhouse.py mephisto.task.task_name=custom-react-tutorial-iterating ``` Note the first one you work on displays your new edited text. But what about the new `edited_by_requester` field? @@ -264,7 +264,7 @@ onClick={() => onSubmit({ rating: "bad", editedText: editedText })} Let's launch one last time. ``` -python run_task.py mephisto.task.task_name=custom-react-tutorial-iterating +python run_task__local__inhouse.py mephisto.task.task_name=custom-react-tutorial-iterating ``` And just like that we're able to see an input field: ![](/static_task_with_corrections_box.png) diff --git a/docs/web/docs/guides/tutorials/first_task.md b/docs/web/docs/guides/tutorials/first_task.md index 15f5c07ca..cd9e6a31f 100644 --- a/docs/web/docs/guides/tutorials/first_task.md +++ b/docs/web/docs/guides/tutorials/first_task.md @@ -27,23 +27,23 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` or without using Docker ```shell cd /mephisto/examples/form_composer_demo/ -python run_task.py +python run_task__local__inhouse.py ``` This will launch a local HTTP server with the task hosted, based on the default configuration options: ```yaml -# examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +# examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse ``` We'll dig into *how* this works [later](#33-default-abstraction-usage). @@ -55,13 +55,13 @@ By default, the task should be hosted at [http://localhost:3000](http://localhos This default is set by the `LocalArchitect`, which is used based on the `- /mephisto/architect: local` line above. Navigating to this address should show you the preview view for the task. -Actually being able to access this task is done by providing `worker_id` and `assignment_id` URL params, like `localhost:3000/?worker_id=x&assignment_id=1`. -The `MockProvider` interprets these to be a test worker, which you can use to try out tasks. +Actually being able to access this task is done by providing `worker_id` and `assignment_id` URL params, like `localhost:3000/?worker_id=x&id=1`. +The `InhouseProvider` interprets these to be a test worker, which you can use to try out tasks. Try navigating here and completing a task by selecting a value (no need to use the file upload). After hitting submit you'll note that the window alerts you to the data that was sent to the Mephisto backend. -To work on another task, you'll want to change the `assignment_id` in your url. This will let `Worker` "x" work on another task. +To work on another task, you'll want to change the `id` in your url. This will let `Worker` "x" work on another task. Complete and submit this too. If you try to work on another task, you'll note that the system states you've worked on the maximum number of tasks. @@ -138,7 +138,7 @@ For our given example task, the values we are using for these options are availa As a simple starting point, we can try launching the server on a different port. Right now the default is `3000`, but with the following command we can set that ourselves: ``` -python run_task.py mephisto.architect.port=1234 +python run_task__local__inhouse.py mephisto.architect.port=1234 ``` This should launch the same task, but now available on the port `1234` rather than `3000`. @@ -151,7 +151,7 @@ Once we see the task is running, we can shut down with `Ctrl-C`. ### 2.3 Using yaml configurations While it makes sense to update some parameters on the command line while iterating, generally it's easier to extend the `conf` files directly, then apply all of the options by overriding `conf`. -Try copying the `examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml` file into `examples/form_composer_demo/hydra_configs/conf/my_config.yaml`. +Try copying the `examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml` file into `examples/form_composer_demo/hydra_configs/conf/my_config.yaml`. Also file `examples/form_composer_demo/data/simple/task_data.json` into `examples/form_composer_demo/data/simple/my_task_data.json` ```yaml @@ -161,7 +161,7 @@ Also file `examples/form_composer_demo/data/simple/task_data.json` into `example defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: @@ -187,14 +187,14 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py conf=my_config + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py conf=my_config ``` or without using Docker ```bash cd /mephisto/examples/form_composer_demo/ -python run_task.py conf=my_config +python run_task__local__inhouse.py conf=my_config ``` Now you'll notice that Mephisto launches your task under the new task name: @@ -205,16 +205,16 @@ Now you'll notice that Mephisto launches your task under the new task name: > **A note on `task_name`s:** > The `task_name` parameter is particularly important for setting up workflows. Many of Mephisto's features are shared under a specific `task_name` namespace, including review flows and unit completion maximums per worker per namespace. Later [guides](../workflows) go more in-depth on best practices. -Navigating to a task (`localhost:3000/?worker_id=x&assignment_id=1` or woth Docker on `3001` port) should now show you a task loaded from a different data file. +Navigating to a task (`localhost:3000/?worker_id=x&id=1` or woth Docker on `3001` port) should now show you a task loaded from a different data file. Completing this task will lead Mephisto to shut down cleanly. You can now review this task with the review script again, this time providing the task name `My first own task`. ### 2.4 (optional) Launch your task live -So far we've been launching our task in a testing mode (with `local` architect and `mock` provider). +So far we've been launching our task in a testing mode (with `local` architect and `inhouse` provider). -To make your Task page accessible to the others (e.g. external workers), you will either need to obtain a publicly accessible static IP address for you machine, or launch the task with a non-`mock` human cloud platform and a non-`local` architect. +To make your Task page accessible to the others (e.g. external workers), you will either need to obtain a publicly accessible static IP address for you machine, or launch the task with a non-`inhouse` human cloud platform and a non-`local` architect. In the below examples we consider an EC2 architect and a few common providers. This configuration is different from the testing mode: - EC2 architect will build a temporary EC2 server in the cloud, which will: @@ -244,10 +244,10 @@ docker-compose -f docker/docker-compose.local.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py + python /mephisto/examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py ``` -Note that this command reads task parameters from task config file referenced within the `run_task_dynamic_ec2_mturk_sandbox.py` script. +Note that this command reads task parameters from task config file referenced within the `run_task_dynamic__ec2__mturk_sandbox.py` script. Generally, you would want to adjust task config file to your needs. You don't need to do this step *right now*, however it's important for understanding how to take one of these tasks live. @@ -298,14 +298,14 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py conf=my_config + python /mephisto/examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py conf=my_config ``` or without using Docker ```bash cd /mephisto/examples/form_composer_demo/ -python run_task_dynamic_ec2_mturk_sandbox.py conf=my_config +python run_task_dynamic__ec2__mturk_sandbox.py conf=my_config ``` Mephisto should print out links to view your task on the mturk sandbox, @@ -334,7 +334,7 @@ docker-compose -f docker/docker-compose.local.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py + python /mephisto/examples/form_composer_demo/run_task_dynamic__ec2__prolific.py ``` The other steps are similar to launching on MTurk, just task config file is a bit different. diff --git a/docs/web/docs/guides/tutorials/worker_controls.md b/docs/web/docs/guides/tutorials/worker_controls.md index 92a362c8e..a6c22691e 100644 --- a/docs/web/docs/guides/tutorials/worker_controls.md +++ b/docs/web/docs/guides/tutorials/worker_controls.md @@ -21,14 +21,14 @@ cp -r examples/static_react_task/ tmp/onboarding_tutorial/ cd tmp/onboarding_tutorial/ ``` -The key for using onboarding is the `onboarding_qualification`, as we can see in `onboarding_example.yaml`: +The key for using onboarding is the `onboarding_qualification`, as we can see in `example_with_onboarding__local__inhouse.yaml`: ```yaml # hydra_configs/conf/onboarding_example.yaml @package _global_ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: ... @@ -37,7 +37,7 @@ mephisto: ... task_name: onboarding-tutorial-iterating # Add a custom task name to group the results ``` -When Mephisto sees the `mephisto.blueprint.onboarding_qualification` argument, it adds a step to the task that requires workers pass onboarding in order to move on to the main task. Why don't we see it in action? You can launch the onboarding version of this task with `python run_task.py conf=onboarding_example`. Once launched, you can follow through to [`localhost:3000/?worker_id=x&assignment_id=1`](localhost:3000/?worker_id=x&assignment_id=1). +When Mephisto sees the `mephisto.blueprint.onboarding_qualification` argument, it adds a step to the task that requires workers pass onboarding in order to move on to the main task. Why don't we see it in action? You can launch the onboarding version of this task with `python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse`. Once launched, you can follow through to [`localhost:3000/?worker_id=x&id=1`](localhost:3000/?worker_id=x&id=1). You should be greeted with a panel claiming it is the onboarding task, suggesting that you click a button to move to the main task. Clicking this button puts you into the example static react task. It's not a very informative task as is, so let's add some details into it! Shut down the server and we'll get back to it @@ -156,7 +156,7 @@ function OnboardingComponent({ onboardingData, onSubmit }) { ); } ``` -With this, let's launch again with `python run_task.py conf=onboarding_example`. Once launched, you can follow through to [`localhost:3000/?worker_id=y&assignment_id=1`](localhost:3000/?worker_id=y&assignment_id=1). **Note** that now we're accessing as worker `y`. We do this because worker `x` has already completed onboarding for our task name, and so they'd skip it! Onboarding is only surfaced the _first time_ a worker completes a task by a given `onboarding_qualification`. +With this, let's launch again with `python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse`. Once launched, you can follow through to [`localhost:3000/?worker_id=y&assignment_id=1`](localhost:3000/?worker_id=y&id=1). **Note** that now we're accessing as worker `y`. We do this because worker `x` has already completed onboarding for our task name, and so they'd skip it! Onboarding is only surfaced the _first time_ a worker completes a task by a given `onboarding_qualification`. ![](/tutorial_onboarding_new_interface.png) @@ -207,7 +207,7 @@ And that's it! Let's move forward to test that this works. ### 2.4 Testing -Let's launch again with `python run_task.py conf=onboarding_example`. Once launched, we should open a tab for [`worker a`](localhost:3000/?worker_id=a&assignment_id=1) and [`worker b`](localhost:3000/?worker_id=b&assignment_id=1). Complete the onboarding correctly as one worker, and assert that you're able to make it through to the task! Complete it incorrectly as the other, and note that you are not qualified to work on the task. With this, you have a functioning basic onboarding task! +Let's launch again with `python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse`. Once launched, we should open a tab for [`worker a`](localhost:3000/?worker_id=a&id=1) and [`worker b`](localhost:3000/?worker_id=b&id=1). Complete the onboarding correctly as one worker, and assert that you're able to make it through to the task! Complete it incorrectly as the other, and note that you are not qualified to work on the task. With this, you have a functioning basic onboarding task! ## 3. Reviews with onboarding diff --git a/docs/web/docs/guides/tutorials/workflows.md b/docs/web/docs/guides/tutorials/workflows.md index 1c5c7cd68..57b79591f 100644 --- a/docs/web/docs/guides/tutorials/workflows.md +++ b/docs/web/docs/guides/tutorials/workflows.md @@ -36,7 +36,7 @@ Generally it's best to put the `task_name` _into_ your Hydra `.yaml` config and defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: ... @@ -71,7 +71,7 @@ This also means you can go back and find the configuration details for a specifi For complex tasks with many configuration arguments, we make it possible to add arguments to your run script to simplify your workflows and allow for code reuse. For instance, say you had the following script: ```python -# examples/static_react_task/run_task.py +# examples/static_react_task/run_task__local__inhouse.py from mephisto.operations.operator import Operator from mephisto.tools.scripts import task_script, build_and_return_custom_bundle from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import ( @@ -81,7 +81,7 @@ from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint impo from omegaconf import DictConfig -@task_script(default_config_file="example") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: def onboarding_always_valid(onboarding_data): return True diff --git a/docs/web/static/python_api/mephisto/abstractions/blueprints.html b/docs/web/static/python_api/mephisto/abstractions/blueprints.html index 38a535418..8e02eded1 100644 --- a/docs/web/static/python_api/mephisto/abstractions/blueprints.html +++ b/docs/web/static/python_api/mephisto/abstractions/blueprints.html @@ -126,7 +126,7 @@

SharedTaskState

Blueprint Mixins

-

Blueprints sometimes share some component functionality that may be useful across a multitude of tasks. We capture these in mixins. Mephisto is able to recognize certain mixins in order to complete additional operations, however custom mixins may help cut down on boiler plate in common run_task.py scripts. As your tasks mature, we suggest utilizing blueprint mixins to share common workflows and design patterns you observe.

+

Blueprints sometimes share some component functionality that may be useful across a multitude of tasks. We capture these in mixins. Mephisto is able to recognize certain mixins in order to complete additional operations, however custom mixins may help cut down on boiler plate in common run_task__local__inhouse.py scripts. As your tasks mature, we suggest utilizing blueprint mixins to share common workflows and design patterns you observe.

OnboardingRequired

@@ -346,4 +346,4 @@

MockBlueprint

} }); - \ No newline at end of file + diff --git a/examples/README.md b/examples/README.md index fa279b55e..dae50e558 100644 --- a/examples/README.md +++ b/examples/README.md @@ -18,16 +18,16 @@ Let's launch Mephisto example tasks, starting from the easiest one A simple project with HTML-based UI task template [simple_static_task](/examples/simple_static_task) -- Default config file: [/examples/simple_static_task/hydra_configs/conf/example.yaml] +- Default config file: [/examples/simple_static_task/hydra_configs/conf/example__local__inhouse.yaml] - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/simple_static_task/static_test_script.py + python /mephisto/examples/simple_static_task/run_task__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1) +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1) - Browser page should display an image, instruction, select and file inputs, and a submit button. --- @@ -36,16 +36,16 @@ A simple project with HTML-based UI task template [simple_static_task](/examples A simple project with React-based UI task template [static_react_task](/examples/static_react_task) -- Default config file: [example.yaml](/examples/static_react_task/hydra_configs/conf/example.yaml). +- Default config file: [example.yaml](/examples/static_react_task/hydra_configs/conf/example__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/static_react_task/run_task.py + python /mephisto/examples/static_react_task/run_task__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - Browser page should display an instruction line and two buttons (green and red). --- @@ -64,9 +64,9 @@ A more complex example featuring worker-generated dynamic input: [mnist](/exampl apt install curl && \ pip install grafana torch pillow numpy && \ mephisto metrics install && \ - python /mephisto/examples/remote_procedure/mnist/run_task.py + python /mephisto/examples/remote_procedure/mnist/run_task__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - Browser page should display instructions and a layout with 3 rectangle fields for drawing numbers with a mouse, each field having inputs at the bottom. --- @@ -81,32 +81,32 @@ There are three FormComposer examples: This is a single-version form containing no variable tokens. -- Default config file: [example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml). +- Default config file: [example__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - You should see a Bootstrap-themed form with fields and sections corresponding to the form config in its [task_data.json](/examples/form_composer_demo/data/simple/task_data.json) file. ##### 3.2. Simple form with Gold Units Same example as 3.1, but one of the Units will be marked as Gold. -- Default config file: [example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_gold_unit.yaml). +- Default config file: [example_with_gold_unit__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/example_with_gold_unit__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_with_gold_unit.py + python /mephisto/examples/form_composer_demo/run_task_with_gold_unit__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - You should see a Bootstrap-themed form with fields and sections corresponding to the form config in its [task_data.json](/examples/form_composer_demo/data/simple/task_data.json) file and Gold units made from [gold_units_data.json](/examples/form_composer_demo/data/simple/gold_units/gold_units_data.json) with special validation for them from [gold_units_validation.py](/examples/form_composer_demo/data/simple/gold_units/gold_units_validation.py). @@ -115,16 +115,16 @@ with special validation for them from [gold_units_validation.py](/examples/form_ Same example as 3.1, but before Units a Worker will see Onboarding widget. -- Default config file: [example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_oboarding.yaml). +- Default config file: [example_with_onboarding__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_with_onboarding.py + python /mephisto/examples/form_composer_demo/run_task_with_onboarding__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - You should see Onboarding widget and after answering the onboarding question you will be blocked or passed further and will see a Bootstrap-themed form with fields and sections corresponding to the form config in its [task_data.json](/examples/form_composer_demo/data/simple/task_data.json) file. @@ -132,16 +132,16 @@ a Bootstrap-themed form with fields and sections corresponding to the form confi Same example as 3.1, but first time before real Units a Worker will see Screening units. -- Default config file: [example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_screening.yaml). +- Default config file: [example_with_screening__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/example_with_screening__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_with_screening.py + python /mephisto/examples/form_composer_demo/run_task_with_screening__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - You should see Onboarding widget and after answering the onboarding question you will be blocked or passed further and will see a Bootstrap-themed form with fields and sections corresponding to the form config in its [task_data.json](/examples/form_composer_demo/data/simple/task_data.json) file and Screening units made from [screening_units_data.json](/examples/form_composer_demo/data/simple/screening_units/screening_units_data.json) @@ -151,37 +151,37 @@ with special validation for them from [screening_units_validation.py](/examples/ Same example as 3.1, but after completing a Unit, a Worker will see a form below the main form where they can leave their opinion and screenshots. -- Default config file: [example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_oboarding.yaml). +- Default config file: [example__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_with_worker_opinion.py + python /mephisto/examples/form_composer_demo/run_task_with_worker_opinion__local__inhouse.py ``` -- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&assignment_id=1](http://localhost:3001/?worker_id=x&assignment_id=1). +- Browser page (for the first task unit): [http://localhost:3001/?worker_id=x&id=1](http://localhost:3001/?worker_id=x&id=1). - You should see a Bootstrap-themed form with fields and sections corresponding to the form config in its [task_data.json](/examples/form_composer_demo/data/simple/task_data.json) file. ##### 3.6. Dynamic form Dynamic form means a multi-version form, where versions are generated by varying values of special tokens embedded into the form config. -- Default config file: [dynamic_example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/dynamic_example_local_mock.yaml). +- Default config file: [dynamic_example__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml). - Launch command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic.py + python /mephisto/examples/form_composer_demo/run_task_dynamic__local__inhouse.py ``` - Browser page is same as for the Simple form example -There are variations of dynamic form config that use different providers. To try that, change `run_task_dynamic.py` in the launch command to: +There are variations of dynamic form config that use different providers. To try that, change `run_task_dynamic__local__inhouse.py` in the launch command to: -- `run_task_dynamic_ec2_prolific.py` for Prolific (requires valid EC2 credentials) -- `run_task_dynamic_ec2_mturk_sandbox.py` for Mechanical Turk sandbox (requires valid EC2 credentials) +- `run_task_dynamic__ec2__prolific.py` for Prolific (requires valid EC2 credentials) +- `run_task_dynamic__ec2__mturk_sandbox.py` for Mechanical Turk sandbox (requires valid EC2 credentials) ##### 3.7. Dynamic form with presigned URLs @@ -191,7 +191,7 @@ This example builds further upon the Dynamic form example. Here we use presigned - Required: valid AWS credentials: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` - Required: a private S3 bucket with files that you will embed in the example form (either replace dummy URLs in the configs by hand, or automatically generate new ones with `mephisto form_composer config` command) - Optional: `S3_URL_EXPIRATION_MINUTES` (default value is 60 minutes) -- Default config file: [dynamic_example_local_mock.yaml](/examples/form_composer_demo/hydra_configs/conf/dynamic_example_local_mock.yaml). +- Default config file: [dynamic_example__local__inhouse.yaml](/examples/form_composer_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml). - Create config: see all options for `form_composer config` command [here](/mephisto/generators/form_composer/README.md#using-formcomposerconfig-utility). Example command: ```shell docker-compose -f docker/docker-compose.dev.yml run \ @@ -206,7 +206,7 @@ This example builds further upon the Dynamic form example. Here we use presigned --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_presigned_urls.py + python /mephisto/examples/form_composer_demo/run_task_dynamic_presigned_urls__ec2__prolific.py ``` @@ -214,7 +214,7 @@ This example builds further upon the Dynamic form example. Here we use presigned Putting it altogether, let's prepare and launch a task featuring a form containing one embedded file plus a few other fields. Here we'll assume working in directory `/mephisto/examples/form_composer_demo/data/dynamic_presigned_urls`. -- Adjust `dynamic_presigned_urls_example_ec2_prolific.yaml` task config as needed +- Adjust `dynamic_example_presigned_urls__ec2__prolific.yaml` task config as needed - Create `unit_config.json` file to define your form fields and layout - it should contain a token named `file_location` - for more details see `mephisto/generators/form_composer/README.md` @@ -250,7 +250,7 @@ Putting it altogether, let's prepare and launch a task featuring a form containi ``` - Launch your task: ```shell - cd /mephisto/examples/form_composer_demo && python run_task_dynamic_presigned_urls_ec2_prolific.py + cd /mephisto/examples/form_composer_demo && python run_task_dynamic_presigned_urls__ec2__prolific.py ``` - After the Task is completed by all workers, launch task review app and acces it at [http://localhost:8081](http://localhost:8081) (for more details see `mephisto/review_app/README.md`): ```shell diff --git a/examples/form_composer_demo/README.md b/examples/form_composer_demo/README.md index dc6ef40ac..2a057d2f8 100644 --- a/examples/form_composer_demo/README.md +++ b/examples/form_composer_demo/README.md @@ -7,17 +7,16 @@ These form-based questionnaires are example of FormComposer task generator. 1. In repo root, launch containers: `docker-compose -f docker/docker-compose.dev.yml up` 2. SSH into running container to run server: `docker exec -it mephisto_dc bash` 3. Inside the container, go to FormComposer examples directory: `cd /mephisto/examples/form_composer_demo` -4. Inside the examples directory, run a desired example with one of these commands: - - Simple form with Mock provider: `python ./run_task.py` +4. Inside the `examples` directory, run a desired example with one of these commands: - Simple form with Inhouse provider: `python ./run_task__local__inhouse.py` - - Dynamic form with Mock provider: `python ./run_task_dynamic.py` - - Dynamic form with Mturk on EC2: `python ./run_task_dynamic_ec2_mturk_sandbox.py` - - Dynamic form with Prolific on EC2: `python ./run_task_dynamic_ec2_prolific.py` - - Dynamic form with Presigned URLs: `python ./run_task_dynamic_presigned_urls_ec2_prolific.py` - - Simple form with Gold Units: `python ./run_task_with_gold_unit.py` - - Simple form with Onboarding: `python ./run_task_with_onboarding.py` - - Simple form with Screening: `python ./run_task_with_screening.py` - - Simple form with Worker Opinion: `python ./run_task_with_worker_opinion.py` + - Dynamic form with Inhouse provider: `python ./run_task_dynamic__local__inhouse.py` + - Dynamic form with Mturk on EC2: `python ./run_task_dynamic__ec2__mturk_sandbox.py` + - Dynamic form with Prolific on EC2: `python ./run_task_dynamic__ec2__prolific.py` + - Dynamic form with Presigned URLs: `python ./run_task_dynamic_presigned_urls__ec2__prolific.py` + - Simple form with Gold Units: `python ./run_task_with_gold_unit__local__inhouse.py` + - Simple form with Onboarding: `python ./run_task_with_onboarding__local__inhouse.py` + - Simple form with Screening: `python ./run_task_with_screening__local__inhouse.py` + - Simple form with Worker Opinion: `python ./run_task_with_worker_opinion__local__inhouse.py` --- @@ -58,5 +57,5 @@ mephisto form_composer config --directory /mephisto/examples/form_composer_demo/ # 2b. Run the Task cd /mephisto/examples/form_composer_demo -python ./run_task_dynamic_presigned_urls_ec2_prolific.py +python ./run_task_dynamic_presigned_urls__ec2__prolific.py ``` diff --git a/examples/form_composer_demo/hydra_configs/conf/dynamic_example_ec2_mturk_sandbox.yaml b/examples/form_composer_demo/hydra_configs/conf/dynamic_example__ec2__mturk_sandbox.yaml similarity index 100% rename from examples/form_composer_demo/hydra_configs/conf/dynamic_example_ec2_mturk_sandbox.yaml rename to examples/form_composer_demo/hydra_configs/conf/dynamic_example__ec2__mturk_sandbox.yaml diff --git a/examples/form_composer_demo/hydra_configs/conf/dynamic_example_ec2_prolific.yaml b/examples/form_composer_demo/hydra_configs/conf/dynamic_example__ec2__prolific.yaml similarity index 100% rename from examples/form_composer_demo/hydra_configs/conf/dynamic_example_ec2_prolific.yaml rename to examples/form_composer_demo/hydra_configs/conf/dynamic_example__ec2__prolific.yaml diff --git a/examples/form_composer_demo/hydra_configs/conf/dynamic_example_local_mock.yaml b/examples/form_composer_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml similarity index 96% rename from examples/form_composer_demo/hydra_configs/conf/dynamic_example_local_mock.yaml rename to examples/form_composer_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml index 49d25bc78..3994d3506 100644 --- a/examples/form_composer_demo/hydra_configs/conf/dynamic_example_local_mock.yaml +++ b/examples/form_composer_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: diff --git a/examples/form_composer_demo/hydra_configs/conf/dynamic_presigned_urls_example_ec2_prolific.yaml b/examples/form_composer_demo/hydra_configs/conf/dynamic_example_presigned_urls__ec2__prolific.yaml similarity index 100% rename from examples/form_composer_demo/hydra_configs/conf/dynamic_presigned_urls_example_ec2_prolific.yaml rename to examples/form_composer_demo/hydra_configs/conf/dynamic_example_presigned_urls__ec2__prolific.yaml diff --git a/examples/form_composer_demo/hydra_configs/conf/example_local_inhouse.yaml b/examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml similarity index 100% rename from examples/form_composer_demo/hydra_configs/conf/example_local_inhouse.yaml rename to examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml diff --git a/examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml b/examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml deleted file mode 100644 index f4f45e10c..000000000 --- a/examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +++ /dev/null @@ -1,26 +0,0 @@ -#@package _global_ - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -defaults: - - /mephisto/blueprint: static_react_task - - /mephisto/architect: local - - /mephisto/provider: mock - -mephisto: - blueprint: - data_json: ${task_dir}/data/simple/task_data.json - task_source: ${task_dir}/webapp/build/bundle.js - task_source_review: ${task_dir}/webapp/build/bundle.review.js - link_task_source: false - extra_source_dir: ${task_dir}/webapp/src/static - units_per_assignment: 2 - task: - task_name: "Sample Questionnaire" - task_title: "Example how to easily create simple form-based Tasks" - task_description: "In this Task, we use FormComposer feature." - task_reward: 0 - task_tags: "test,simple,form,form-composer" - force_rebuild: true diff --git a/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_gold_unit.yaml b/examples/form_composer_demo/hydra_configs/conf/example_with_gold_unit__local__inhouse.yaml similarity index 97% rename from examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_gold_unit.yaml rename to examples/form_composer_demo/hydra_configs/conf/example_with_gold_unit__local__inhouse.yaml index ff3dfabb3..0dd48447b 100644 --- a/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_gold_unit.yaml +++ b/examples/form_composer_demo/hydra_configs/conf/example_with_gold_unit__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: diff --git a/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_oboarding.yaml b/examples/form_composer_demo/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml similarity index 96% rename from examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_oboarding.yaml rename to examples/form_composer_demo/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml index a6f7dc950..ed0d88938 100644 --- a/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_oboarding.yaml +++ b/examples/form_composer_demo/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: diff --git a/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_screening.yaml b/examples/form_composer_demo/hydra_configs/conf/example_with_screening__local__inhouse.yaml similarity index 97% rename from examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_screening.yaml rename to examples/form_composer_demo/hydra_configs/conf/example_with_screening__local__inhouse.yaml index c8f3f4e5f..4e636068e 100644 --- a/examples/form_composer_demo/hydra_configs/conf/example_local_mock_with_screening.yaml +++ b/examples/form_composer_demo/hydra_configs/conf/example_with_screening__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: diff --git a/examples/form_composer_demo/run_task.py b/examples/form_composer_demo/run_task.py deleted file mode 100644 index 4169435e6..000000000 --- a/examples/form_composer_demo/run_task.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from omegaconf import DictConfig - -from mephisto.operations.operator import Operator -from mephisto.tools.building_react_apps import examples -from mephisto.tools.scripts import task_script - - -@task_script(default_config_file="example_local_mock") -def main(operator: Operator, cfg: DictConfig) -> None: - examples.build_form_composer_simple( - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - operator.launch_task_run(cfg.mephisto) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -if __name__ == "__main__": - main() diff --git a/examples/form_composer_demo/run_task__local__inhouse.py b/examples/form_composer_demo/run_task__local__inhouse.py index c88b1c495..2a403ec9c 100644 --- a/examples/form_composer_demo/run_task__local__inhouse.py +++ b/examples/form_composer_demo/run_task__local__inhouse.py @@ -11,7 +11,7 @@ from mephisto.tools.scripts import task_script -@task_script(default_config_file="example_local_inhouse") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_form_composer_simple( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py b/examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py similarity index 98% rename from examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py rename to examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py index 2f161ada8..c3ad65134 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py +++ b/examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py @@ -101,7 +101,7 @@ def _generate_preview_html(): f.write(preview_template) -@task_script(default_config_file="dynamic_example_ec2_mturk_sandbox") +@task_script(default_config_file="dynamic_example__ec2__mturk_sandbox") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_form_composer_dynamic_ec2_mturk_sandbox( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic__ec2__prolific.py similarity index 97% rename from examples/form_composer_demo/run_task_dynamic_ec2_prolific.py rename to examples/form_composer_demo/run_task_dynamic__ec2__prolific.py index 734b43175..77b4e86a4 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic__ec2__prolific.py @@ -51,7 +51,7 @@ def _generate_data_json_config(): ) -@task_script(default_config_file="dynamic_example_ec2_prolific") +@task_script(default_config_file="dynamic_example__ec2__prolific") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_form_composer_dynamic_ec2_prolific( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/form_composer_demo/run_task_dynamic.py b/examples/form_composer_demo/run_task_dynamic__local__inhouse.py similarity index 97% rename from examples/form_composer_demo/run_task_dynamic.py rename to examples/form_composer_demo/run_task_dynamic__local__inhouse.py index 1d4ad05e0..3190e1093 100644 --- a/examples/form_composer_demo/run_task_dynamic.py +++ b/examples/form_composer_demo/run_task_dynamic__local__inhouse.py @@ -46,7 +46,7 @@ def _generate_task_data_json_config(): ) -@task_script(default_config_file="dynamic_example_local_mock") +@task_script(default_config_file="dynamic_example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_form_composer_dynamic( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_presigned_urls__ec2__prolific.py similarity index 97% rename from examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py rename to examples/form_composer_demo/run_task_dynamic_presigned_urls__ec2__prolific.py index 131c69275..ebf0f0bf7 100644 --- a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_presigned_urls__ec2__prolific.py @@ -57,7 +57,7 @@ def _generate_data_json_config(): ) -@task_script(default_config_file="dynamic_presigned_urls_example_ec2_prolific") +@task_script(default_config_file="dynamic_example_presigned_urls__ec2__prolific") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_form_composer_dynamic_presigned_urls_ec2_prolific( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/form_composer_demo/run_task_with_gold_unit.py b/examples/form_composer_demo/run_task_with_gold_unit__local__inhouse.py similarity index 96% rename from examples/form_composer_demo/run_task_with_gold_unit.py rename to examples/form_composer_demo/run_task_with_gold_unit__local__inhouse.py index e02cffc24..08eca5fcc 100644 --- a/examples/form_composer_demo/run_task_with_gold_unit.py +++ b/examples/form_composer_demo/run_task_with_gold_unit__local__inhouse.py @@ -46,7 +46,7 @@ def _get_gold_data() -> List[Dict[str, Any]]: return gold_data -@task_script(default_config_file="example_local_mock_with_gold_unit") +@task_script(default_config_file="example_with_gold_unit__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: # 1. Build packages examples.build_form_composer_simple_with_gold_unit( diff --git a/examples/form_composer_demo/run_task_with_onboarding.py b/examples/form_composer_demo/run_task_with_onboarding__local__inhouse.py similarity index 94% rename from examples/form_composer_demo/run_task_with_onboarding.py rename to examples/form_composer_demo/run_task_with_onboarding__local__inhouse.py index 0b3167518..b349f1d69 100644 --- a/examples/form_composer_demo/run_task_with_onboarding.py +++ b/examples/form_composer_demo/run_task_with_onboarding__local__inhouse.py @@ -21,7 +21,7 @@ def _handle_onboarding(onboarding_data: dict) -> bool: return False -@task_script(default_config_file="example_local_mock_with_oboarding") +@task_script(default_config_file="example_with_onboarding__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: # 1. Build packages examples.build_form_composer_simple_with_onboarding( diff --git a/examples/form_composer_demo/run_task_with_screening.py b/examples/form_composer_demo/run_task_with_screening__local__inhouse.py similarity index 96% rename from examples/form_composer_demo/run_task_with_screening.py rename to examples/form_composer_demo/run_task_with_screening__local__inhouse.py index f8d3f1326..5577c024c 100644 --- a/examples/form_composer_demo/run_task_with_screening.py +++ b/examples/form_composer_demo/run_task_with_screening__local__inhouse.py @@ -46,7 +46,7 @@ def _screening_unit_factory() -> dict: yield screening_data -@task_script(default_config_file="example_local_mock_with_screening") +@task_script(default_config_file="example_with_screening__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: # 1. Build packages examples.build_form_composer_simple_with_screening( diff --git a/examples/form_composer_demo/run_task_with_worker_opinion.py b/examples/form_composer_demo/run_task_with_worker_opinion__local__inhouse.py similarity index 93% rename from examples/form_composer_demo/run_task_with_worker_opinion.py rename to examples/form_composer_demo/run_task_with_worker_opinion__local__inhouse.py index 769c0502b..f4a6ccbca 100644 --- a/examples/form_composer_demo/run_task_with_worker_opinion.py +++ b/examples/form_composer_demo/run_task_with_worker_opinion__local__inhouse.py @@ -13,7 +13,7 @@ from mephisto.tools.scripts import task_script -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: os.environ["REACT_APP__WITH_WORKER_OPINION"] = "true" examples.build_form_composer_simple( diff --git a/examples/parlai_chat_task_demo/README.md b/examples/parlai_chat_task_demo/README.md index a4bec6707..f698ddbcd 100644 --- a/examples/parlai_chat_task_demo/README.md +++ b/examples/parlai_chat_task_demo/README.md @@ -21,26 +21,26 @@ pip install parlai The baseline chat example can be run from this directory with: ```console -python run_task.py +python run_task__local__inhouse.py ``` You can also run an example that has onboarding set up with: ```console -python run_task.py conf=onboarding_example +python run_task__local__inhouse.py conf=onboarding_example__local__inhouse ``` Which is mostly a wrapper around adding an onboarding qualification, which you can do manually for any of the other configurations. ```console -python run_task.py conf=... mephisto.blueprint.onboarding_qualification=test_onboard_qualification +python run_task__local__inhouse.py conf=... mephisto.blueprint.onboarding_qualification=test_onboard_qualification ``` Further, you can try running a task using a prebuilt customized frontend bundle (built from the `webapp` directory) with: ```console -python run_task.py conf=custom_prebuilt +python run_task__local__inhouse.py conf=custom_prebuilt__local__inhouse ``` Further, you can try running a task using an auto-building task (built from the `custom_simple` directory) with: ```console -python run_task.py conf=custom_simple +python run_task__local__inhouse.py conf=custom_simple__local__inhouse ``` ### Common ParlAI blueprint argument overrides @@ -107,10 +107,10 @@ Custom frontend bundles can be provided that override the view of how the ParlAI ## Simple chat collection (no custom frontend) If you are able to provide your workers enough context just using a task description and perhaps a message in the chat thread with directions, you should be able to work on a task without a custom frontend. In order to get started on a task like this, you'll likely do the following: -1. Copy the `demo_worlds.py`, `run_task.py`, and `task_description.html` files to a new directory for your task. This generally would go in the project directory you are launching tasks for, but you can use `mephisto/tasks` if you're just experimenting. +1. Copy the `demo_worlds.py`, `run_task__local__inhouse.py`, and `task_description.html` files to a new directory for your task. This generally would go in the project directory you are launching tasks for, but you can use `mephisto/tasks` if you're just experimenting. 2. Update any task-related variables in your `conf/my_new_config.yaml` file to make sense for your task. 3. Update your worlds file to specify the kind of conversation you are creating between agents. -4. Run `run_task.py` to pilot your task over localhost. You can use different `worker_id` URL parameters in different windows to play the part of multiple workers at the same time. +4. Run `run_task__local__inhouse.py` to pilot your task over localhost. You can use different `worker_id` URL parameters in different windows to play the part of multiple workers at the same time. 5. Repeat 3 & 4 until you're happy with your task. 6. Launch a small batch on a crowd provider to see how real workers handle your task. 7. Iterate more - use a review script (like the one present in `examples/simple_static_task/examine_results`) to make it easy to see what data you're getting. @@ -125,7 +125,7 @@ If your task needs additional input beyond simple forms (tutorial TODO, see the 3. Update your worlds file to specify the kind of conversation you are creating between agents. 4. Update `app.jsx` to alter parts of your frontend job to do what you want. 5. Rebuild your frontend with `npm install; npm run dev` -6. Run `run_task.py` to pilot your task over localhost. You can use different `worker_id` URL parameters in different windows to play the part of multiple workers at the same time. +6. Run `run_task__local__inhouse.py` to pilot your task over localhost. You can use different `worker_id` URL parameters in different windows to play the part of multiple workers at the same time. 7. Repeat 3 - 6 until you're happy with your task. 8. Launch a small batch on a crowd provider to see how real workers handle your task. 9. Iterate more - use a review script (like the one present in `examples/simple_static_task/examine_results`) to make it easy to see what data you're getting. @@ -133,4 +133,7 @@ If your task needs additional input beyond simple forms (tutorial TODO, see the You may also find success using the options for the simple custom frontend functionality, described in the "Simple custom frontend bundles" section. -If you do require frontend customization, we recommend using [React Dev Tools](https://chrome.google.com/webstore/detail/react-developer-tools/fmkadmapgofadopljbjfkapdkoienihi?hl=en) to inspect the specific elements you want to change and debug your frontend as you work with it. Note that after rebuilding your frontend (by using `npm install; npm run dev`) you may need to do a force refresh (shift-cmd-R in chrome) to ensure you load the new version of your bundle. +If you do require frontend customization, we recommend using [React Dev Tools](https://chrome.google.com/webstore/detail/react-developer-tools/fmkadmapgofadopljbjfkapdkoienihi?hl=en) +to inspect the specific elements you want to change and debug your frontend as you work with it. +Note that after rebuilding your frontend (by using `npm install; npm run dev`) you may need to do a force refresh (shift-cmd-R in chrome) +to ensure you load the new version of your bundle. diff --git a/examples/parlai_chat_task_demo/custom_input_and_messages/README.md b/examples/parlai_chat_task_demo/custom_input_and_messages/README.md index 50a4a0850..082454d88 100644 --- a/examples/parlai_chat_task_demo/custom_input_and_messages/README.md +++ b/examples/parlai_chat_task_demo/custom_input_and_messages/README.md @@ -6,7 +6,7 @@ **NOTE: These instructions are from before Hydra integration. They will need to be upated after Hydra integration** -To run the example in this directory, modify `run_task.py` in the parent directory replacing: +To run the example in this directory, modify `run_task__local__inhouse.py` in the parent directory replacing: ``` source_dir_path = f"{TASK_DIRECTORY}/custom_simple" @@ -20,7 +20,7 @@ source_dir_path = f"{TASK_DIRECTORY}/custom_input_and_messages" You can then run the task with: ``` -python run_task.py --build-custom-task True +python run_task__local__inhouse.py --build-custom-task True ``` --- diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/_base.yaml similarity index 94% rename from examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml rename to examples/parlai_chat_task_demo/hydra_configs/conf/_base.yaml index 9c14009a6..ae59af518 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/base.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/_base.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: parlai_chat - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: world_file: ${task_dir}/demo_worlds.py diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/custom_prebuilt.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/custom_prebuilt__local__inhouse.yaml similarity index 98% rename from examples/parlai_chat_task_demo/hydra_configs/conf/custom_prebuilt.yaml rename to examples/parlai_chat_task_demo/hydra_configs/conf/custom_prebuilt__local__inhouse.yaml index 086f28bfb..7584b6035 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/custom_prebuilt.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/custom_prebuilt__local__inhouse.yaml @@ -5,7 +5,8 @@ # LICENSE file in the root directory of this source tree. defaults: - - base + - _base + mephisto: blueprint: custom_source_bundle: ${task_dir}/webapp/build/bundle.js diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/custom_simple.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/custom_simple__local__inhouse.yaml similarity index 98% rename from examples/parlai_chat_task_demo/hydra_configs/conf/custom_simple.yaml rename to examples/parlai_chat_task_demo/hydra_configs/conf/custom_simple__local__inhouse.yaml index 067370da2..3d2c45d7f 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/custom_simple.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/custom_simple__local__inhouse.yaml @@ -5,7 +5,8 @@ # LICENSE file in the root directory of this source tree. defaults: - - base + - _base + mephisto: blueprint: custom_source_dir: ${task_dir}/custom_simple diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/example.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/example.yaml deleted file mode 100644 index 363b1d901..000000000 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/example.yaml +++ /dev/null @@ -1,18 +0,0 @@ -#@package _global_ - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -defaults: - - base -mephisto: - task: - task_name: parlai-chat-example - task_title: "Test ParlAI Chat Task" - task_description: > - This is a simple chat between two people - used to demonstrate the functionalities around using Mephisto - for ParlAI tasks. - task_reward: 0.3 - task_tags: "dynamic,chat,testing" diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/example__local__inhouse.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/example__local__inhouse.yaml index 44837d00a..fc9750e7a 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/example__local__inhouse.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/example__local__inhouse.yaml @@ -5,9 +5,7 @@ # LICENSE file in the root directory of this source tree. defaults: - - /mephisto/blueprint: parlai_chat - - /mephisto/architect: local - - /mephisto/provider: inhouse + - _base mephisto: blueprint: diff --git a/examples/parlai_chat_task_demo/hydra_configs/conf/onboarding_example.yaml b/examples/parlai_chat_task_demo/hydra_configs/conf/onboarding_example__local__inhouse.yaml similarity index 98% rename from examples/parlai_chat_task_demo/hydra_configs/conf/onboarding_example.yaml rename to examples/parlai_chat_task_demo/hydra_configs/conf/onboarding_example__local__inhouse.yaml index 479591980..80a45c6f3 100644 --- a/examples/parlai_chat_task_demo/hydra_configs/conf/onboarding_example.yaml +++ b/examples/parlai_chat_task_demo/hydra_configs/conf/onboarding_example__local__inhouse.yaml @@ -5,7 +5,8 @@ # LICENSE file in the root directory of this source tree. defaults: - - base + - _base + mephisto: blueprint: onboarding_qualification: test-parlai-chat-qualification diff --git a/examples/parlai_chat_task_demo/run_task.py b/examples/parlai_chat_task_demo/run_task.py deleted file mode 100644 index 73e03567e..000000000 --- a/examples/parlai_chat_task_demo/run_task.py +++ /dev/null @@ -1,64 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -import os -from dataclasses import dataclass -from dataclasses import field - -from omegaconf import DictConfig - -from mephisto.abstractions.blueprints.parlai_chat.parlai_chat_blueprint import ( - SharedParlAITaskState, -) -from mephisto.operations.hydra_config import build_default_task_config -from mephisto.operations.operator import Operator -from mephisto.tools.building_react_apps import examples -from mephisto.tools.scripts import task_script - - -@dataclass -class ParlAITaskConfig(build_default_task_config("example")): # type: ignore - num_turns: int = field( - default=3, - metadata={"help": "Number of turns before a conversation is complete"}, - ) - turn_timeout: int = field( - default=300, - metadata={ - "help": "Maximum response time before kicking " "a worker out, default 300 seconds", - }, - ) - - -@task_script(config=ParlAITaskConfig) -def main(operator: "Operator", cfg: DictConfig) -> None: - examples.build_parlai_chat_task_demo( - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - - world_opt = { - "num_turns": cfg.num_turns, - "turn_timeout": cfg.turn_timeout, - } - - custom_bundle_path = cfg.mephisto.blueprint.get("custom_source_bundle", None) - if custom_bundle_path is not None: - assert os.path.exists(custom_bundle_path), ( - "Must build the custom bundle with `npm install; npm run dev` from within " - f"the {cfg.task_dir}/webapp directory in order to demo a custom bundle " - ) - - world_opt["send_task_data"] = True - - shared_state = SharedParlAITaskState(world_opt=world_opt, onboarding_world_opt=world_opt) - - operator.launch_task_run(cfg.mephisto, shared_state) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -if __name__ == "__main__": - main() diff --git a/examples/remote_procedure/mnist/hydra_configs/conf/example_local_mock.yaml b/examples/remote_procedure/mnist/hydra_configs/conf/example__local__inhouse.yaml similarity index 93% rename from examples/remote_procedure/mnist/hydra_configs/conf/example_local_mock.yaml rename to examples/remote_procedure/mnist/hydra_configs/conf/example__local__inhouse.yaml index f3550fc2f..550a38574 100644 --- a/examples/remote_procedure/mnist/hydra_configs/conf/example_local_mock.yaml +++ b/examples/remote_procedure/mnist/hydra_configs/conf/example__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: remote_procedure - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: task_source: ${task_dir}/webapp/build/bundle.js @@ -18,7 +18,7 @@ mephisto: units_per_assignment: 1 task: allowed_concurrent: 1 - task_name: remote-procedure-test-task + task_name: remote-procedure-mnist task_title: "Provide feedback on our MNIST model" # NOTE you'll want to update your task description task_description: "You will draw digits. Try to fool our MNIST model, and provide us the correct label." diff --git a/examples/remote_procedure/mnist/hydra_configs/conf/screening_example_local_mock.yaml b/examples/remote_procedure/mnist/hydra_configs/conf/example_with_screening__local__inhouse.yaml similarity index 94% rename from examples/remote_procedure/mnist/hydra_configs/conf/screening_example_local_mock.yaml rename to examples/remote_procedure/mnist/hydra_configs/conf/example_with_screening__local__inhouse.yaml index c4c916d4f..d6d4cc1a7 100644 --- a/examples/remote_procedure/mnist/hydra_configs/conf/screening_example_local_mock.yaml +++ b/examples/remote_procedure/mnist/hydra_configs/conf/example_with_screening__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: remote_procedure - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: task_source: ${task_dir}/webapp/build/bundle.js @@ -19,7 +19,7 @@ mephisto: use_screening_task: true max_screening_units: 3 task: - task_name: remote-procedure-test-task + task_name: remote-procedure-mnist task_title: "Provide feedback on our MNIST model" # NOTE you'll want to update your task description task_description: "You will draw digits. Try to fool our MNIST model, and provide us the correct label." diff --git a/examples/remote_procedure/mnist/run_task.py b/examples/remote_procedure/mnist/run_task__local__inhouse.py similarity index 98% rename from examples/remote_procedure/mnist/run_task.py rename to examples/remote_procedure/mnist/run_task__local__inhouse.py index ac72b9cd4..a4c8f6191 100644 --- a/examples/remote_procedure/mnist/run_task.py +++ b/examples/remote_procedure/mnist/run_task__local__inhouse.py @@ -92,7 +92,7 @@ def _handle_with_model( } -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_remote_procedure_mnist( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/remote_procedure/mnist/webapp/cypress.config.js b/examples/remote_procedure/mnist/webapp/cypress.config.js index a8f21d574..9856cf734 100644 --- a/examples/remote_procedure/mnist/webapp/cypress.config.js +++ b/examples/remote_procedure/mnist/webapp/cypress.config.js @@ -8,7 +8,7 @@ module.exports = { video: false, e2e: { - baseUrl: "http://localhost:3000/?worker_id=x&assignment_id=1", + baseUrl: "http://localhost:3000/?worker_id=x&id=1", supportFile: false, }, }; diff --git a/examples/remote_procedure/mnist/webapp/cypress/e2e/remote_procedure_mnist.cy.js b/examples/remote_procedure/mnist/webapp/cypress/e2e/remote_procedure_mnist.cy.js index 0180d316f..acc141c5a 100644 --- a/examples/remote_procedure/mnist/webapp/cypress/e2e/remote_procedure_mnist.cy.js +++ b/examples/remote_procedure/mnist/webapp/cypress/e2e/remote_procedure_mnist.cy.js @@ -34,7 +34,9 @@ describe("Loads remote_procedure_mnist", () => { it("Submitting with three corrected annotations", () => { cy.on("window:alert", (txt) => { - expect(txt).to.contain("The task has been submitted!"); + expect(txt).to.contain( + "Thank you for your submission.\nYou may close this message to view the next task." + ); }); cy.get('[data-cy="clear-button-0"]').as("clearButton0"); diff --git a/examples/simple_static_task/README.md b/examples/simple_static_task/README.md index ae12bea9e..bf1babc89 100644 --- a/examples/simple_static_task/README.md +++ b/examples/simple_static_task/README.md @@ -5,17 +5,19 @@ --> # Simple Static Task -This example script is to demonstrate how to launch a simple task using a html file. The "static" nature of this task means that all of the content required for a worker to complete the task must be set before the task is launched, and must be able to be sent to the app upon initialization. +This example script is to demonstrate how to launch a simple task using a html file. +The "static" nature of this task means that all of the content required for a worker to complete the task must be set before the task is launched, +and must be able to be sent to the app upon initialization. This specific example can be run with: ```console -python run_task.py +python run_task__local__inhouse.py ``` and can additionally be launched with an onboarding step by specifying an onboarding qualification: ```console -python run_task_with_onboarding.py +python run_task_with_onboarding__local__inhouse.py ``` ## Submit button customization @@ -40,7 +42,7 @@ You can get window properties as such: To run tests locally you should first launch the task as follows: ```bash -python run_task.py +python run_task__local__inhouse.py ``` This will run the task. diff --git a/examples/simple_static_task/hydra_configs/conf/example_ec2_mturk_sandbox.yaml b/examples/simple_static_task/hydra_configs/conf/example__ec2__mturk_sandbox.yaml similarity index 100% rename from examples/simple_static_task/hydra_configs/conf/example_ec2_mturk_sandbox.yaml rename to examples/simple_static_task/hydra_configs/conf/example__ec2__mturk_sandbox.yaml diff --git a/examples/simple_static_task/hydra_configs/conf/example_ec2_prolific.yaml b/examples/simple_static_task/hydra_configs/conf/example__ec2__prolific.yaml similarity index 100% rename from examples/simple_static_task/hydra_configs/conf/example_ec2_prolific.yaml rename to examples/simple_static_task/hydra_configs/conf/example__ec2__prolific.yaml diff --git a/examples/simple_static_task/hydra_configs/conf/example_local_mock.yaml b/examples/simple_static_task/hydra_configs/conf/example__local__inhouse.yaml similarity index 96% rename from examples/simple_static_task/hydra_configs/conf/example_local_mock.yaml rename to examples/simple_static_task/hydra_configs/conf/example__local__inhouse.yaml index 7ddf36cb0..e383830d8 100644 --- a/examples/simple_static_task/hydra_configs/conf/example_local_mock.yaml +++ b/examples/simple_static_task/hydra_configs/conf/example__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: data_csv: ${task_dir}/data/data.csv diff --git a/examples/simple_static_task/hydra_configs/conf/onboarding_example_local_mock.yaml b/examples/simple_static_task/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml similarity index 96% rename from examples/simple_static_task/hydra_configs/conf/onboarding_example_local_mock.yaml rename to examples/simple_static_task/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml index 8045dc1a0..4fceb061e 100644 --- a/examples/simple_static_task/hydra_configs/conf/onboarding_example_local_mock.yaml +++ b/examples/simple_static_task/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: data_csv: ${task_dir}/data/data.csv diff --git a/examples/simple_static_task/run_task_ec2_mturk_sandbox.py b/examples/simple_static_task/run_task__ec2__mturk_sandbox.py similarity index 94% rename from examples/simple_static_task/run_task_ec2_mturk_sandbox.py rename to examples/simple_static_task/run_task__ec2__mturk_sandbox.py index 8967c7acd..ae35d3201 100644 --- a/examples/simple_static_task/run_task_ec2_mturk_sandbox.py +++ b/examples/simple_static_task/run_task__ec2__mturk_sandbox.py @@ -14,7 +14,7 @@ from mephisto.utils.qualifications import make_qualification_dict -@task_script(default_config_file="example_ec2_mturk_sandbox") +@task_script(default_config_file="example__ec2__mturk_sandbox") def main(operator, cfg: DictConfig) -> None: shared_state = SharedStaticTaskState() diff --git a/examples/simple_static_task/run_task_ec2_prolific.py b/examples/simple_static_task/run_task__ec2__prolific.py similarity index 95% rename from examples/simple_static_task/run_task_ec2_prolific.py rename to examples/simple_static_task/run_task__ec2__prolific.py index 538d37484..b7e59b732 100644 --- a/examples/simple_static_task/run_task_ec2_prolific.py +++ b/examples/simple_static_task/run_task__ec2__prolific.py @@ -14,7 +14,7 @@ from mephisto.utils.qualifications import make_qualification_dict -@task_script(default_config_file="example_ec2_prolific") +@task_script(default_config_file="example__ec2__prolific") def main(operator, cfg: DictConfig) -> None: shared_state = SharedStaticTaskState() diff --git a/examples/simple_static_task/run_task.py b/examples/simple_static_task/run_task__local__inhouse.py similarity index 88% rename from examples/simple_static_task/run_task.py rename to examples/simple_static_task/run_task__local__inhouse.py index 0a0e37f2c..95c7d92fe 100644 --- a/examples/simple_static_task/run_task.py +++ b/examples/simple_static_task/run_task__local__inhouse.py @@ -8,7 +8,7 @@ from omegaconf import DictConfig -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator, cfg: DictConfig) -> None: operator.launch_task_run(cfg.mephisto) operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) diff --git a/examples/simple_static_task/run_task_with_onboarding.py b/examples/simple_static_task/run_task_with_onboarding__local__inhouse.py similarity index 92% rename from examples/simple_static_task/run_task_with_onboarding.py rename to examples/simple_static_task/run_task_with_onboarding__local__inhouse.py index e4c70d921..fc5519ed2 100644 --- a/examples/simple_static_task/run_task_with_onboarding.py +++ b/examples/simple_static_task/run_task_with_onboarding__local__inhouse.py @@ -18,7 +18,7 @@ @dataclass -class OnboardingConfig(build_default_task_config("onboarding_example_local_mock")): # type: ignore +class OnboardingConfig(build_default_task_config("example_with_onboarding__local__inhouse")): # type: ignore correct_answer: str = CORRECT_ANSWER diff --git a/examples/static_react_task/README.md b/examples/static_react_task/README.md index 930b9741b..adafd52e4 100644 --- a/examples/static_react_task/README.md +++ b/examples/static_react_task/README.md @@ -9,44 +9,59 @@ This example script is to demonstrate how to launch a simple task using a fronte This specific example can be run with: ```console -python run_task.py +python run_task__local__inhouse.py ``` and can additionally be launched with an onboarding step by specifying an onboarding qualification: ```console -python run_task.py mephisto.blueprint.onboarding_qualification=test-react-static-qualification +python run_task__local__inhouse.py mephisto.blueprint.onboarding_qualification=test-react-static-qualification ``` or by specifying the config file that already has this set: ```console -python run_task.py conf=onboarding_example_local_mock +python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse ``` ## Implementation + ### Configuration + This task is configured using [Hydra](https://hydra.cc/) - details about using hydra to configure tasks can be read here and in other examples. For more about being able to customize the configuration files, please refer to the [Hydra documentation](https://hydra.cc/docs/intro). Under our current setup, using Hydra means that you must be in the same directory as the python script for hydra to correctly pick up the config files. + #### Setting reasonable defaults In this script, we set certain configuration variables by default in every run: ```yaml defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse ``` -These defaults are handed to Hydra in order to ensure that by default, we run a task locally with a mock provider (so we can demo). We also set `conf` to `example`, which means this script by default will also load in all of the configuration variables set in `conf/example.yaml`. +These defaults are handed to Hydra in order to ensure that by default, we run a task locally with a mock provider (so we can demo). +We also set `conf` to `example`, which means this script by default will also load in all of the configuration variables set in `conf/example__local__inhouse.yaml`. + +If your task has other variables that you think will almost always be set to particular values (say you always expect `mephisto.blueprint.units_per_assignment` to be `1`) +that differ from Mephisto's default for those values (if such a default exists), +you can include them by default by adding a config file with your defaults to `conf` and adding the string path to it here in the list, like `conf/base`. +See the ParlAI Chat demo for an example of this. -If your task has other variables that you think will almost always be set to particular values (say you always expect `mephisto.blueprint.units_per_assignment` to be `1`) that differ from Mephisto's default for those values (if such a default exists), you can include them by default by adding a config file with your defaults to `conf` and adding the string path to it here in the list, like `conf/base`. See the ParlAI Chat demo for an example of this. #### Creating and using override files -You can create override configuration files, such as `example_local_mock.yaml` vs `onboarding_example_local_mock.yaml` in the `conf` directory. Having these files makes it really easy to set multiple values at once. You can only select one such configuration file per run, using the `conf=example` argument. +You can create override configuration files, such as `example__local__inhouse.yaml` vs `example_with_onboarding__local__inhouse.yaml` in the `conf` directory. +Having these files makes it really easy to set multiple values at once. You can only select one such configuration file per run, using the `conf=example` argument. + #### Overriding on the command line You can also override configuration variables on the command line. Say you want to launch your server on port 4000 rather than 3000, you can use: + ```console -python run_task.py mephisto.architect.port=4000 +python run_task__local__inhouse.py mephisto.architect.port=4000 ``` + To be able to launch with a `HerokuArchitect` rather than the default `LocalArchitect`, you can use: + ```console -python run_task.py mephisto/architect=heroku +python run_task__local__inhouse.py mephisto/architect=heroku ``` + ### Providing task data -In this case, the provided app demonstrates being able to send task data forward to the frontend. See the `run_task.py` script. Here we have: +In this case, the provided app demonstrates being able to send task data forward to the frontend. See the `run_task__local__inhouse.py` script. Here we have: + ```python shared_state = SharedStaticTaskState( static_task_data=[ @@ -56,9 +71,11 @@ shared_state = SharedStaticTaskState( validate_onboarding=onboarding_always_valid, ) ``` -This block of code is preparing two tasks, each with a `"text"` field set. When the task run is launched with `operator.launch_task_run(cfg.mephisto, shared_state)`, this creates two tasks for workers to work on, one for each entry in the `static_task_data` array. +This block of code is preparing two tasks, each with a `"text"` field set. When the task run is launched with `operator.launch_task_run(cfg.mephisto, shared_state)`, +this creates two tasks for workers to work on, one for each entry in the `static_task_data` array. -This data is later pulled via the `useMephistoTask` hook, and when a worker accepts a task, they will be given the contents of one of the entries as their `initialTaskData`. See the `webapp/src/app.jsx` file. We render +This data is later pulled via the `useMephistoTask` hook, and when a worker accepts a task, they will be given the contents of one of the entries as their `initialTaskData`. +See the `webapp/src/app.jsx` file. We render ```js bool: return False -@task_script(default_config_file="example_local_mock.yaml") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_static_react_task( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/static_react_task/webapp/cypress.config.js b/examples/static_react_task/webapp/cypress.config.js index a8f21d574..5ef34a64e 100644 --- a/examples/static_react_task/webapp/cypress.config.js +++ b/examples/static_react_task/webapp/cypress.config.js @@ -8,7 +8,8 @@ module.exports = { video: false, e2e: { - baseUrl: "http://localhost:3000/?worker_id=x&assignment_id=1", + baseUrl: "http://localhost:3000/?worker_id=x&id=1", + includeShadowDom: true, supportFile: false, }, }; diff --git a/examples/static_react_task/webapp/cypress/e2e/static_react_task.cy.js b/examples/static_react_task/webapp/cypress/e2e/static_react_task.cy.js index beda3c177..9e03c732a 100644 --- a/examples/static_react_task/webapp/cypress/e2e/static_react_task.cy.js +++ b/examples/static_react_task/webapp/cypress/e2e/static_react_task.cy.js @@ -14,6 +14,7 @@ describe("Loads static_react_task", () => { }); it("Loads correct react elements", () => { + cy.visit("/"); cy.get('[data-cy="directions-container"]'); cy.get('[data-cy="task-data-text"]'); cy.get('[data-cy="good-button"]'); @@ -23,7 +24,7 @@ describe("Loads static_react_task", () => { describe("Submits static_react_task", () => { it("Gets request from pressing good button", () => { - cy.reload(); + cy.visit("/"); cy.intercept({ pathname: "/submit_task" }).as("goodTaskSubmit"); cy.get('[data-cy="good-button"]').click(); cy.wait("@goodTaskSubmit").then((interception) => { @@ -32,17 +33,17 @@ describe("Submits static_react_task", () => { }); it("Shows alert from pressing good button", () => { - cy.reload(); + cy.visit("/"); cy.on("window:alert", (txt) => { expect(txt).to.contains( - 'The task has been submitted! Data: {"rating":"good"}' + "Thank you for your submission.\nYou may close this message to view the next task." ); }); cy.get('[data-cy="good-button"]').click(); }); it("Gets request from pressing bad button", () => { - cy.reload(); + cy.visit("/"); cy.intercept({ pathname: "/submit_task" }).as("badTaskSubmit"); cy.get('[data-cy="bad-button"]').click(); cy.wait("@badTaskSubmit").then((interception) => { @@ -51,10 +52,10 @@ describe("Submits static_react_task", () => { }); it("Shows alert from pressing bad button", () => { - cy.reload(); + cy.visit("/"); cy.on("window:alert", (txt) => { expect(txt).to.contains( - 'The task has been submitted! Data: {"rating":"bad"}' + "Thank you for your submission.\nYou may close this message to view the next task." ); }); cy.get('[data-cy="bad-button"]').click(); diff --git a/examples/static_react_task_with_worker_opinion/README.md b/examples/static_react_task_with_worker_opinion/README.md index 5b91b052a..5f59489f3 100644 --- a/examples/static_react_task_with_worker_opinion/README.md +++ b/examples/static_react_task_with_worker_opinion/README.md @@ -6,7 +6,7 @@ # Simple Static React Task with Worker Opinion -This task is essentially the same as the `static-react-task` task except that there is a worker opinon form after finishing a Task. +This task is essentially the same as the `static_react_task` task except that there is a worker opinon form after finishing a Task. The [static-react-task README](../static_react_task/README.md) should be read before running this task. diff --git a/examples/static_react_task_with_worker_opinion/hydra_configs/conf/example_local_mock.yaml b/examples/static_react_task_with_worker_opinion/hydra_configs/conf/example__local__inhouse.yaml similarity index 96% rename from examples/static_react_task_with_worker_opinion/hydra_configs/conf/example_local_mock.yaml rename to examples/static_react_task_with_worker_opinion/hydra_configs/conf/example__local__inhouse.yaml index 17a6906c0..0561563fe 100644 --- a/examples/static_react_task_with_worker_opinion/hydra_configs/conf/example_local_mock.yaml +++ b/examples/static_react_task_with_worker_opinion/hydra_configs/conf/example__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: task_source: ${task_dir}/webapp/build/bundle.js diff --git a/examples/static_react_task_with_worker_opinion/hydra_configs/conf/onboarding_example_local_mock.yaml b/examples/static_react_task_with_worker_opinion/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml similarity index 96% rename from examples/static_react_task_with_worker_opinion/hydra_configs/conf/onboarding_example_local_mock.yaml rename to examples/static_react_task_with_worker_opinion/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml index a2c0a717f..8ce5caa42 100644 --- a/examples/static_react_task_with_worker_opinion/hydra_configs/conf/onboarding_example_local_mock.yaml +++ b/examples/static_react_task_with_worker_opinion/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: task_source: ${task_dir}/webapp/build/bundle.js diff --git a/examples/static_react_task_with_worker_opinion/run_task.py b/examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py similarity index 95% rename from examples/static_react_task_with_worker_opinion/run_task.py rename to examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py index 9de968c73..dd9201473 100644 --- a/examples/static_react_task_with_worker_opinion/run_task.py +++ b/examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py @@ -18,7 +18,7 @@ def _onboarding_always_valid(onboarding_data: dict) -> bool: return True -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_static_react_task_with_worker_opinion( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/static_react_task_with_worker_opinion/webapp/cypress.config.js b/examples/static_react_task_with_worker_opinion/webapp/cypress.config.js index 563d67b50..4d122559f 100644 --- a/examples/static_react_task_with_worker_opinion/webapp/cypress.config.js +++ b/examples/static_react_task_with_worker_opinion/webapp/cypress.config.js @@ -8,7 +8,7 @@ module.exports = { video: false, e2e: { - baseUrl: "http://localhost:3000/?worker_id=x&assignment_id=1", + baseUrl: "http://localhost:3000/?worker_id=x&id=1", includeShadowDom: true, }, }; diff --git a/examples/video_annotator_demo/README.md b/examples/video_annotator_demo/README.md index d4157ff2c..8ace9285d 100644 --- a/examples/video_annotator_demo/README.md +++ b/examples/video_annotator_demo/README.md @@ -8,7 +8,7 @@ These video annotators are example of VideoAnnotator task generator. 2. SSH into running container to run server: `docker exec -it mephisto_dc bash` 3. Inside the container, go to VideoAnnotator examples directory: `cd /mephisto/examples/video_annotator_demo` 4. Inside the examples directory, run a desired example with one of these commands: - - Simple form: `python ./run_task.py` + - Simple form: `python ./run_task__local__inhouse.py` --- diff --git a/examples/video_annotator_demo/hydra_configs/conf/dynamic_example_local_mock.yaml b/examples/video_annotator_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml similarity index 94% rename from examples/video_annotator_demo/hydra_configs/conf/dynamic_example_local_mock.yaml rename to examples/video_annotator_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml index 3c0da6dd8..f400cb038 100644 --- a/examples/video_annotator_demo/hydra_configs/conf/dynamic_example_local_mock.yaml +++ b/examples/video_annotator_demo/hydra_configs/conf/dynamic_example__local__inhouse.yaml @@ -7,7 +7,7 @@ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: @@ -22,5 +22,5 @@ mephisto: task_title: "Example how to easily create dynamic Tasks to annotate video" task_description: "In this Task, we use VideoAnnotator feature." task_reward: 0 - task_tags: "test,dynamic,generated,video,annotation,video-annotator,mock" + task_tags: "test,dynamic,generated,video,annotation,video-annotator,inhouse" force_rebuild: true diff --git a/examples/video_annotator_demo/hydra_configs/conf/example_local_inhouse.yaml b/examples/video_annotator_demo/hydra_configs/conf/example__local__inhouse.yaml similarity index 100% rename from examples/video_annotator_demo/hydra_configs/conf/example_local_inhouse.yaml rename to examples/video_annotator_demo/hydra_configs/conf/example__local__inhouse.yaml diff --git a/examples/video_annotator_demo/hydra_configs/conf/example_local_mock.yaml b/examples/video_annotator_demo/hydra_configs/conf/example_local_mock.yaml deleted file mode 100644 index 394746594..000000000 --- a/examples/video_annotator_demo/hydra_configs/conf/example_local_mock.yaml +++ /dev/null @@ -1,26 +0,0 @@ -#@package _global_ - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -defaults: - - /mephisto/blueprint: static_react_task - - /mephisto/architect: local - - /mephisto/provider: mock - -mephisto: - blueprint: - data_json: ${task_dir}/data/simple/task_data.json - task_source: ${task_dir}/webapp/build/bundle.js - task_source_review: ${task_dir}/webapp/build/bundle.review.js - link_task_source: false - extra_source_dir: ${task_dir}/webapp/src/static - units_per_assignment: 2 - task: - task_name: "Video annotator" - task_title: "Example how to easily create simple Tasks to annotate video" - task_description: "In this Task, we use VideoAnnotator feature." - task_reward: 0 - task_tags: "test,simple,generated,video,annotation,video-annotator,mock" - force_rebuild: true diff --git a/examples/video_annotator_demo/run_task.py b/examples/video_annotator_demo/run_task.py deleted file mode 100644 index 7a888d3af..000000000 --- a/examples/video_annotator_demo/run_task.py +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright (c) Meta Platforms and its affiliates. -# This source code is licensed under the MIT license found in the -# LICENSE file in the root directory of this source tree. - -from omegaconf import DictConfig - -from mephisto.operations.operator import Operator -from mephisto.tools.building_react_apps import examples -from mephisto.tools.scripts import task_script - - -@task_script(default_config_file="example_local_mock") -def main(operator: Operator, cfg: DictConfig) -> None: - examples.build_video_annotator_simple( - force_rebuild=cfg.mephisto.task.force_rebuild, - post_install_script=cfg.mephisto.task.post_install_script, - ) - operator.launch_task_run(cfg.mephisto) - operator.wait_for_runs_then_shutdown(skip_input=True, log_rate=30) - - -if __name__ == "__main__": - main() diff --git a/examples/video_annotator_demo/run_task__local__inhouse.py b/examples/video_annotator_demo/run_task__local__inhouse.py index 8c6d42199..c09a3db5d 100644 --- a/examples/video_annotator_demo/run_task__local__inhouse.py +++ b/examples/video_annotator_demo/run_task__local__inhouse.py @@ -11,7 +11,7 @@ from mephisto.tools.scripts import task_script -@task_script(default_config_file="example_local_inhouse") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_video_annotator_simple( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/examples/video_annotator_demo/run_task_dynamic.py b/examples/video_annotator_demo/run_task_dynamic__local__inhouse.py similarity index 97% rename from examples/video_annotator_demo/run_task_dynamic.py rename to examples/video_annotator_demo/run_task_dynamic__local__inhouse.py index 2914293cb..8bec67d48 100644 --- a/examples/video_annotator_demo/run_task_dynamic.py +++ b/examples/video_annotator_demo/run_task_dynamic__local__inhouse.py @@ -48,7 +48,7 @@ def _generate_task_data_json_config(): ) -@task_script(default_config_file="dynamic_example_local_mock") +@task_script(default_config_file="dynamic_example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: examples.build_video_annotator_dynamic( force_rebuild=cfg.mephisto.task.force_rebuild, diff --git a/mephisto/README.md b/mephisto/README.md index 50f176013..b996dddcb 100644 --- a/mephisto/README.md +++ b/mephisto/README.md @@ -114,7 +114,7 @@ Here's a list of steps on how to build and run your own custom data collection T In order to launch your own customized project, you will need to write a React app that will display instructions/inputs to workers. You can start by duplicating an existing Task App code (e.g. `examples/static_react_task` directory) and customizing it to your needs. The process goes like this: 1. Copy `static_react_task` directory to your project directory within Mephisto repo -2. Customize task's back-end code in `run_task.py` script to pass relevant data to `SharedStaticTaskState`, set `shared_state.prolific_specific_qualifications`, `shared_state.qualifications` (for custom qualifications), etc +2. Customize task's back-end code in `run_task__local__inhouse.py` script to pass relevant data to `SharedStaticTaskState`, set `shared_state.prolific_specific_qualifications`, `shared_state.qualifications` (for custom qualifications), etc 3. Customize task-related parameters variables in your `conf/.yaml` file as needed. - Some examples of variables from `blueprint` category are: - `extra_source_dir`: optional path to sources that Task App may refer to (images, video, css, scripts, etc) @@ -122,7 +122,7 @@ In order to launch your own customized project, you will need to write a React a - To see other configurable blueprint variables, type `mephisto wut blueprint=static_task` 4. Customize task's front-end code, with starting point being `//webapp/src/components/core_components.jsx` (you caninclude an onboarding step if you like). 5. Add the ability to review results of your task app. In short, you need to implement additional component or logic to render json data that TaskReview app will provide. For more details, read this [doc](/mephisto/review_app/README.md). -6. Run `run_task.py` to dry-run your task on localhost. +6. Run `run_task__local__inhouse.py` to dry-run your task on localhost. 7. Repeat 5 & 6 until you're happy with your task. 8. Launch a small batch with a chosen crowd provider to see how real workers handle your task. 9. Iterate more. @@ -178,7 +178,8 @@ This is a sample YAML configuration to run your Task on **AWS EC2** architect wi For all available Prolific-specific parameters see `mephisto.abstractions.providers.prolific.prolific_provider.ProlificProviderArgs` class and [Prolific API Docs](https://docs.prolific.com/docs/api-docs/public/#tag/Studies). - Note that `prolific_eligibility_requirements` does not include custom worker qualifications, these are maintained in your local Mephisto database. These can be specified in a Task launching script (usually called `run_task.py`, for example, `examples/simple_static_task/run_task.py`) + Note that `prolific_eligibility_requirements` does not include custom worker qualifications, these are maintained in your local Mephisto database. + These can be specified in a Task launching script (usually called `run_task__local__inhouse.py`, for example, `examples/simple_static_task/run_task__local__inhouse.py`) --- @@ -197,7 +198,7 @@ or simply embed that command into your docker-compose entrypoint script. --build \ --rm mephisto_dc \ rm -rf /mephisto/tmp && \ - HYDRA_FULL_ERROR=1 python /mephisto/examples/simple_static_task/run_task.py + HYDRA_FULL_ERROR=1 python /mephisto/examples/simple_static_task/run_task__local__inhouse.py ``` This TaskRun script will spin up an EC2 server, upload your React Task App to it, and create a Study on Prolific. Now all eligible workers will see your Task Units (with links poiting to EC2 server) on Prolific, and can complete it. diff --git a/mephisto/abstractions/blueprints/README.md b/mephisto/abstractions/blueprints/README.md index 9a5dc1bd7..f711a4269 100644 --- a/mephisto/abstractions/blueprints/README.md +++ b/mephisto/abstractions/blueprints/README.md @@ -5,49 +5,89 @@ --> # Blueprints + ## Overview -Blueprints serve to package tasks (and groups of similar tasks) into a reusable format. They can be used to work through piloting tasks, collecting data, testing different formats, etc. They're also used by the architecture to simplify the data accumulation and review processes. The `StaticBlueprint` is a good starting example of how to implement a blueprint. +Blueprints serve to package tasks (and groups of similar tasks) into a reusable format. +They can be used to work through piloting tasks, collecting data, testing different formats, etc. +They're also used by the architecture to simplify the data accumulation and review processes. +The `StaticBlueprint` is a good starting example of how to implement a blueprint. ## Implementation Details + ### `AgentState` -The agent state is responsible for defining the data that is important to store for a specific `Unit`, as well as methods for writing that locally to disk. To abstract this, it must implement the following methods: +The agent state is responsible for defining the data that is important to store for a specific `Unit`, as well as methods for writing that locally to disk. +To abstract this, it must implement the following methods: + - `set_init_state(data)`: given data provided by the `get_init_data_for_agent` method, initialize this agent state to whatever starting state is relevant for this `Unit`. - `get_init_state()`: Return the initial state to be sent to the agent for use in the frontend. -- `load_data()`: Load data that is saved to file to re-initialize the state for this `AgentState`. Generally data should be stored in `self.agent.get_data_dir()`, however any storage solution will work as long as it remains consistent. +- `load_data()`: Load data that is saved to file to re-initialize the state for this `AgentState`. +Generally data should be stored in `self.agent.get_data_dir()`, however any storage solution will work as long as it remains consistent. - `get_data()`: Return the stored data for this task in the format containing everything the frontend needs to render and run the task. - `get_parsed_data()`: Return the stored data for this task in the format that is relevant for review or packaging the data. -- `save_data()`: Save data to a file such that it can be re-initialized later. Generally data should be stored in `self.agent.get_data_dir()`, however any storage solution will work as long as it remains consistent, and `load_data()` will be able to find it. -- `update_data()`: Update the local state stored in this `AgentState` given the data sent from the frontend. Given your frontend is what packages data to send, this is entirely customizable by the task creator. +- `save_data()`: Save data to a file such that it can be re-initialized later. +Generally data should be stored in `self.agent.get_data_dir()`, however any storage solution will work as long as it remains consistent, and `load_data()` will be able to find it. +- `update_data()`: Update the local state stored in this `AgentState` given the data sent from the frontend. +Given your frontend is what packages data to send, this is entirely customizable by the task creator. ### `TaskBuilder` -`TaskBuilder`s exist to abstract away the portion of building a frontend to however one would want to, allowing Mephisto users to design tasks however they'd like. They also can take build options to customize what ends up built. They must implement the following: -- `build_in_dir(build_dir)`: Take any important source files and put them into the given build dir. This directory will be deployed to the frontend and will become the static target for completing the task. +`TaskBuilder`s exist to abstract away the portion of building a frontend to however one would want to, allowing Mephisto users to design tasks however they'd like. +They also can take build options to customize what ends up built. They must implement the following: + +- `build_in_dir(build_dir)`: Take any important source files and put them into the given build dir. +This directory will be deployed to the frontend and will become the static target for completing the task. - `get_extra_options()`: Return the specific task options that are relevant to customize the frontend when `build_in_dir` is called. ### `TaskRunner` -The `TaskRunner` component of a blueprint is responsible for actually stepping `Agent`s through the task when it is live. It is, in short, able to set up task control. A `TaskRunner` needs to implement the following: -- `get_init_data_for_agent`: Provide initial data for an assignment. If this agent is reconnecting (and as such attached to an existing task), update that task to point to the new agent (as the old agent object will no longer receive data from the frontend). -- `run_assignment`: Handle setup for any resources required to get this assignment running. It will be launched in a background thread, and should be tolerant to being interrupted by cleanup_assignment. -- `cleanup_assignment`: Send any signals to the required thread for the given assignment to tell it to terminate, then clean up any resources that were set within it. -- `get_data_for_assignment` (optional): Get the data that an assignment is going to use when run. By default, this pulls from `assignment.get_assignment_data()` however if a task has a special storage mechanism or data type, the assignment data can be fetched here. +The `TaskRunner` component of a blueprint is responsible for actually stepping `Agent`s through the task when it is live. +It is, in short, able to set up task control. A `TaskRunner` needs to implement the following: + +- `get_init_data_for_agent`: Provide initial data for an assignment. +If this agent is reconnecting (and as such attached to an existing task), update that task to point to the new agent +(as the old agent object will no longer receive data from the frontend). +- `run_assignment`: Handle setup for any resources required to get this assignment running. +It will be launched in a background thread, and should be tolerant to being interrupted by cleanup_assignment. +- `cleanup_assignment`: Send any signals to the required thread for the given assignment to tell it to terminate, +then clean up any resources that were set within it. +- `get_data_for_assignment` (optional): Get the data that an assignment is going to use when run. +By default, this pulls from `assignment.get_assignment_data()` however if a task has a special storage mechanism or data type, +the assignment data can be fetched here. ### `SharedTaskState` -A blueprint is able to create a container that handles any shared data that is initialized during a task or modified between tasks, or for function hooks that are used across a run. The following hooks are already provided in the base: -- `validate_onboarding`: A function that takes in an onboarding agent's `AgentState.get_data()` call, and should always return a boolean of if that state represents a successful onboarding completion. -- `worker_can_do_unit`: A function that takes in a `Worker` and a `Unit`, and should return a boolean representing if the worker is eligible to work on that particular unit. -- `on_unit_submitted`: A function that takes in a `Unit` after a `TaskRunner` ends, and is able to do any automatic post-processing operations on that unit that a Mephisto user may want. +A blueprint is able to create a container that handles any shared data that is initialized during a task or modified between tasks, +or for function hooks that are used across a run. +The following hooks are already provided in the base: + +- `validate_onboarding`: A function that takes in an onboarding agent's `AgentState.get_data()` call, +and should always return a boolean of if that state represents a successful onboarding completion. +- `worker_can_do_unit`: A function that takes in a `Worker` and a `Unit`, +and should return a boolean representing if the worker is eligible to work on that particular unit. +- `on_unit_submitted`: A function that takes in a `Unit` after a `TaskRunner` ends, +and is able to do any automatic post-processing operations on that unit that a Mephisto user may want. ## `Blueprint` Mixins -Blueprints sometimes share some component functionality that may be useful across a multitude of tasks. We capture these in mixins. Mephisto is able to recognize certain mixins in order to complete additional operations, however custom mixins may help cut down on boiler plate in common `run_task.py` scripts. As your tasks mature, we suggest utilizing blueprint mixins to share common workflows and design patterns you observe. +Blueprints sometimes share some component functionality that may be useful across a multitude of tasks. +We capture these in mixins. Mephisto is able to recognize certain mixins in order to complete additional operations, +however custom mixins may help cut down on boiler plate in common `run_task__local__inhouse.py` scripts. +As your tasks mature, we suggest utilizing blueprint mixins to share common workflows and design patterns you observe. + ### `OnboardingRequired` -This mixin allows for blueprints that require people to complete an onboarding task _before_ they're even able to start on their first task. Usually this is useful for providing task context, and then quizzing workers to see if they understand what's provided. Tasks using this mixin will activate onboarding mode for new `Worker`s whenever the `mephisto.blueprint.onboarding_qualification` hydra argument is provided. +This mixin allows for blueprints that require people to complete an onboarding task _before_ they're even able to start on their first task. +Usually this is useful for providing task context, and then quizzing workers to see if they understand what's provided. +Tasks using this mixin will activate onboarding mode for new `Worker`s whenever the `mephisto.blueprint.onboarding_qualification` hydra argument is provided. + ### `ScreenTaskRequired` -This mixin allows for blueprints that require people to complete a _test_ version of the real task the first time a worker does the task. This allows you to validate workers on a run of the real task, either on your actual data (when providing `SharedTaskState.screening_data_factory=False`) or on test data that you may more easily validate using (when providing a generator to `SharedTaskState.screening_data_factory`). The tasks should be the same as your standard task, just able to be easily validated. You **do pay** for screening tasks, and as such we ask you set `mephisto.blueprint.max_screening_units` to put a cap on how many screening units you want to launch. +This mixin allows for blueprints that require people to complete a _test_ version of the real task the first time a worker does the task. +This allows you to validate workers on a run of the real task, either on your actual data (when providing `SharedTaskState.screening_data_factory=False`) +or on test data that you may more easily validate using (when providing a generator to `SharedTaskState.screening_data_factory`). +The tasks should be the same as your standard task, just able to be easily validated. +You **do pay** for screening tasks, and as such we ask you set `mephisto.blueprint.max_screening_units` to put a cap on how many screening units you want to launch. ## Implementations + ### `StaticBlueprint` -The `StaticBlueprint` class allows a replication of the interface that MTurk provides, being able to take a snippet of `HTML` and a `.csv` file and deploy tasks that fill templates of the `HTML` with values from the `.csv`. +The `StaticBlueprint` class allows a replication of the interface that MTurk provides, +being able to take a snippet of `HTML` and a `.csv` file and deploy tasks that fill templates of the `HTML` with values from the `.csv`. You can also specify the task data in a `.json` file, or by passing the data array or a generator to `SharedStaticTaskState.static_task_data`. diff --git a/mephisto/abstractions/blueprints/static_html_task/README.md b/mephisto/abstractions/blueprints/static_html_task/README.md index c65373132..c1884889a 100644 --- a/mephisto/abstractions/blueprints/static_html_task/README.md +++ b/mephisto/abstractions/blueprints/static_html_task/README.md @@ -7,27 +7,45 @@ # `StaticHTMLBlueprint` ## Overview -The `StaticHTMLBlueprint` exists to create a simple transition stand-in to allow users of other platforms (which generally push for customization with HTML) to begin using Mephisto with minimal changes. This generally exists in the form of specifying a templated HTML file and providing assignment data that fills the templates. +The `StaticHTMLBlueprint` exists to create a simple transition stand-in to allow users of other platforms +(which generally push for customization with HTML) to begin using Mephisto with minimal changes. +This generally exists in the form of specifying a templated HTML file and providing assignment data that fills the templates. -There are no plans to extend the `StaticHTMLBlueprint`, or provide other HTML support, as we believe this to be more an onboarding ramp rather than a fully fleshed out feature. The `React`-based codebase contains better features and cleaner, reusable modules, and as such our focus is on tasks in that area. +There are no plans to extend the `StaticHTMLBlueprint`, or provide other HTML support, +as we believe this to be more an onboarding ramp rather than a fully fleshed out feature. +The `React`-based codebase contains better features and cleaner, reusable modules, and as such our focus is on tasks in that area. ## Usage -An example usage case is available [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/). General usage for this `Blueprint` type can be summed up as copying that directory structure, providing your own templated HTML in the `server_files`, and then providng the `data.csv` or other data to post those templates to workers. More in-depth descriptions can be seen there. +An example usage case is available [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/). +General usage for this `Blueprint` type can be summed up as copying that directory structure, providing your own templated HTML in the `server_files`, +and then providng the `data.csv` or other data to post those templates to workers. More in-depth descriptions can be seen there. ## Implementation Details -At a high level, this is a deeper implementation of the abstract `StaticArchitect`, which contains all of the logic for deploying arbitrary tasks where the worker is given some content, and we ask for one response, as a complete `Unit`. This module adds the logic to ensure that the frontend where the worker is given content is able to render arbitrary HTML. +At a high level, this is a deeper implementation of the abstract `StaticArchitect`, +which contains all of the logic for deploying arbitrary tasks where the worker is given some content, and we ask for one response, as a complete `Unit`. +This module adds the logic to ensure that the frontend where the worker is given content is able to render arbitrary HTML. ### `app.jsx` -The `app.jsx` file contains most of the logic to ensure we can render HTML, including being able to attach and execute the arbitrary scripts that are included or linked in the given HTML file. It creates a react application using the `mephisto-core` package, and creates an empty form with a submit button. +The `app.jsx` file contains most of the logic to ensure we can render HTML, including being able to attach and execute the arbitrary scripts that are included or linked in the given HTML file. +It creates a react application using the `mephisto-core` package, and creates an empty form with a submit button. It then populates the inner form using two primary methods: -- `handleUpdatingRemainingScripts`: This method walks through the list of scripts that are given in the attached HTML, either loading the external script and putting it into the head, or directly evaluating the content of local scripts. As the page has already loaded by the time we are loading in the remaining scripts, this all must be injected in an asynchronous manner. _Ultimately_ this isn't incredibly safe if you don't trust the attached scripts. -- `interpolateHtml`: This method injects the values for the specific task into their template variable slots. These template variables are specified in the `InitializationData` for an assignment, and populate fields as noted in the **Template Variables** section below. + +- `handleUpdatingRemainingScripts`: This method walks through the list of scripts that are given in the attached HTML, +either loading the external script and putting it into the head, or directly evaluating the content of local scripts. +As the page has already loaded by the time we are loading in the remaining scripts, this all must be injected in an asynchronous manner. +_Ultimately_ this isn't incredibly safe if you don't trust the attached scripts. + +- `interpolateHtml`: This method injects the values for the specific task into their template variable slots. +These template variables are specified in the `InitializationData` for an assignment, +and populate fields as noted in the **Template Variables** section below. Upon submit, the data in the form (as marked by the `name` fields of any inputs) will be sent to the backend and stored in the `agent_data.json` file. #### Template Variables -You can provide template variables when running your task, generally in the form of template variables that are given names. When you specify these (either via `.csv` or directly providing a parsed array of dicts for this data), any named variable `my_special_variable` will be filled into the frontend in all locations containing `${my_special_variable}`. +You can provide template variables when running your task, generally in the form of template variables that are given names. +When you specify these (either via `.csv` or directly providing a parsed array of dicts for this data), +any named variable `my_special_variable` will be filled into the frontend in all locations containing `${my_special_variable}`. #### Mephisto-specific Template Variables As of now, we also make available the following variables to the HTML via templating: @@ -35,7 +53,8 @@ As of now, we also make available the following variables to the HTML via templa - `${provider_worker_id}`: The worker id that the provider uses locally to identify the worker. #### `useMephistoGlobalConfig` -This hook allows for a state to be updated when an event is consumed. An event is emitted for consumption whenever a window variable is set in `demo_task.html` using: +This hook allows for a state to be updated when an event is consumed. +An event is emitted for consumption whenever a window variable is set in `demo_task.html` using: ```html