diff --git a/.github/workflows/cypress-end-to-end-tests.yml b/.github/workflows/cypress-end-to-end-tests.yml index 56c04ec8c..0a162d963 100644 --- a/.github/workflows/cypress-end-to-end-tests.yml +++ b/.github/workflows/cypress-end-to-end-tests.yml @@ -124,8 +124,8 @@ jobs: # browser: chrome # project: ./examples/form_composer_demo/webapp # config-file: ./cypress.config.js - # start: python examples/form_composer_demo/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - # wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + # start: python examples/form_composer_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + # wait-on: "http://localhost:3000/?worker_id=x&=1" # headless: true # # video_annotator_demo: @@ -177,8 +177,8 @@ jobs: # browser: chrome # project: ./examples/video_annotator_demo/webapp # config-file: ./cypress.config.js - # start: python examples/video_annotator_demo/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - # wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + # start: python examples/video_annotator_demo/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + # wait-on: "http://localhost:3000/?worker_id=x&id=1" # headless: true # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/881 @@ -209,6 +209,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -224,12 +227,12 @@ jobs: browser: chrome project: ./mephisto/abstractions/blueprints/static_html_task/source config-file: ./cypress.config.js - start: python ./examples/simple_static_task/run_task.py - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python ./examples/simple_static_task/run_task__local__inhouse.py + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/795 - static-react-task: + static_react_task: needs: changes if: ${{ (needs.changes.outputs.static_react_task == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} runs-on: ubuntu-latest @@ -256,6 +259,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -271,8 +277,8 @@ jobs: browser: chrome project: ./examples/static_react_task/webapp config-file: ./cypress.config.js - start: python examples/static_react_task/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/static_react_task/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # Learn more about the remote_procedure_tests here: https://github.com/facebookresearch/Mephisto/pull/800 @@ -303,7 +309,7 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data - - name: 🖋 Create Inhouse provider + - name: 🚚 Create Inhouse provider run: mephisto register inhouse - name: 📦 Setting up mephisto-core package @@ -322,7 +328,7 @@ jobs: project: ./examples/remote_procedure/elementary_remote_procedure/webapp config-file: ./cypress.config.js start: python examples/remote_procedure/elementary_remote_procedure/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # TODO: Add tests and enable @@ -375,13 +381,13 @@ jobs: # browser: chrome # project: ./examples/remote_procedure/interactive_image_generation/webapp # config-file: ./cypress.config.js - # start: python examples/remote_procedure/interactive_image_generation/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - # wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + # start: python examples/remote_procedure/interactive_image_generation/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + # wait-on: "http://localhost:3000/?worker_id=x&id=1" # headless: true remote_procedure_mnist: needs: changes - if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-core == 'true') }} + if: ${{ (needs.changes.outputs.mnist == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} runs-on: ubuntu-latest steps: - name: 🔀 Checking out repo @@ -408,6 +414,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -423,13 +432,13 @@ jobs: browser: chrome project: ./examples/remote_procedure/mnist/webapp config-file: ./cypress.config.js - start: python examples/remote_procedure/mnist/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/remote_procedure/mnist/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true remote_procedure_toxicity_detection: needs: changes - if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-core == 'true') }} + if: ${{ (needs.changes.outputs.toxicity_detection == 'true') || (needs.changes.outputs.mephisto-core == 'true') || (needs.changes.outputs.abstractions == 'true') || (needs.changes.outputs.data_model == 'true') || (needs.changes.outputs.operations == 'true') || (needs.changes.outputs.tools == 'true')}} runs-on: ubuntu-latest steps: - name: 🔀 Checking out repo @@ -456,7 +465,7 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data - - name: 🖋 Create Inhouse provider + - name: 🚚 Create Inhouse provider run: mephisto register inhouse - name: 📦 Setting up mephisto-core package @@ -475,7 +484,7 @@ jobs: project: ./examples/remote_procedure/toxicity_detection/webapp config-file: ./cypress.config.js start: python examples/remote_procedure/toxicity_detection/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true # Learn more about this test here: https://github.com/facebookresearch/Mephisto/pull/833 @@ -507,6 +516,9 @@ jobs: - name: 📂 Set the data directory run: mephisto config core.main_data_directory ~/mephisto/data + - name: 🚚 Create Inhouse provider + run: mephisto register inhouse + - name: 📦 Setting up mephisto-core package run: | cd packages/mephisto-core @@ -530,8 +542,8 @@ jobs: project: ./examples/static_react_task_with_worker_opinion/webapp config-file: ./cypress.config.js spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/pre_submission_tests/* - start: python examples/static_react_task_with_worker_opinion/run_task.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_mephisto_task.sh - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.force_rebuild=true mephisto.task.post_install_script=link_mephisto_task.sh + wait-on: "http://localhost:3000/?worker_id=x&id=1" headless: true - name: 🔪 Killing the web server @@ -604,7 +616,7 @@ jobs: project: ./examples/static_react_task_with_worker_opinion/webapp config-file: cypress.config.js spec: ./examples/static_react_task_with_worker_opinion/webapp/cypress/e2e/post_submission_tests/* - start: python examples/static_react_task_with_worker_opinion/run_task.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true - wait-on: "http://localhost:3000/?worker_id=x&assignment_id=1" + start: python examples/static_react_task_with_worker_opinion/run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true + wait-on: "http://localhost:3000/?worker_id=x&id=1" browser: chrome headless: true diff --git a/.gitignore b/.gitignore index 35d4a14d2..8c61f2496 100644 --- a/.gitignore +++ b/.gitignore @@ -28,8 +28,6 @@ mephisto/scripts/metrics/* # Examples examples/simple_static_task/hydra_configs/conf/* !examples/simple_static_task/hydra_configs/conf/example*.yaml -!examples/simple_static_task/hydra_configs/conf/prolific_example.yaml -!examples/simple_static_task/hydra_configs/conf/onboarding_example*.yaml examples/**/build/* examples/form_composer_demo/preview/*_preview.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7f8945d72..5b9b5c7e2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,10 +23,10 @@ We actively welcome your pull requests. 6. If you haven't already, complete the Contributor License Agreement ("CLA"). ## Cypress Testing -For cypress testing the base url is: http://localhost:3000/?worker_id=x&assignment_id=1 +For cypress testing the base url is: http://localhost:3000/?worker_id=x&id=1 ### Running end-to-end tests on a task: -1. Run the task by running python run_task.py in the appropriate task folder +1. Run the task by running python `run_task__local__inhouse.py` in the appropriate task folder 2. In a separate terminal window go into the webapp directory and run `npm run test` 3. This should open a cypress app 4. It is advised to test in Chrome(Chrome, Electron, and Firefox are all the options) as this browser works well with Cypress. @@ -40,7 +40,7 @@ Suppose you ran the toxicity detection task and then closed it. This would use a If you then ran the mnist task, for example, then assignmentId=3 and assignmentId=4 would be used. While this task is running you can choose to run cypress tests in a different terminal window by going into the webapp folder and running `npm run test`. -These tests will fail because the base url of http://localhost:3000/?worker_id=x&assignment_id=1 is not associated with the mnist task, it is associated with the toxicity detection task. The correct react-elements will not show up. +These tests will fail because the base url of http://localhost:3000/?worker_id=x&id=1 is not associated with the mnist task, it is associated with the toxicity detection task. The correct react-elements will not show up. There is a way to fix this: * You can change the base url(found in the cypress.config.js file in the webapp folder) to the current url that you are on. @@ -51,7 +51,7 @@ If you are modifying either the `mephisto-core` or `mephisto-addons` packages yo The easiest way to do this is to run a task by doing: ```bash -python run_task.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true +python run_task__local__inhouse.py mephisto.task.post_install_script=link_mephisto_task.sh mephisto.task.force_rebuild=true ``` Setting `mephisto.task.force_rebuild=true` runs `npm build` before running your task. By default the task is only rebuilt if a file is changed in the webapp, not if a linked package is changed. @@ -60,7 +60,7 @@ Setting `mephisto.task.post_install_script=link_mephisto_task.sh` runs the `link Alternatively, these values can be set in the task's hydra_configs/conf yaml file if you want to forgo typing the above and just type ```bash -python run_task.py +python run_task__local__inhouse.py ``` instead. diff --git a/docker/docker-compose.dev.vscode.yml b/docker/docker-compose.dev.vscode.yml index 46896bfd8..92834cded 100644 --- a/docker/docker-compose.dev.vscode.yml +++ b/docker/docker-compose.dev.vscode.yml @@ -29,6 +29,6 @@ services: /tmp/debugpy --wait-for-client --listen 0.0.0.0:5678 - /mephisto/examples/simple_static_task/run_task.py + /mephisto/examples/simple_static_task/run_task__local__inhouse.py ", ] diff --git a/docs/web/docs/guides/how_to_contribute/frontend_development.md b/docs/web/docs/guides/how_to_contribute/frontend_development.md index e6da5aaa7..66cd85080 100644 --- a/docs/web/docs/guides/how_to_contribute/frontend_development.md +++ b/docs/web/docs/guides/how_to_contribute/frontend_development.md @@ -23,7 +23,7 @@ To setup your local codebase to auto-lint and avoid lint test failures for your This repo uses cypress to conduct frontend end-to-end tests. Tasks in the examples folder have cypress tests. To run the tests for a task: -* Launch the task using `python run_task.py`. +* Launch the task using `python run_task__local__inhouse.py`. * Open cypress by running `npm run test` in the tasks' webapp folder. * Choose the Chrome browser to run the tests (it is the most consistent). * Click one of the specs to run its tests. diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md b/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md index bf3a04c96..97fa0d326 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/docker.md @@ -22,7 +22,7 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` ## Customizing Docker settings @@ -50,5 +50,5 @@ docker-compose -f docker/docker-compose.local.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md b/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md index 473718301..c69e9d4dd 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/reusing_configs.md @@ -13,7 +13,7 @@ As you begin launching many Mephisto tasks, you may find that there are some spe Setting up profiles is pretty easy, and makes it so that you don't have to be writing architect and crowdprovider args on every launch: ``` -python run_task.py mephisto/architect=heroku mephisto/provider=mturk_sandbox mephisto.provider.requester_name=MY_REQUESTER +python run_task__local__inhouse.py mephisto/architect=heroku mephisto/provider=mturk_sandbox mephisto.provider.requester_name=MY_REQUESTER ``` Instead you can move these common configurations into a file in your `~/.mephisto/hydra_configs/profile` dir. @@ -61,9 +61,9 @@ mephisto: Then augmenting your launch configs is as easy as doing: ``` -python run_task.py +profile=local_testing +python run_task__local__inhouse.py +profile=local_testing ... -python run_task.py +profile=live_launch +python run_task__local__inhouse.py +profile=live_launch ``` Using `profile` can be an effective way to simplify the configuration for your most common workflows. diff --git a/docs/web/docs/guides/how_to_use/form_composer/embedding.md b/docs/web/docs/guides/how_to_use/form_composer/embedding.md index 98a613eb8..4990df5c4 100644 --- a/docs/web/docs/guides/how_to_use/form_composer/embedding.md +++ b/docs/web/docs/guides/how_to_use/form_composer/embedding.md @@ -11,7 +11,7 @@ sidebar_position: 4 A few tips if you wish to embed FormComposer in your custom application: - To extrapolate form config (and generate the `task_data.json` file), call the extrapolator function `mephisto.generators.generators_utils.config_validation.task_data_config.create_extrapolated_config` - - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/form_composer_demo/run_task_dynamic.py) module + - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/form_composer_demo/run_task_dynamic__local__inhouse.py) module - To use code insertions: - for custom validators: - Point `WEBAPP__GENERATOR__CUSTOM_VALIDATORS` backend env variable to the location of `custom_validators.js` module (before building all webapp applications) diff --git a/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md b/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md index ba9ef3707..9da372d8b 100644 --- a/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md +++ b/docs/web/docs/guides/how_to_use/providers/prolific/eligibility_requirements.md @@ -33,7 +33,7 @@ You can specify Prolific qualifications via Task config file, or directly in Tas ### Usage in shared state -Shared state can handle Prolific-supported qualifications. Example in `run_task.py`: +Shared state can handle Prolific-supported qualifications. Example in `run_task__ec2__prolific.py`: ```python shared_state.prolific_specific_qualifications = [ diff --git a/docs/web/docs/guides/how_to_use/review_app/overview.md b/docs/web/docs/guides/how_to_use/review_app/overview.md index e75f70ce0..99dbd8c3a 100644 --- a/docs/web/docs/guides/how_to_use/review_app/overview.md +++ b/docs/web/docs/guides/how_to_use/review_app/overview.md @@ -8,9 +8,12 @@ sidebar_position: 1 # Overview -Generally, to view/export the data, you could write a Python script using the Mephisto [`DataBrowser` class](https://github.com/facebookresearch/Mephisto/blob/main/mephisto/tools/data_browser.py) to access the submitted data. +Generally, to view/export the data, you could write a Python script using the Mephisto +[`DataBrowser` class](https://github.com/facebookresearch/Mephisto/blob/main/mephisto/tools/data_browser.py) to access the submitted data. -For example, for the `html-static-task-example` task such a script already exists in the task folder, called [`examine_results.py`](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/examine_results.py). (This file uses the Mephisto `DataBrowser` class through the helpers in `mephisto.tools.examine_utils`.) +For example, for the `html-static-task-example` task such a script already exists in the task folder, +called [`examine_results.py`](https://github.com/facebookresearch/Mephisto/blob/main/examples/simple_static_task/examine_results.py). +(This file uses the Mephisto `DataBrowser` class through the helpers in `mephisto.tools.examine_utils`.) Another example is TaskReview app, an application with convenient rich UI. diff --git a/docs/web/docs/guides/how_to_use/task_creation/task_run.md b/docs/web/docs/guides/how_to_use/task_creation/task_run.md index 52eb4e63a..a2489441b 100644 --- a/docs/web/docs/guides/how_to_use/task_creation/task_run.md +++ b/docs/web/docs/guides/how_to_use/task_creation/task_run.md @@ -9,7 +9,8 @@ sidebar_position: 1 # How task run works -Let's understand basic components of the task launch, such as configs and the `run_task.py` script. This will help with customization of tash launch behaviors. +Let's understand basic components of the task launch, such as configs and the `run_task__local__inhouse.py` script. +This will help with customization of tash launch behaviors. ### 3.1 Config registration @@ -17,7 +18,7 @@ Mephisto wires up to configuration using standard Hydra syntax, but with both `y Here's the config we've set up for this example: ```python -# examples/form_composer_demo/run_task.py +# examples/form_composer_demo/run_task__local__inhouse.py import os from omegaconf import DictConfig @@ -27,7 +28,7 @@ from mephisto.tools.scripts import build_custom_bundle from mephisto.tools.scripts import task_script -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: ``` @@ -36,7 +37,7 @@ This is all you really *need* to launch a Mephisto task! The `@task_script` deco Of course, there's quite a bit of 'magic' happening underneath the hood thanks to the script utilities. This version is explicit to show where you may add customization, and re-ordered for understanding: ```python -# modified examples/form_composer_demo/run_task.py +# modified examples/form_composer_demo/run_task__local__inhouse.py import os from dataclasses import dataclass from typing import Any @@ -49,7 +50,7 @@ from mephisto.tools.scripts import build_custom_bundle from mephisto.tools.scripts import task_script @dataclass -class MyTaskConfig(build_default_task_config('example_local_mock')): +class MyTaskConfig(build_default_task_config('example__local__inhouse')): custom_args: Any = 4 @task_script(config=MyTaskConfig) @@ -57,20 +58,20 @@ def main(operator: Operator, cfg: DictConfig) -> None: ``` In this snippet, we do a few things: -1. We set up the default [`conf`](https://hydra.cc/docs/tutorials/basic/your_first_app/config_file/) file to be `example_local_mock`, +1. We set up the default [`conf`](https://hydra.cc/docs/tutorials/basic/your_first_app/config_file/) file to be `example__local__inhouse`, using `build_default_task_config`, which returns a `TaskConfig` that we can extend. 2. We extend the returned `TaskConfig` with `MyTaskConfig`, which allows us to specify custom arguments. 3. We decorate the main, noting that the correct config is `MyTaskConfig`. Note that the `default_config_file` version of this simply takes care of the above steps inline in the decorator. -With all the above, we're able to just make edits to `example_local_mock.yaml` or make other configs in the `conf/` directory and route to them directly. +With all the above, we're able to just make edits to `example__local__inhouse.yaml` or make other configs in the `conf/` directory and route to them directly. ### 3.2 Invoking Mephisto Mephisto itself is actually invoked just a little later: ```python -@task_script(default_config_file="example_local_mock") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: # Build packages _build_custom_bundles(cfg) @@ -89,17 +90,17 @@ To ensure we're not frozen, the operator takes in a `log_rate` in seconds to pri Again we can look back at the `example_local_mock.yaml` file to see this setup: ```yaml -# examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +# examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse ``` -These ensure that, when not provided other arguments, we launch this task locally using a `LocalArchitect` and `MockProvider`. -With these defaults, this and other example tasks are run using a "local" architect, and a "mock" requester without arguments. +These ensure that, when not provided other arguments, we launch this task locally using a `LocalArchitect` and `InhouseProvider`. +With these defaults, this and other example tasks are run using a "local" architect, and a "inhouse" requester without arguments. The "local" architect is reponsible for running a server on your local machine to host the task, -and the "mock" requester lets *you* simulate a worker without using an external crowd-provider platform such as Prolific or MTurk to launch the task. +and the "inhouse" requester lets *you* simulate a worker without using an external crowd-provider platform such as Prolific or MTurk to launch the task. ### 3.4 `Unit` creation explained @@ -108,7 +109,7 @@ It's useful to understand how this happens. Taking a look at the config and data: ```yaml -# examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +# examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml #@package _global_ defaults: diff --git a/docs/web/docs/guides/how_to_use/video_annotator/embedding.md b/docs/web/docs/guides/how_to_use/video_annotator/embedding.md index 893409896..77b69f89c 100644 --- a/docs/web/docs/guides/how_to_use/video_annotator/embedding.md +++ b/docs/web/docs/guides/how_to_use/video_annotator/embedding.md @@ -11,7 +11,7 @@ sidebar_position: 4 A few tips if you wish to embed VideoAnnotator in your custom application: - To extrapolate annotator config (and generate the `task_data.json` file), call the extrapolator function `mephisto.generators.generators_utils.config_validation.task_data_config.create_extrapolated_config` - - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/video_annotator_demo/run_task_dynamic.py) module + - For a live example, you can explore the source code of [run_task_dynamic.py](https://github.com/facebookresearch/Mephisto/blob/main/examples/video_annotator_demo/run_task_dynamic__local__inhouse.py) module - To use code insertions: - for custom validators: - Point `WEBAPP__GENERATOR__CUSTOM_VALIDATORS` backend env variable to the location of `custom_validators.js` module (before building all webapp applications) diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md b/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md index 6f64d3606..47fa4eba7 100644 --- a/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md +++ b/docs/web/docs/guides/how_to_use/worker_quality/using_golds.md @@ -61,7 +61,7 @@ You can run an example project to try gold units for yourself. docker-compose -f docker/docker-compose.dev.yml up docker exec -it mephisto_dc bash cd /mephisto/examples/form_composer_demo -python ./run_task_with_gold_unit.py +python ./run_task_with_gold_unit__local__inhouse.py ``` The first unit that you will see will be the gold one. @@ -78,8 +78,8 @@ To get past these example gold units, provide these predefined values: For an in-depth look at code underlying this example, you can read these Python files in `examples/form_composer_demo` directory: -- `run_task_with_gold_unit.py` - script to configure and launch this Task -- `hydra_configs/conf/example_local_mock_with_gold_unit.yaml` - YAML configuration for this Task +- `run_task_with_gold_unit__local__inhouse.py` - script to configure and launch this Task +- `hydra_configs/conf/example_with_gold_unit__local__inhouse.yaml` - YAML configuration for this Task - `data/simple/gold_units/gold_units_data.json` - configuration for form that will be used specifically for gold units - `data/simple/gold_units/gold_units_validation.py` - logic of validating worker's output in gold unit form diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx b/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx index 76d263546..503b799fc 100644 --- a/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx +++ b/docs/web/docs/guides/how_to_use/worker_quality/using_onboarding.mdx @@ -37,7 +37,7 @@ refer to [using screening units](../using_screen_units), as Mephisto doesn't pay ### Things to note in the showcase: -- The `static_react_task` example is ran with the `onboarding_example` configuration enabled to ensure that onboarding page will be shown. +- The `static_react_task` example is ran with the `example_with_onboarding__local__inhouse` configuration enabled to ensure that onboarding page will be shown. - Worker "x" clicks the "Get Blocked" button and this doesn't allow the worker to progress - Worker "y" clicks the "Move To Main Task" button and this allows the worker to go to the main task. @@ -59,7 +59,7 @@ def handle_onboarding(onboarding_data): return True return False -@task_script(default_config_file="example.yaml") +@task_script(default_config_file="example__local__inhouse.yaml") def main(operator: Operator, cfg: DictConfig) -> None: ... shared_state = SharedStaticTaskState( @@ -72,9 +72,9 @@ def main(operator: Operator, cfg: DictConfig) -> None: ... ``` -### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/run_task.py) +### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/run_task__local__inhouse.py) -### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/hydra_configs/conf/onboarding_example.yaml) +### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/static_react_task/hydra_configs/conf/example_with_onboarding__local__inhouse.yaml) Unlike Screening and Gold units, Onboarding expects that you set up a custom frontend compared to your main task. You want to provide workers with an in-depth exploration of your task up-front diff --git a/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx b/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx index 833f25ad9..c0990c3a3 100644 --- a/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx +++ b/docs/web/docs/guides/how_to_use/worker_quality/using_screen_units.mdx @@ -33,7 +33,7 @@ or on specific 'test' data you believe it's easier to validate on. ### Things to note in the showcase: -- The `remote_procedure/mnist` example is ran with the `screening_example` configuration enabled to ensure that screening units are generated. +- The `remote_procedure/mnist` example is ran with the `example_with_screening__local__inhouse` configuration enabled to ensure that screening units are generated. - When a worker goes to an assignment for the first time they see a screening unit. - Drawing a "3" gives the worker the passing qualification - Drawing any number other than "3" gives the worker the blocked qualification @@ -64,7 +64,7 @@ to register the required qualifications and the screening validation function. A shortened version of the run script for the video above looks like: ```python -# run_task.py +# run_task__local__inhouse.py def my_screening_unit_generator(): """ @@ -85,7 +85,7 @@ def validate_screening_unit(unit: Unit): return True return False -@task_script(default_config_file="example.yaml") +@task_script(default_config_file="example__local__inhouse.yaml") def main(operator: Operator, cfg: DictConfig) -> None: is_using_screening_units = cfg.mephisto.blueprint["use_screening_task"] tasks: List[Dict[str, Any]] = [{"isScreeningUnit": False}] * cfg.num_tasks @@ -108,9 +108,9 @@ def main(operator: Operator, cfg: DictConfig) -> None: ... ``` -### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/run_task.py) +### See the full code [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/run_task__local__inhouse.py) -### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/hydra_configs/conf/screening_example.yaml) +### See hydra configuration [here](https://github.com/facebookresearch/Mephisto/blob/main/examples/remote_procedure/mnist/hydra_configs/conf/example_with_screening__local__inhouse.yaml) ### Simple example of `ScreeningComponent` diff --git a/docs/web/docs/guides/quickstart.md b/docs/web/docs/guides/quickstart.md index 117fb21dc..1e140389f 100644 --- a/docs/web/docs/guides/quickstart.md +++ b/docs/web/docs/guides/quickstart.md @@ -24,9 +24,9 @@ sidebar_position: 1 --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` -5. After the script finishes building, in your console output you will see URLs like this: `localhost:3000/?worker_id=x&assignment_id=1`. Each URL represents a unit of your Task. Copy them one-by-one into your browser, changing port from `3000` to `3001`. +5. After the script finishes building, in your console output you will see URLs like this: `localhost:3000/?worker_id=x&id=1`. Each URL represents a unit of your Task. Copy them one-by-one into your browser, changing port from `3000` to `3001`. 6. After completing all units, you will see your task automatically shut down 7. Now we are ready to review Task results. Run command: ```shell diff --git a/docs/web/docs/guides/tutorials/custom_react.md b/docs/web/docs/guides/tutorials/custom_react.md index b7451f1ca..98a58280b 100644 --- a/docs/web/docs/guides/tutorials/custom_react.md +++ b/docs/web/docs/guides/tutorials/custom_react.md @@ -26,7 +26,7 @@ Now that we're here, we should also set up your config file defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: task_source: ${task_dir}/webapp/build/bundle.js @@ -50,13 +50,13 @@ From the current directory, you should be able to execute the run script and get You can update the `task_name` and `link_task_source` values in your config and run the task like below ```bash -python run_task.py +python run_task__local__inhouse.py ``` or you can set them when you run the task: ```bash -python run_task.py mephisto.task.task_name=custom-react-tutorial-iterating mephisto.blueprint.link_task_source=true +python run_task__local__inhouse.py mephisto.task.task_name=custom-react-tutorial-iterating mephisto.blueprint.link_task_source=true ``` This will launch a simple task where an annotator is supposed to note a sentence as being good or bad. Clicking a button auto-submits the task. In the next sections we'll add other content. @@ -123,7 +123,7 @@ static_task_data=[ At this point you can run the task again. ```bash -python run_task.py mephisto.task.task_name=custom-react-tutorial-iterating +python run_task__local__inhouse.py mephisto.task.task_name=custom-react-tutorial-iterating ``` Note the first one you work on displays your new edited text. But what about the new `edited_by_requester` field? @@ -264,7 +264,7 @@ onClick={() => onSubmit({ rating: "bad", editedText: editedText })} Let's launch one last time. ``` -python run_task.py mephisto.task.task_name=custom-react-tutorial-iterating +python run_task__local__inhouse.py mephisto.task.task_name=custom-react-tutorial-iterating ``` And just like that we're able to see an input field: ![](/static_task_with_corrections_box.png) diff --git a/docs/web/docs/guides/tutorials/first_task.md b/docs/web/docs/guides/tutorials/first_task.md index 15f5c07ca..cd9e6a31f 100644 --- a/docs/web/docs/guides/tutorials/first_task.md +++ b/docs/web/docs/guides/tutorials/first_task.md @@ -27,23 +27,23 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py ``` or without using Docker ```shell cd /mephisto/examples/form_composer_demo/ -python run_task.py +python run_task__local__inhouse.py ``` This will launch a local HTTP server with the task hosted, based on the default configuration options: ```yaml -# examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml +# examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse ``` We'll dig into *how* this works [later](#33-default-abstraction-usage). @@ -55,13 +55,13 @@ By default, the task should be hosted at [http://localhost:3000](http://localhos This default is set by the `LocalArchitect`, which is used based on the `- /mephisto/architect: local` line above. Navigating to this address should show you the preview view for the task. -Actually being able to access this task is done by providing `worker_id` and `assignment_id` URL params, like `localhost:3000/?worker_id=x&assignment_id=1`. -The `MockProvider` interprets these to be a test worker, which you can use to try out tasks. +Actually being able to access this task is done by providing `worker_id` and `assignment_id` URL params, like `localhost:3000/?worker_id=x&id=1`. +The `InhouseProvider` interprets these to be a test worker, which you can use to try out tasks. Try navigating here and completing a task by selecting a value (no need to use the file upload). After hitting submit you'll note that the window alerts you to the data that was sent to the Mephisto backend. -To work on another task, you'll want to change the `assignment_id` in your url. This will let `Worker` "x" work on another task. +To work on another task, you'll want to change the `id` in your url. This will let `Worker` "x" work on another task. Complete and submit this too. If you try to work on another task, you'll note that the system states you've worked on the maximum number of tasks. @@ -138,7 +138,7 @@ For our given example task, the values we are using for these options are availa As a simple starting point, we can try launching the server on a different port. Right now the default is `3000`, but with the following command we can set that ourselves: ``` -python run_task.py mephisto.architect.port=1234 +python run_task__local__inhouse.py mephisto.architect.port=1234 ``` This should launch the same task, but now available on the port `1234` rather than `3000`. @@ -151,7 +151,7 @@ Once we see the task is running, we can shut down with `Ctrl-C`. ### 2.3 Using yaml configurations While it makes sense to update some parameters on the command line while iterating, generally it's easier to extend the `conf` files directly, then apply all of the options by overriding `conf`. -Try copying the `examples/form_composer_demo/hydra_configs/conf/example_local_mock.yaml` file into `examples/form_composer_demo/hydra_configs/conf/my_config.yaml`. +Try copying the `examples/form_composer_demo/hydra_configs/conf/example__local__inhouse.yaml` file into `examples/form_composer_demo/hydra_configs/conf/my_config.yaml`. Also file `examples/form_composer_demo/data/simple/task_data.json` into `examples/form_composer_demo/data/simple/my_task_data.json` ```yaml @@ -161,7 +161,7 @@ Also file `examples/form_composer_demo/data/simple/task_data.json` into `example defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: @@ -187,14 +187,14 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task.py conf=my_config + python /mephisto/examples/form_composer_demo/run_task__local__inhouse.py conf=my_config ``` or without using Docker ```bash cd /mephisto/examples/form_composer_demo/ -python run_task.py conf=my_config +python run_task__local__inhouse.py conf=my_config ``` Now you'll notice that Mephisto launches your task under the new task name: @@ -205,16 +205,16 @@ Now you'll notice that Mephisto launches your task under the new task name: > **A note on `task_name`s:** > The `task_name` parameter is particularly important for setting up workflows. Many of Mephisto's features are shared under a specific `task_name` namespace, including review flows and unit completion maximums per worker per namespace. Later [guides](../workflows) go more in-depth on best practices. -Navigating to a task (`localhost:3000/?worker_id=x&assignment_id=1` or woth Docker on `3001` port) should now show you a task loaded from a different data file. +Navigating to a task (`localhost:3000/?worker_id=x&id=1` or woth Docker on `3001` port) should now show you a task loaded from a different data file. Completing this task will lead Mephisto to shut down cleanly. You can now review this task with the review script again, this time providing the task name `My first own task`. ### 2.4 (optional) Launch your task live -So far we've been launching our task in a testing mode (with `local` architect and `mock` provider). +So far we've been launching our task in a testing mode (with `local` architect and `inhouse` provider). -To make your Task page accessible to the others (e.g. external workers), you will either need to obtain a publicly accessible static IP address for you machine, or launch the task with a non-`mock` human cloud platform and a non-`local` architect. +To make your Task page accessible to the others (e.g. external workers), you will either need to obtain a publicly accessible static IP address for you machine, or launch the task with a non-`inhouse` human cloud platform and a non-`local` architect. In the below examples we consider an EC2 architect and a few common providers. This configuration is different from the testing mode: - EC2 architect will build a temporary EC2 server in the cloud, which will: @@ -244,10 +244,10 @@ docker-compose -f docker/docker-compose.local.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py + python /mephisto/examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py ``` -Note that this command reads task parameters from task config file referenced within the `run_task_dynamic_ec2_mturk_sandbox.py` script. +Note that this command reads task parameters from task config file referenced within the `run_task_dynamic__ec2__mturk_sandbox.py` script. Generally, you would want to adjust task config file to your needs. You don't need to do this step *right now*, however it's important for understanding how to take one of these tasks live. @@ -298,14 +298,14 @@ docker-compose -f docker/docker-compose.dev.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py conf=my_config + python /mephisto/examples/form_composer_demo/run_task_dynamic__ec2__mturk_sandbox.py conf=my_config ``` or without using Docker ```bash cd /mephisto/examples/form_composer_demo/ -python run_task_dynamic_ec2_mturk_sandbox.py conf=my_config +python run_task_dynamic__ec2__mturk_sandbox.py conf=my_config ``` Mephisto should print out links to view your task on the mturk sandbox, @@ -334,7 +334,7 @@ docker-compose -f docker/docker-compose.local.yml run \ --build \ --publish 3001:3000 \ --rm mephisto_dc \ - python /mephisto/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py + python /mephisto/examples/form_composer_demo/run_task_dynamic__ec2__prolific.py ``` The other steps are similar to launching on MTurk, just task config file is a bit different. diff --git a/docs/web/docs/guides/tutorials/worker_controls.md b/docs/web/docs/guides/tutorials/worker_controls.md index 92a362c8e..a6c22691e 100644 --- a/docs/web/docs/guides/tutorials/worker_controls.md +++ b/docs/web/docs/guides/tutorials/worker_controls.md @@ -21,14 +21,14 @@ cp -r examples/static_react_task/ tmp/onboarding_tutorial/ cd tmp/onboarding_tutorial/ ``` -The key for using onboarding is the `onboarding_qualification`, as we can see in `onboarding_example.yaml`: +The key for using onboarding is the `onboarding_qualification`, as we can see in `example_with_onboarding__local__inhouse.yaml`: ```yaml # hydra_configs/conf/onboarding_example.yaml @package _global_ defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: ... @@ -37,7 +37,7 @@ mephisto: ... task_name: onboarding-tutorial-iterating # Add a custom task name to group the results ``` -When Mephisto sees the `mephisto.blueprint.onboarding_qualification` argument, it adds a step to the task that requires workers pass onboarding in order to move on to the main task. Why don't we see it in action? You can launch the onboarding version of this task with `python run_task.py conf=onboarding_example`. Once launched, you can follow through to [`localhost:3000/?worker_id=x&assignment_id=1`](localhost:3000/?worker_id=x&assignment_id=1). +When Mephisto sees the `mephisto.blueprint.onboarding_qualification` argument, it adds a step to the task that requires workers pass onboarding in order to move on to the main task. Why don't we see it in action? You can launch the onboarding version of this task with `python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse`. Once launched, you can follow through to [`localhost:3000/?worker_id=x&id=1`](localhost:3000/?worker_id=x&id=1). You should be greeted with a panel claiming it is the onboarding task, suggesting that you click a button to move to the main task. Clicking this button puts you into the example static react task. It's not a very informative task as is, so let's add some details into it! Shut down the server and we'll get back to it @@ -156,7 +156,7 @@ function OnboardingComponent({ onboardingData, onSubmit }) { ); } ``` -With this, let's launch again with `python run_task.py conf=onboarding_example`. Once launched, you can follow through to [`localhost:3000/?worker_id=y&assignment_id=1`](localhost:3000/?worker_id=y&assignment_id=1). **Note** that now we're accessing as worker `y`. We do this because worker `x` has already completed onboarding for our task name, and so they'd skip it! Onboarding is only surfaced the _first time_ a worker completes a task by a given `onboarding_qualification`. +With this, let's launch again with `python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse`. Once launched, you can follow through to [`localhost:3000/?worker_id=y&assignment_id=1`](localhost:3000/?worker_id=y&id=1). **Note** that now we're accessing as worker `y`. We do this because worker `x` has already completed onboarding for our task name, and so they'd skip it! Onboarding is only surfaced the _first time_ a worker completes a task by a given `onboarding_qualification`. ![](/tutorial_onboarding_new_interface.png) @@ -207,7 +207,7 @@ And that's it! Let's move forward to test that this works. ### 2.4 Testing -Let's launch again with `python run_task.py conf=onboarding_example`. Once launched, we should open a tab for [`worker a`](localhost:3000/?worker_id=a&assignment_id=1) and [`worker b`](localhost:3000/?worker_id=b&assignment_id=1). Complete the onboarding correctly as one worker, and assert that you're able to make it through to the task! Complete it incorrectly as the other, and note that you are not qualified to work on the task. With this, you have a functioning basic onboarding task! +Let's launch again with `python run_task__local__inhouse.py conf=example_with_onboarding__local__inhouse`. Once launched, we should open a tab for [`worker a`](localhost:3000/?worker_id=a&id=1) and [`worker b`](localhost:3000/?worker_id=b&id=1). Complete the onboarding correctly as one worker, and assert that you're able to make it through to the task! Complete it incorrectly as the other, and note that you are not qualified to work on the task. With this, you have a functioning basic onboarding task! ## 3. Reviews with onboarding diff --git a/docs/web/docs/guides/tutorials/workflows.md b/docs/web/docs/guides/tutorials/workflows.md index 1c5c7cd68..57b79591f 100644 --- a/docs/web/docs/guides/tutorials/workflows.md +++ b/docs/web/docs/guides/tutorials/workflows.md @@ -36,7 +36,7 @@ Generally it's best to put the `task_name` _into_ your Hydra `.yaml` config and defaults: - /mephisto/blueprint: static_react_task - /mephisto/architect: local - - /mephisto/provider: mock + - /mephisto/provider: inhouse mephisto: blueprint: ... @@ -71,7 +71,7 @@ This also means you can go back and find the configuration details for a specifi For complex tasks with many configuration arguments, we make it possible to add arguments to your run script to simplify your workflows and allow for code reuse. For instance, say you had the following script: ```python -# examples/static_react_task/run_task.py +# examples/static_react_task/run_task__local__inhouse.py from mephisto.operations.operator import Operator from mephisto.tools.scripts import task_script, build_and_return_custom_bundle from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import ( @@ -81,7 +81,7 @@ from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint impo from omegaconf import DictConfig -@task_script(default_config_file="example") +@task_script(default_config_file="example__local__inhouse") def main(operator: Operator, cfg: DictConfig) -> None: def onboarding_always_valid(onboarding_data): return True diff --git a/docs/web/static/python_api/mephisto/abstractions/blueprints.html b/docs/web/static/python_api/mephisto/abstractions/blueprints.html index 38a535418..8e02eded1 100644 --- a/docs/web/static/python_api/mephisto/abstractions/blueprints.html +++ b/docs/web/static/python_api/mephisto/abstractions/blueprints.html @@ -126,7 +126,7 @@
SharedTaskState
Blueprint
MixinsBlueprints sometimes share some component functionality that may be useful across a multitude of tasks. We capture these in mixins. Mephisto is able to recognize certain mixins in order to complete additional operations, however custom mixins may help cut down on boiler plate in common run_task.py
scripts. As your tasks mature, we suggest utilizing blueprint mixins to share common workflows and design patterns you observe.
Blueprints sometimes share some component functionality that may be useful across a multitude of tasks. We capture these in mixins. Mephisto is able to recognize certain mixins in order to complete additional operations, however custom mixins may help cut down on boiler plate in common run_task__local__inhouse.py
scripts. As your tasks mature, we suggest utilizing blueprint mixins to share common workflows and design patterns you observe.
OnboardingRequired
MockBlueprint