From 80955ed51bd61d7916ad08775e5ba6fbd5629cdf Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Thu, 29 Feb 2024 23:06:01 -0500 Subject: [PATCH 1/9] Added code insertions feature for Form Composer --- .../data/dynamic/form_config.json | 7 +- .../insertions/about_section_items.html | 5 + .../dynamic/insertions/custom_validators.js | 23 +++ .../dynamic/insertions/form_instruction.html | 10 ++ .../dynamic/separate_token_values_config.json | 1 + .../data/dynamic/task_data.json | 26 +-- .../dynamic/token_sets_values_config.json | 6 +- .../form_composer_demo/run_task_dynamic.py | 14 +- .../run_task_dynamic_ec2_mturk_sandbox.py | 13 +- .../run_task_dynamic_ec2_prolific.py | 13 +- ...ask_dynamic_presigned_urls_ec2_prolific.py | 5 +- .../webapp/src/components/core_components.jsx | 5 + .../webapp/webpack.config.js | 4 + .../webapp/webpack.config.review.js | 4 + mephisto/client/cli.py | 17 +- mephisto/generators/form_composer/README.md | 157 +++++++++++++++--- .../config_validation/common_validation.py | 40 +++++ .../config_validation_constants.py | 6 + .../config_validation/form_config.py | 8 +- .../config_validation/task_data_config.py | 73 ++++++-- .../form_composer/config_validation/utils.py | 26 +++ .../data/insertions/custom_validators.js | 0 .../webapp/src/components/core_components.jsx | 2 + .../form_composer/webapp/webpack.config.js | 3 + .../webapp/webpack.config.review.js | 3 + .../InReviewFileModal/InReviewFileModal.tsx | 9 +- .../src/FormComposer/FormComposer.css | 44 +++++ .../src/FormComposer/FormComposer.js | 7 +- .../src/FormComposer/constants.js | 39 +++++ .../src/FormComposer/fields/FileField.js | 46 +++++ .../src/FormComposer/validation/helpers.js | 16 +- 31 files changed, 550 insertions(+), 82 deletions(-) create mode 100644 examples/form_composer_demo/data/dynamic/insertions/about_section_items.html create mode 100644 examples/form_composer_demo/data/dynamic/insertions/custom_validators.js create mode 100644 examples/form_composer_demo/data/dynamic/insertions/form_instruction.html create mode 100644 mephisto/generators/form_composer/data/insertions/custom_validators.js diff --git a/examples/form_composer_demo/data/dynamic/form_config.json b/examples/form_composer_demo/data/dynamic/form_config.json index fc47f682f..1a200fc5b 100644 --- a/examples/form_composer_demo/data/dynamic/form_config.json +++ b/examples/form_composer_demo/data/dynamic/form_config.json @@ -1,12 +1,12 @@ { "form": { "title": "Form example", - "instruction": "Please answer all questions to the best of your ability as part of our study.", + "instruction": "insertions/form_instruction.html", "sections": [ { "name": "section_about", "title": "About you", - "instruction": "

Please introduce yourself. We would like to know more about your:

", + "instruction": "{{about_section_instruction}}", "collapsable": false, "fieldsets": [ { @@ -154,7 +154,8 @@ "tooltip": "Your bio in a few paragraphs", "type": "textarea", "validators": { - "required": false + "required": false, + "checkForbiddenWords": true }, "value": "" } diff --git a/examples/form_composer_demo/data/dynamic/insertions/about_section_items.html b/examples/form_composer_demo/data/dynamic/insertions/about_section_items.html new file mode 100644 index 000000000..feb4977b1 --- /dev/null +++ b/examples/form_composer_demo/data/dynamic/insertions/about_section_items.html @@ -0,0 +1,5 @@ + diff --git a/examples/form_composer_demo/data/dynamic/insertions/custom_validators.js b/examples/form_composer_demo/data/dynamic/insertions/custom_validators.js new file mode 100644 index 000000000..aa4fcc580 --- /dev/null +++ b/examples/form_composer_demo/data/dynamic/insertions/custom_validators.js @@ -0,0 +1,23 @@ +const FORBIDDEN_WORDS = ["fool", "silly", "stupid"]; + +export function checkForbiddenWords(field, value, check) { + if (!check) { + return null; + } + + let invalid = false; + + FORBIDDEN_WORDS.forEach((word) => { + if (value.includes(word)) { + invalid = true; + } + }); + + if (invalid) { + return `Field cannot contain any of these words: ${FORBIDDEN_WORDS.join( + ", " + )}.`; + } + + return null; +} diff --git a/examples/form_composer_demo/data/dynamic/insertions/form_instruction.html b/examples/form_composer_demo/data/dynamic/insertions/form_instruction.html new file mode 100644 index 000000000..d9fcbafc4 --- /dev/null +++ b/examples/form_composer_demo/data/dynamic/insertions/form_instruction.html @@ -0,0 +1,10 @@ +
+ Please answer all questions to the best of your ability as part of our + study. +
+ + diff --git a/examples/form_composer_demo/data/dynamic/separate_token_values_config.json b/examples/form_composer_demo/data/dynamic/separate_token_values_config.json index 2728dfd06..db4ff25b2 100644 --- a/examples/form_composer_demo/data/dynamic/separate_token_values_config.json +++ b/examples/form_composer_demo/data/dynamic/separate_token_values_config.json @@ -1,4 +1,5 @@ { "company_name": ["Facebook", "Mephisto"], + "about_section_items": ["insertions/about_section_items.html"], "since_age": [18] } diff --git a/examples/form_composer_demo/data/dynamic/task_data.json b/examples/form_composer_demo/data/dynamic/task_data.json index 48dc38683..4ab999d73 100644 --- a/examples/form_composer_demo/data/dynamic/task_data.json +++ b/examples/form_composer_demo/data/dynamic/task_data.json @@ -2,12 +2,12 @@ { "form": { "title": "Form example", - "instruction": "Please answer all questions to the best of your ability as part of our study.", + "instruction": "
Please answer all questions to the best of your ability as part of our study.
\n\n\n", "sections": [ { "name": "section_about", "title": "About you", - "instruction": "

Please introduce yourself. We would like to know more about your:

", + "instruction": "

Please introduce yourself. We would like to know more about your:

\n\n", "collapsable": false, "fieldsets": [ { @@ -50,12 +50,12 @@ { "fields": [ { - "help": "We may contact you later at your Mephisto email for additional information", + "help": "We may contact you later at your Facebook email for additional information", "id": "id_email", - "label": "Email address for Mephisto", + "label": "Email address for Facebook", "name": "email", "placeholder": "user@mephisto.ai", - "tooltip": "Email address for Mephisto", + "tooltip": "Email address for Facebook", "type": "email", "validators": { "required": true, @@ -155,7 +155,8 @@ "tooltip": "Your bio in a few paragraphs", "type": "textarea", "validators": { - "required": false + "required": false, + "checkForbiddenWords": true }, "value": "" } @@ -313,12 +314,12 @@ { "form": { "title": "Form example", - "instruction": "Please answer all questions to the best of your ability as part of our study.", + "instruction": "
Please answer all questions to the best of your ability as part of our study.
\n\n\n", "sections": [ { "name": "section_about", "title": "About you", - "instruction": "

Please introduce yourself. We would like to know more about your:

", + "instruction": "

Please introduce yourself. We would like to know more about your:

\n\n", "collapsable": false, "fieldsets": [ { @@ -361,12 +362,12 @@ { "fields": [ { - "help": "We may contact you later at your Facebook email for additional information", + "help": "We may contact you later at your Mephisto email for additional information", "id": "id_email", - "label": "Email address for Facebook", + "label": "Email address for Mephisto", "name": "email", "placeholder": "user@mephisto.ai", - "tooltip": "Email address for Facebook", + "tooltip": "Email address for Mephisto", "type": "email", "validators": { "required": true, @@ -466,7 +467,8 @@ "tooltip": "Your bio in a few paragraphs", "type": "textarea", "validators": { - "required": false + "required": false, + "checkForbiddenWords": true }, "value": "" } diff --git a/examples/form_composer_demo/data/dynamic/token_sets_values_config.json b/examples/form_composer_demo/data/dynamic/token_sets_values_config.json index 2747d91b8..59280361b 100644 --- a/examples/form_composer_demo/data/dynamic/token_sets_values_config.json +++ b/examples/form_composer_demo/data/dynamic/token_sets_values_config.json @@ -1,13 +1,15 @@ [ { "tokens_values": { - "company_name": "Mephisto", + "company_name": "Facebook", + "about_section_items": "insertions/about_section_items.html", "since_age": 18 } }, { "tokens_values": { - "company_name": "Facebook", + "company_name": "Mephisto", + "about_section_items": "insertions/about_section_items.html", "since_age": 18 } } diff --git a/examples/form_composer_demo/run_task_dynamic.py b/examples/form_composer_demo/run_task_dynamic.py index 17c2d9185..28372d9e6 100644 --- a/examples/form_composer_demo/run_task_dynamic.py +++ b/examples/form_composer_demo/run_task_dynamic.py @@ -14,6 +14,9 @@ from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) from mephisto.operations.operator import Operator from mephisto.tools.scripts import build_custom_bundle from mephisto.tools.scripts import task_script @@ -74,22 +77,25 @@ def generate_task_data_json_config(): based on existing form and tokens values config files """ app_path = os.path.dirname(os.path.abspath(__file__)) - data_path = os.path.join(app_path, "data") + data_path = os.path.join(app_path, "data", "dynamic") - form_config_path = os.path.join(data_path, "dynamic", FORM_COMPOSER__FORM_CONFIG_NAME) + form_config_path = os.path.join(data_path, FORM_COMPOSER__FORM_CONFIG_NAME) token_sets_values_config_path = os.path.join( data_path, - "dynamic", FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME, ) - task_data_config_path = os.path.join(data_path, "dynamic", FORM_COMPOSER__DATA_CONFIG_NAME) + task_data_config_path = os.path.join(data_path, FORM_COMPOSER__DATA_CONFIG_NAME) create_extrapolated_config( form_config_path=form_config_path, token_sets_values_config_path=token_sets_values_config_path, task_data_config_path=task_data_config_path, + data_path=data_path, ) + # Set env var for `custom_validators.js` + set_custom_validators_js_env_var(data_path) + if __name__ == "__main__": generate_task_data_json_config() diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py b/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py index f158bbec3..6c3d8fa8b 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py +++ b/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py @@ -20,6 +20,9 @@ from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) from mephisto.generators.form_composer.constants import TOKEN_END_REGEX from mephisto.generators.form_composer.constants import TOKEN_START_REGEX from mephisto.operations.operator import Operator @@ -94,15 +97,14 @@ def generate_data_json_config(): based on existing form and tokens values config files """ app_path = os.path.dirname(os.path.abspath(__file__)) - data_path = os.path.join(app_path, "data") + data_path = os.path.join(app_path, "data", "dynamic") - form_config_path = os.path.join(data_path, "dynamic", FORM_COMPOSER__FORM_CONFIG_NAME) + form_config_path = os.path.join(data_path, FORM_COMPOSER__FORM_CONFIG_NAME) token_sets_values_config_path = os.path.join( data_path, - "dynamic", FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME, ) - task_data_config_path = os.path.join(data_path, "dynamic", FORM_COMPOSER__DATA_CONFIG_NAME) + task_data_config_path = os.path.join(data_path, FORM_COMPOSER__DATA_CONFIG_NAME) create_extrapolated_config( form_config_path=form_config_path, @@ -110,6 +112,9 @@ def generate_data_json_config(): task_data_config_path=task_data_config_path, ) + # Set env var for `custom_validators.js` + set_custom_validators_js_env_var(data_path) + def generate_preview_html(): """ diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py index daf11efcf..db61b7b9d 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py @@ -18,6 +18,9 @@ from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) from mephisto.operations.operator import Operator from mephisto.tools.scripts import build_custom_bundle from mephisto.tools.scripts import task_script @@ -96,15 +99,14 @@ def generate_data_json_config(): based on existing form and tokens values config files """ app_path = os.path.dirname(os.path.abspath(__file__)) - data_path = os.path.join(app_path, "data") + data_path = os.path.join(app_path, "data", "dynamic") - form_config_path = os.path.join(data_path, "dynamic", FORM_COMPOSER__FORM_CONFIG_NAME) + form_config_path = os.path.join(data_path, FORM_COMPOSER__FORM_CONFIG_NAME) token_sets_values_config_path = os.path.join( data_path, - "dynamic", FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME, ) - task_data_config_path = os.path.join(data_path, "dynamic", FORM_COMPOSER__DATA_CONFIG_NAME) + task_data_config_path = os.path.join(data_path, FORM_COMPOSER__DATA_CONFIG_NAME) create_extrapolated_config( form_config_path=form_config_path, @@ -112,6 +114,9 @@ def generate_data_json_config(): task_data_config_path=task_data_config_path, ) + # Set env var for `custom_validators.js` + set_custom_validators_js_env_var(data_path) + if __name__ == "__main__": generate_data_json_config() diff --git a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py index 3f11b51cf..5500dec9a 100644 --- a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py @@ -93,21 +93,18 @@ def generate_data_json_config(): based on existing form and tokens values config files """ app_path = os.path.dirname(os.path.abspath(__file__)) - data_path = os.path.join(app_path, "data") + data_path = os.path.join(app_path, "data", "dynamic_presigned_urls") form_config_path = os.path.join( data_path, - "dynamic_presigned_urls", FORM_COMPOSER__FORM_CONFIG_NAME, ) token_sets_values_config_path = os.path.join( data_path, - "dynamic_presigned_urls", FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME, ) task_data_config_path = os.path.join( data_path, - "dynamic_presigned_urls", FORM_COMPOSER__DATA_CONFIG_NAME, ) diff --git a/examples/form_composer_demo/webapp/src/components/core_components.jsx b/examples/form_composer_demo/webapp/src/components/core_components.jsx index ceb95b52e..e4537e498 100644 --- a/examples/form_composer_demo/webapp/src/components/core_components.jsx +++ b/examples/form_composer_demo/webapp/src/components/core_components.jsx @@ -7,6 +7,9 @@ import React from "react"; import { FormComposer } from "react-form-composer"; +// Required import for custom validators +import * as customValidators from "custom-validators"; + function LoadingScreen() { return Loading...; } @@ -41,6 +44,8 @@ function FormComposerBaseFrontend({ data={initialConfigFormData} onSubmit={onSubmit} finalResults={finalResults} + // Required for custom validators + customValidators={customValidators} /> ); diff --git a/examples/form_composer_demo/webapp/webpack.config.js b/examples/form_composer_demo/webapp/webpack.config.js index 2ffa5a115..146c1ebba 100644 --- a/examples/form_composer_demo/webapp/webpack.config.js +++ b/examples/form_composer_demo/webapp/webpack.config.js @@ -25,6 +25,10 @@ module.exports = { __dirname, "../../../packages/react-form-composer" ), + // Required for custom validators + "custom-validators": path.resolve( + process.env.WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS + ), }, fallback: { net: false, diff --git a/examples/form_composer_demo/webapp/webpack.config.review.js b/examples/form_composer_demo/webapp/webpack.config.review.js index e2c644582..6b21a1c5b 100644 --- a/examples/form_composer_demo/webapp/webpack.config.review.js +++ b/examples/form_composer_demo/webapp/webpack.config.review.js @@ -21,6 +21,10 @@ module.exports = { __dirname, "../../../packages/react-form-composer" ), + // Required for custom validators + "custom-validators": path.resolve( + process.env.WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS + ), }, fallback: { net: false, diff --git a/mephisto/client/cli.py b/mephisto/client/cli.py index e31fb67b7..e1fe70b6a 100644 --- a/mephisto/client/cli.py +++ b/mephisto/client/cli.py @@ -15,6 +15,7 @@ from rich_click import RichCommand from rich_click import RichGroup +import mephisto.scripts.form_composer.rebuild_all_apps as rebuild_all_apps_form_composer import mephisto.scripts.heroku.initialize_heroku as initialize_heroku import mephisto.scripts.local_db.clear_worker_onboarding as clear_worker_onboarding_local_db import mephisto.scripts.local_db.load_data_to_mephisto_db as load_data_local_db @@ -26,23 +27,24 @@ import mephisto.scripts.mturk.cleanup as cleanup_mturk import mephisto.scripts.mturk.identify_broken_units as identify_broken_units_mturk import mephisto.scripts.mturk.launch_makeup_hits as launch_makeup_hits_mturk -import mephisto.scripts.mturk.print_outstanding_hit_status as print_outstanding_hit_status_mturk import mephisto.scripts.mturk.print_outstanding_hit_status as soft_block_workers_by_mturk_id_mturk -import mephisto.scripts.form_composer.rebuild_all_apps as rebuild_all_apps_form_composer from mephisto.client.cli_commands import get_wut_arguments +from mephisto.generators.form_composer.config_validation.separate_token_values_config import ( + update_separate_token_values_config_with_file_urls, +) from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) from mephisto.generators.form_composer.config_validation.task_data_config import ( verify_form_composer_configs, ) -from mephisto.generators.form_composer.config_validation.separate_token_values_config import ( - update_separate_token_values_config_with_file_urls, -) from mephisto.generators.form_composer.config_validation.token_sets_values_config import ( update_token_sets_values_config_with_premutated_data, ) from mephisto.generators.form_composer.config_validation.utils import is_s3_url +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) from mephisto.operations.registry import get_valid_provider_types from mephisto.tools.scripts import build_custom_bundle from mephisto.utils.rich import console @@ -465,6 +467,9 @@ def form_composer(task_data_config_only: bool = True): # Change dir to app dir os.chdir(app_path) + # Set env var for `custom_validators.js` + set_custom_validators_js_env_var(app_data_path) + verify_form_composer_configs( task_data_config_path=task_data_config_path, task_data_config_only=task_data_config_only, @@ -552,6 +557,7 @@ def form_composer_config( token_sets_values_config_path=token_sets_values_config_path, separate_token_values_config_path=separate_token_values_config_path, task_data_config_only=False, + data_path=app_data_path, ) print(f"Finished configs verification") @@ -590,6 +596,7 @@ def form_composer_config( form_config_path=form_config_path, token_sets_values_config_path=token_sets_values_config_path, task_data_config_path=task_data_config_path, + data_path=app_data_path, ) print(f"[green]Finished successfully[/green]") diff --git a/mephisto/generators/form_composer/README.md b/mephisto/generators/form_composer/README.md index c3681abd9..69ed05df7 100644 --- a/mephisto/generators/form_composer/README.md +++ b/mephisto/generators/form_composer/README.md @@ -96,7 +96,9 @@ and place it in `generators/form-composer/data` directory. - If you want to slightly vary your form within a Task (by inserting different values into its text), you need to add two files (that will be used to auto-generate `task_data.json` file): - `token_sets_values_config.json` containing a JSON array of objects (each with one key `tokens_values` and value representing name-value pairs for a set of text tokens to be used in one form version). - `form_config.json` containing a single JSON object with one key `form`. -- For more detail, read on about dynamic form configs. + - For more details, read on about dynamic form configs. +- If you want to insert code (HTML or JS) into your form config, you need to create `insertions` directory in the form config directory, and place these files there + - For more details, read on about insertions. For detailed structure of each config file, see [Config file reference](#config-file-reference). @@ -112,9 +114,22 @@ Working config examples are provided in `examples/form_composer_demo/data` direc A few tips if you wish to embed FormComposer in your custom application: -- to extrapolate form config (and generate the `task_data.json` file), call the extrapolator function `mephisto.generators.form_composer.configs_validation.extrapolated_config.create_extrapolated_config` +- To extrapolate form config (and generate the `task_data.json` file), call the extrapolator function `mephisto.generators.form_composer.configs_validation.extrapolated_config.create_extrapolated_config` - For a live example, you can explore the source code of [run_task_dynamic.py](/examples/form_composer_demo/run_task_dynamic.py) module - +- To use code insertions: + - Point `WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS` backend env variable to the location of `custom_validators.js` module (before building all webapp applications) + - When using `FormComposer` component, import validators with `import * as customValidators from "custom-validators";` and pass them to your `FormComposer` component as an argument: `customValidators={customValidators}` + - Set this alias in your webpack config (to avoid build-time exception that `custom-validators` cannot be found): + ```js + resolve: { + alias: { + ... + "custom-validators": path. resolve( + process.env.WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS + ), + }, + } + ``` --- @@ -346,24 +361,121 @@ mephisto form_composer_config --update-file-location-values "https://s3.amazonaw ``` This is how URL pre-signing works: - - When a worker opens the Task page and the form HTML is generated, it will contain so-called "procedure tokens", i.e. token values that look like this: `{{getMultiplePresignedUrls()}}` +- When a worker opens the Task page and the form HTML is generated, it will contain so-called "procedure tokens", i.e. token values that look like this: `{{getMultiplePresignedUrls()}}` - the "wrapper" part of a procedure token is the name of a Javascript function that will render itself dynamically (e.g. by calling some remote API to receive additional data) - the argument part is the argument value provided suring the function call - - As soon as the form HTML is in place, the remote procedure gets called - - Mephisto's predefined remote procedure generates presigned URL, and its expiration starts ticking +- As soon as the form HTML is in place, the remote procedure gets called +- Mephisto's predefined remote procedure generates presigned URL, and its expiration starts ticking Presigned S3 URLs use the following environment variables: - - Required: valid AWS credentials: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` +- Required: valid AWS credentials: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_DEFAULT_REGION` form_composer_config` command) - - Optional: URL expiration time `S3_URL_EXPIRATION_MINUTES` (if missing the default value is 60 minutes) +- Optional: URL expiration time `S3_URL_EXPIRATION_MINUTES` (if missing the default value is 60 minutes) ## Custom callbacks You can write your own remote procedures. A good place to start is looking at how S3 URL presigning is implemented in the `examples/form_composer_demo` example project. ------ +--- + + +# Insertions + +FormComposer allows for insertion of code into its config in these scenarios: +- Specify lengthy content of an attribute (e.g. "instruction") in a separate HTML file +- Define custom validators for form fileds in a JS file + +The inserted code must reside in separate files (called "insertion files") located in `insertions` subdirectory of your form config directory. +- _Remember that you can change default config directory path using `--directory` option of `form_composer_config` command_ + +--- + +## HTML content insertion + +An HTML insertion file is specified as a file path that's relative to the form config. It can be inserted directly into `form_config.json` config, or via a token. + +#### Insertion without token + +Simply set entire value of an attribute to the insertion file's path. This is equivalent to setting value of that attribute to content of the HTML file (except now you don't have to stitch all HTML content into a single unreadable JSON line). + +Attributes that support HTML insertions are the same ones that support tokens + +Example in `form_config.json`: +```json +{ + ... + "instruction": "insertions/some_content.html" + ... +} +``` + +#### Insertion via token + +Use an extrapolated token as usual, and set that token's value to the insertion file's path. Upon extrapolation, value of such token will be automatically replaced with content of the HTML file. + +Example in `token_sets_values_config.json`: +```json +[ + { + "tokens_values": { + "html_file": "insertions/some_content.html" + } + } +] +``` + +## JS validator insertion + +You can define your own custom field validators as Javascript functions, and place them in a special file `insertions/custom_validators.js` inside your form config directory. When a Task is rendered in the browser, your functions will be imported from this file. + +Each validator function must have the following signature: +- Accept 2 required arguments `field` and `value`, and any number of optional arguments + - `field` is a JS object representing a rendered form field + - `value` is provided value of the field (format depends on the field type) + - optional arguments are the parameters you specified in the form config + - This can be a single value (Boolean, String, Number) + - This can also be an Array of values + - _In this case, note that Array-type arguments will be passed as separate positional arguments after the `value` argument. If you need to use them as an array in your code, combine them like so: `fn(field, value, ...optionalArgs)`._ +- Return value must be either: + - `null` if validation passed successfully + - String if validation failed + - This value will be shown to user as an error message underneath the field, and in the error summary block + +Example in `custom_validators.js`... + +```js +// You can import some functions from another file +import { someHelper } from "./helpers.js"; + +export function fieldContainsWord(field, value, word) { + someHelper(); + + if (value.includes(word)) { + return null; + } + + return `Field ${field.name} must contain a word "${word}".`; +} + +// This way you can separate all your validators into separate files, for convenience +export { phoneValidatorFunction } from "./phone_validator_code.js"; +``` +...and its usage in `form_config.json`: +```json +{ + ... + "validators": { + "required": true, + ... + "fieldContainsWord": "Mephisto" + }, + ... +} +``` + +----- # Config file reference @@ -387,7 +499,7 @@ Task data config file `task_data.json` specifies layout of all form versions tha // Two fieldsets { "title": "Personal information", - "instruction": "", + "instruction": "insertions/personal_info_instruction.html", "rows": [ // Two rows { @@ -541,8 +653,8 @@ Here's example of a single field config: "required": true, "minLength": 2, "maxLength": 20, - "regexp": ["^[a-zA-Z0-9._-]+@mephisto\\.ai$", "ig"] - // or can use this --> "regexp": "^[a-zA-Z0-9._-]+@mephisto\\.ai$" + "regexp": ["^[a-z\.\-']+$", "ig"] + // or can use this --> "regexp": "^[a-z\.\-']+$" }, "value": "" } @@ -570,6 +682,11 @@ The most important attributes are: `label`, `name`, `type`, `validators` - `value` - Initial value of the field (String, Optional) +######## Attributes - file field + +- `show_preview` - Show preview of selected file before the form is submitted (Boolean, Optional) + + ######## Attributes - select field - `multiple` - Support selection of multiple provided options, not just one (Boolean. Default: false) @@ -603,16 +720,16 @@ Example: [ { "tokens_values": { - "actor": "Carrie Fisher", - "movie_name": "Star Wars", - "genre": "Sci-Fi" + "model": "Volkswagen", + "make": "Beetle", + "review_criteria": "insertions/review_criteria.html" } }, { "tokens_values": { - "actor": "Keanu Reeves", - "movie_name": "The Matrix", - "genre": "Sci-Fi" + "model": "Nissan", + "make": "Murano", + "review_criteria": "insertions/review_criteria.html" } } ] @@ -627,8 +744,8 @@ Example: ```json { "actor": ["Carrie Fisher", "Keanu Reeves"], - "movie_name": ["Star Wars", "The Matrix"], - "genre": ["Sci-Fi"] + "genre": ["Sci-Fi"], + "movie_name": ["Star Wars", "The Matrix"] } ``` diff --git a/mephisto/generators/form_composer/config_validation/common_validation.py b/mephisto/generators/form_composer/config_validation/common_validation.py index 251c84654..a08452e19 100644 --- a/mephisto/generators/form_composer/config_validation/common_validation.py +++ b/mephisto/generators/form_composer/config_validation/common_validation.py @@ -3,10 +3,14 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +import os from typing import List +from typing import Optional +from .config_validation_constants import ATTRS_SUPPORTING_TOKENS from .config_validation_constants import AvailableAttrsType from .config_validation_constants import PY_JSON_TYPES_MAPPING +from .utils import is_insertion_file def validate_config_dict_item( @@ -14,6 +18,7 @@ def validate_config_dict_item( item_log_name: str, available_attrs: AvailableAttrsType, errors: List[str], + data_path: Optional[str] = None, ) -> bool: is_valid = True @@ -57,4 +62,39 @@ def validate_config_dict_item( f"must be `{PY_JSON_TYPES_MAPPING[attr_type]}`." ) + if data_path: + for attr_name in ATTRS_SUPPORTING_TOKENS: + item_attr = item.get(attr_name) + if not item_attr: + continue + + if is_insertion_file(item_attr): + file_path = os.path.abspath(os.path.join(data_path, item_attr)) + if not os.path.exists(file_path): + is_valid = False + errors.append( + f"Could not find insertion file '{file_path}'. " + f"Either create the file, or update the config." + ) + return is_valid + + +def replace_path_to_file_with_its_content( + value: str, + data_path: str, +) -> str: + """ + Attributes may contain tokens whose value is relative HTML file paths. + We replace such token values with content from the indicated file. + """ + if is_insertion_file(value): + file_path = os.path.abspath(os.path.join(data_path, value)) + if not os.path.exists(file_path): + raise FileNotFoundError(f"Could not open insertion file '{file_path}'") + + with open(file_path) as html_file: + file_content_value = html_file.read() + return file_content_value + + return value diff --git a/mephisto/generators/form_composer/config_validation/config_validation_constants.py b/mephisto/generators/form_composer/config_validation/config_validation_constants.py index 6d01fc28b..e01eec8d3 100644 --- a/mephisto/generators/form_composer/config_validation/config_validation_constants.py +++ b/mephisto/generators/form_composer/config_validation/config_validation_constants.py @@ -10,6 +10,8 @@ TOKENS_VALUES_KEY = "tokens_values" FILE_URL_TOKEN_KEY = "file_location" +INSERTIONS_PATH_NAME = "insertions" +CUSTOM_VALIDATORS_JS_FILE_NAME = "custom_validators.js" AVAILABLE_CONFIG_ATTRS: AvailableAttrsType = { "form": { @@ -134,6 +136,10 @@ "type": str, "required": False, }, + "show_preview": { + "type": bool, + "required": False, + }, "tooltip": { "type": str, "required": False, diff --git a/mephisto/generators/form_composer/config_validation/form_config.py b/mephisto/generators/form_composer/config_validation/form_config.py index 63dd5ae23..2630d4cb8 100644 --- a/mephisto/generators/form_composer/config_validation/form_config.py +++ b/mephisto/generators/form_composer/config_validation/form_config.py @@ -5,6 +5,7 @@ from typing import Dict from typing import List +from typing import Optional from typing import Tuple from .common_validation import validate_config_dict_item @@ -56,7 +57,10 @@ def _duplicate_values_exist(unique_names: UniqueAttrsType, errors: List[str]) -> return is_valid -def validate_form_config(config_data: dict) -> Tuple[bool, List[str]]: +def validate_form_config( + config_data: dict, + data_path: Optional[str] = None, +) -> Tuple[bool, List[str]]: is_valid = True errors = [] @@ -126,7 +130,7 @@ def validate_form_config(config_data: dict) -> Tuple[bool, List[str]]: # Run structure validation for item in items_to_validate: - config_is_valid = validate_config_dict_item(*item, errors=errors) + config_is_valid = validate_config_dict_item(*item, errors=errors, data_path=data_path) if not config_is_valid: is_valid = False diff --git a/mephisto/generators/form_composer/config_validation/task_data_config.py b/mephisto/generators/form_composer/config_validation/task_data_config.py index 82e558096..b0b58c1f9 100644 --- a/mephisto/generators/form_composer/config_validation/task_data_config.py +++ b/mephisto/generators/form_composer/config_validation/task_data_config.py @@ -16,6 +16,7 @@ from mephisto.generators.form_composer.constants import TOKEN_END_REGEX from mephisto.generators.form_composer.constants import TOKEN_START_REGEX from mephisto.generators.form_composer.remote_procedures import ProcedureName +from .common_validation import replace_path_to_file_with_its_content from .config_validation_constants import ATTRS_SUPPORTING_TOKENS from .config_validation_constants import TOKENS_VALUES_KEY from .form_config import validate_form_config @@ -29,14 +30,23 @@ FILE_LOCATION_TOKEN_NAME = "file_location" -def _extrapolate_tokens_values(text: str, tokens_values: dict) -> str: +def _extrapolate_tokens_values( + text: str, + tokens_values: dict, + data_path: Optional[str] = None, +) -> str: for token, value in tokens_values.items(): + # For HTML paths + value = replace_path_to_file_with_its_content(value, data_path) + + # For other values text = re.sub( ( TOKEN_START_REGEX + r"(\s*)" + - # Escape and add parentheses around the token, in case it has special characters + # Escape and add regexp grouping parentheses around the token + # (in case it has special characters) r"(" + re.escape(token) + r")" @@ -49,13 +59,17 @@ def _extrapolate_tokens_values(text: str, tokens_values: dict) -> str: return text -def _set_tokens_in_form_config_item(item: dict, tokens_values: dict): +def _set_tokens_in_form_config_item( + item: dict, + tokens_values: dict, + data_path: Optional[str] = None, +): for attr_name in ATTRS_SUPPORTING_TOKENS: item_attr = item.get(attr_name) if not item_attr: continue - item[attr_name] = _extrapolate_tokens_values(item_attr, tokens_values) + item[attr_name] = _extrapolate_tokens_values(item_attr, tokens_values, data_path) def _collect_form_config_items_to_extrapolate(config_data: dict) -> List[dict]: @@ -133,10 +147,14 @@ def _collect_tokens_from_form_config( return tokens_in_form_config, tokens_in_unexpected_attrs_errors -def _extrapolate_tokens_in_form_config(config_data: dict, tokens_values: dict) -> dict: +def _extrapolate_tokens_in_form_config( + config_data: dict, + tokens_values: dict, + data_path: Optional[str] = None, +) -> dict: items_to_extrapolate = _collect_form_config_items_to_extrapolate(config_data) for item in items_to_extrapolate: - _set_tokens_in_form_config_item(item, tokens_values) + _set_tokens_in_form_config_item(item, tokens_values, data_path) return config_data @@ -164,6 +182,7 @@ def _validate_tokens_in_both_configs( def _combine_extrapolated_form_configs( form_config_data: dict, token_sets_values_config_data: List[dict], + data_path: Optional[str] = None, ) -> List[dict]: errors = [] @@ -231,6 +250,7 @@ def _combine_extrapolated_form_configs( form_config_data_with_tokens = _extrapolate_tokens_in_form_config( deepcopy(form_config_data), token_sets_values[TOKENS_VALUES_KEY], + data_path=data_path, ) combined_config.append(form_config_data_with_tokens) else: @@ -241,31 +261,58 @@ def _combine_extrapolated_form_configs( return combined_config +def _replace_html_paths_with_html_file_content( + config_data: dict, + resources_html_path: str, +) -> dict: + items_to_replace = _collect_form_config_items_to_extrapolate(config_data) + + for item in items_to_replace: + for attr_name in ATTRS_SUPPORTING_TOKENS: + item_attr = item.get(attr_name) + if not item_attr: + continue + + item[attr_name] = replace_path_to_file_with_its_content(item_attr, resources_html_path) + + return config_data + + def create_extrapolated_config( form_config_path: str, token_sets_values_config_path: str, task_data_config_path: str, + data_path: Optional[str] = None, ): - # Check if files exist + # Ensure form config file exists if not os.path.exists(form_config_path): - raise FileNotFoundError(f"Create file '{form_config_path}' and add form configuration") + raise FileNotFoundError(f"Create file '{form_config_path}' with form configuration.") # Read JSON from files form_config_data = read_config_file(form_config_path) + # Handle HTML insertion files (replace their paths with file content) + if data_path: + form_config_data = _replace_html_paths_with_html_file_content( + form_config_data, + data_path, + ) + + # Get token sets values if os.path.exists(token_sets_values_config_path): token_sets_values_data = read_config_file(token_sets_values_config_path) else: token_sets_values_data = [] - # Create combined config + # Create Task data config (with multiple form versions) try: extrapolated_form_config_data = _combine_extrapolated_form_configs( form_config_data, token_sets_values_data, + data_path, ) write_config_to_file(extrapolated_form_config_data, task_data_config_path) - except ValueError as e: + except (ValueError, FileNotFoundError) as e: print(f"\n[red]Could not extrapolate form configs:[/red] {e}\n") exit() @@ -299,6 +346,7 @@ def verify_form_composer_configs( token_sets_values_config_path: Optional[str] = None, separate_token_values_config_path: Optional[str] = None, task_data_config_only: bool = False, + data_path: Optional[str] = None, ): errors = [] @@ -332,7 +380,10 @@ def verify_form_composer_configs( if form_config_data is None: pass else: - form_config_is_valid, form_config_errors = validate_form_config(form_config_data) + form_config_is_valid, form_config_errors = validate_form_config( + form_config_data, + data_path, + ) if not form_config_is_valid: errors.append(make_error_message("Form config is invalid", form_config_errors)) diff --git a/mephisto/generators/form_composer/config_validation/utils.py b/mephisto/generators/form_composer/config_validation/utils.py index 12fd7887c..07c070ecf 100644 --- a/mephisto/generators/form_composer/config_validation/utils.py +++ b/mephisto/generators/form_composer/config_validation/utils.py @@ -19,6 +19,12 @@ from botocore.exceptions import NoCredentialsError from rich import print +from mephisto.generators.form_composer.config_validation.config_validation_constants import ( + CUSTOM_VALIDATORS_JS_FILE_NAME, +) +from mephisto.generators.form_composer.config_validation.config_validation_constants import ( + INSERTIONS_PATH_NAME, +) from mephisto.generators.form_composer.constants import CONTENTTYPE_BY_EXTENSION from mephisto.generators.form_composer.constants import JSON_IDENTATION from mephisto.generators.form_composer.constants import S3_URL_EXPIRATION_MINUTES @@ -69,6 +75,26 @@ def get_file_ext(file_name: str) -> str: return Path(file_name).suffix.lower()[1:] +def is_insertion_file(path: str, ext: str = "html") -> bool: + if not isinstance(path, str): + return False + + if f"{INSERTIONS_PATH_NAME}/" in path and path.endswith(f".{ext}"): + return True + + return False + + +def set_custom_validators_js_env_var(data_path: str): + custom_validators_js_file_path = os.path.abspath( + os.path.join(data_path, INSERTIONS_PATH_NAME, CUSTOM_VALIDATORS_JS_FILE_NAME) + ) + custom_validators_js_file_exists = os.path.exists(custom_validators_js_file_path) + os.environ["WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS"] = ( + custom_validators_js_file_path if custom_validators_js_file_exists else "" + ) + + # ----- S3 ----- diff --git a/mephisto/generators/form_composer/data/insertions/custom_validators.js b/mephisto/generators/form_composer/data/insertions/custom_validators.js new file mode 100644 index 000000000..e69de29bb diff --git a/mephisto/generators/form_composer/webapp/src/components/core_components.jsx b/mephisto/generators/form_composer/webapp/src/components/core_components.jsx index 7397cfbe5..7cc69d155 100644 --- a/mephisto/generators/form_composer/webapp/src/components/core_components.jsx +++ b/mephisto/generators/form_composer/webapp/src/components/core_components.jsx @@ -10,6 +10,7 @@ import { prepareFormData, prepareRemoteProcedures, } from "react-form-composer"; +import * as customValidators from "custom-validators"; function LoadingScreen() { return Loading...; @@ -103,6 +104,7 @@ function FormComposerBaseFrontend({ onSubmit={onSubmit} finalResults={finalResults} setRenderingErrors={setFormComposerRenderingErrors} + customValidators={customValidators} /> )} diff --git a/mephisto/generators/form_composer/webapp/webpack.config.js b/mephisto/generators/form_composer/webapp/webpack.config.js index 40b8859da..9429876d3 100644 --- a/mephisto/generators/form_composer/webapp/webpack.config.js +++ b/mephisto/generators/form_composer/webapp/webpack.config.js @@ -25,6 +25,9 @@ module.exports = { __dirname, "../../../../packages/react-form-composer" ), + "custom-validators": path.resolve( + process.env.WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS + ), }, fallback: { net: false, diff --git a/mephisto/generators/form_composer/webapp/webpack.config.review.js b/mephisto/generators/form_composer/webapp/webpack.config.review.js index a60613f58..3095fc838 100644 --- a/mephisto/generators/form_composer/webapp/webpack.config.review.js +++ b/mephisto/generators/form_composer/webapp/webpack.config.review.js @@ -25,6 +25,9 @@ module.exports = { __dirname, "../../../../packages/react-form-composer" ), + "custom-validators": path.resolve( + process.env.WEBAPP__FORM_COMPOSER__CUSTOM_VALIDATORS + ), }, fallback: { net: false, diff --git a/mephisto/review_app/client/src/pages/TaskPage/InReviewFileModal/InReviewFileModal.tsx b/mephisto/review_app/client/src/pages/TaskPage/InReviewFileModal/InReviewFileModal.tsx index be7257546..d1f07f04b 100644 --- a/mephisto/review_app/client/src/pages/TaskPage/InReviewFileModal/InReviewFileModal.tsx +++ b/mephisto/review_app/client/src/pages/TaskPage/InReviewFileModal/InReviewFileModal.tsx @@ -85,14 +85,10 @@ function InReviewFileModal(props: InReviewFileModalProps) { {fileType ? ( <> {fileType === FileType.IMAGE && ( - {`image + {`image )} {fileType === FileType.VIDEO && ( -