From 8565fcd19bb5dde439cb4315dd8110486e236df6 Mon Sep 17 00:00:00 2001 From: Paul Abumov Date: Fri, 5 Jul 2024 14:48:33 -0400 Subject: [PATCH] Update Grafana deployment, dockerize properly, update docs --- Dockerfile | 2 +- docker/docker-compose.dev.vscode.yml | 7 +- docker/docker-compose.dev.yml | 5 +- docker/envs/env.dev | 7 + .../explanations/architecture_overview.md | 7 +- ...k_run_sequence_diagram__source_file.drawio | 229 +++++++++ ...s_dashboarding.md => metrics_dashboard.md} | 47 +- ...eview_sequence_diagram__source_file.drawio | 186 +++++++ .../form_composer_demo/run_task_dynamic.py | 8 +- .../run_task_dynamic_ec2_mturk_sandbox.py | 8 +- .../run_task_dynamic_ec2_prolific.py | 8 +- ...ask_dynamic_presigned_urls_ec2_prolific.py | 8 +- mephisto/client/README.md | 2 +- mephisto/client/cli.py | 468 +----------------- mephisto/client/cli_db_commands.py | 2 +- mephisto/client/cli_form_composer_commands.py | 212 ++++++++ mephisto/client/cli_metrics_commands.py | 100 ++++ mephisto/client/cli_review_app_commands.py | 103 ++++ mephisto/client/cli_scripts_commands.py | 136 +++++ .../{cli_commands.py => cli_wut_commands.py} | 54 +- .../scripts/form_composer/rebuild_all_apps.py | 4 +- .../gh_actions/auto_generate_architect.py | 2 +- .../gh_actions/auto_generate_blueprint.py | 2 +- .../gh_actions/auto_generate_provider.py | 2 +- .../gh_actions/auto_generate_requester.py | 2 +- .../local_db/gh_actions/auto_generate_task.py | 2 +- mephisto/scripts/metrics/install_metrics.sh | 66 ++- .../metrics/resources/grafana_defaults.ini | 8 +- mephisto/scripts/metrics/view_metrics.py | 38 +- mephisto/utils/metrics.py | 132 +++-- .../test_separate_token_values_config.py | 4 +- .../test_task_data_config.py | 10 +- .../test_token_sets_values_config.py | 6 +- 33 files changed, 1296 insertions(+), 581 deletions(-) create mode 100644 docs/web/docs/explanations/files/task_run_sequence_diagram__source_file.drawio rename docs/web/docs/guides/how_to_use/efficiency_organization/{metrics_dashboarding.md => metrics_dashboard.md} (87%) create mode 100644 docs/web/docs/guides/how_to_use/review_app/files/task_review_sequence_diagram__source_file.drawio create mode 100644 mephisto/client/cli_form_composer_commands.py create mode 100644 mephisto/client/cli_metrics_commands.py create mode 100644 mephisto/client/cli_review_app_commands.py create mode 100644 mephisto/client/cli_scripts_commands.py rename mephisto/client/{cli_commands.py => cli_wut_commands.py} (78%) diff --git a/Dockerfile b/Dockerfile index 5969bd1f7..95e36dcc8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,7 +9,7 @@ FROM $BASE_IMAGE # Firstly, remove `yarn` repo as it causes error that stops building a container. Error: # (Error: The repository 'https://dl.yarnpkg.com/debian stable InRelease' is not signed) RUN apt update -RUN apt install keychain -y +RUN apt install keychain curl -y COPY . /mephisto RUN mkdir ~/.mephisto diff --git a/docker/docker-compose.dev.vscode.yml b/docker/docker-compose.dev.vscode.yml index e09562771..f421ac7bf 100644 --- a/docker/docker-compose.dev.vscode.yml +++ b/docker/docker-compose.dev.vscode.yml @@ -11,9 +11,10 @@ services: context: .. dockerfile: Dockerfile ports: - - "8081:8000" - - "3001:3000" - - "5678:5678" + - "8081:8000" # TaskReview app + - "3001:3000" # Task app + - "3032:3032" # Grafana metrics + - "5678:5678" # VSCode Python debugger volumes: - ..:/mephisto - ./entrypoints/server.sh:/entrypoint.sh diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml index 301841e7c..5b2521dec 100644 --- a/docker/docker-compose.dev.yml +++ b/docker/docker-compose.dev.yml @@ -11,8 +11,9 @@ services: context: .. dockerfile: Dockerfile ports: - - "8081:8000" - - "3001:3000" + - "8081:8000" # TaskReview app + - "3001:3000" # Task app + - "3032:3032" # Grafana metrics volumes: - ..:/mephisto - ./entrypoints/server.prolific.sh:/entrypoint.sh diff --git a/docker/envs/env.dev b/docker/envs/env.dev index 931d2241b..06cc23333 100644 --- a/docker/envs/env.dev +++ b/docker/envs/env.dev @@ -12,4 +12,11 @@ AWS_SECRET_ACCESS_KEY=your_key AWS_DEFAULT_REGION=us-east-1 S3_URL_EXPIRATION_MINUTES=60 +# Cypress CYPRESS_CACHE_FOLDER=/tmp + +# Grafana +GRAFANA_HOST=localhost +GRAFANA_PORT=3032 +GRAFANA_USER=admin +GRAFANA_PASSWORD=admin diff --git a/docs/web/docs/explanations/architecture_overview.md b/docs/web/docs/explanations/architecture_overview.md index 82590e86c..42554ed1c 100644 --- a/docs/web/docs/explanations/architecture_overview.md +++ b/docs/web/docs/explanations/architecture_overview.md @@ -7,13 +7,14 @@ sidebar_position: 1 --- -# Architecture diagram +## Architecture diagram At a high level, Mephisto runs its data collection/annotation tasks as shown in this sequence diagram. -!`task_run_sequence_diagram.png`(./images/task_run_sequence_diagram.png) +![task_run_sequence_diagram.png](./images/task_run_sequence_diagram.png) -# Codebase structure + +## Codebase structure This is a quick guide over file directories in Mephisto project. Note that some directories include their own `README.md` file with further details. diff --git a/docs/web/docs/explanations/files/task_run_sequence_diagram__source_file.drawio b/docs/web/docs/explanations/files/task_run_sequence_diagram__source_file.drawio new file mode 100644 index 000000000..7e754f636 --- /dev/null +++ b/docs/web/docs/explanations/files/task_run_sequence_diagram__source_file.drawio @@ -0,0 +1,229 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboarding.md b/docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboard.md similarity index 87% rename from docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboarding.md rename to docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboard.md index 28d6ca540..180303e14 100644 --- a/docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboarding.md +++ b/docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboard.md @@ -20,12 +20,23 @@ Seem exciting? Let's dig in! ## Installation Installation is easy, just run the setup script: -``` +```shell mephisto metrics install ``` This script creates a local `prometheus` and `grafana` folder in mephisto's `metrics` directory, and populates them with default configurations for being able to view Mephisto run metrics. +> NOTE: You may see a request to install `curl`. +> In some systems it may not be installed. Do this manually. +> Steps may be different depending on your system. + +If you use Docker, you need to connect to its environment (applies to other metrics commands as well): +```shell +docker-compose -f docker/docker-compose.dev.yml up +docker exec -it mephisto_dc bash +mephisto metrics install +``` + ## Usage Once you have metrics installed, all future runs will be logging metrics by default. You can view these metrics with: @@ -101,3 +112,37 @@ For this section, each of the plots can show the health of your system in a numb ### Onboarding and Other Validation Metrics Similarly to the above dashboards, these panels provide details about onboarding-specific or other validation-specific processes. + +## Other commands + +### Cleanup + +If you need to shut down Prometheus and Grafana resources that may have persisted: +```shell +mephisto metrics cleanup +``` + +### Remove + +If you do not need metrics after testing anymore, you can remove all files to save space: +```shell +mephisto metrics remove +``` + +### Reinstall + +If you already had metrics installed, but they were updated in a newer Mephisto version, you can update them locally: +```shell +mephisto metrics reinstall +``` + +## Configuration + +If you wish to customize host, port, user or password for Grafana, you can set the following environment variables: + +- `GRAFANA_HOST` - Host of Grafana server (Default in Docker: `localhost`) +- `GRAFANA_PORT` - Port of Grafana server (Default in Docker: `3032`) +- `GRAFANA_USER` - Name of a user to login in web client (Default in Docker: `admin`) +- `GRAFANA_PASSWORD` - Password of a user to login in web client (Default in Docker: `admin`) + +> NOTE: If you created Mephisto environment on your own, setting these variables is required. diff --git a/docs/web/docs/guides/how_to_use/review_app/files/task_review_sequence_diagram__source_file.drawio b/docs/web/docs/guides/how_to_use/review_app/files/task_review_sequence_diagram__source_file.drawio new file mode 100644 index 000000000..0ce6f3195 --- /dev/null +++ b/docs/web/docs/guides/how_to_use/review_app/files/task_review_sequence_diagram__source_file.drawio @@ -0,0 +1,186 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/examples/form_composer_demo/run_task_dynamic.py b/examples/form_composer_demo/run_task_dynamic.py index cdbcbd3bd..06e8b0085 100644 --- a/examples/form_composer_demo/run_task_dynamic.py +++ b/examples/form_composer_demo/run_task_dynamic.py @@ -8,10 +8,10 @@ from omegaconf import DictConfig -from mephisto.client.cli import FORM_COMPOSER__DATA_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__DATA_DIR_NAME -from mephisto.client.cli import FORM_COMPOSER__FORM_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__FORM_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py b/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py index c131ce413..415905ee0 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py +++ b/examples/form_composer_demo/run_task_dynamic_ec2_mturk_sandbox.py @@ -14,10 +14,10 @@ from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import ( SharedStaticTaskState, ) -from mephisto.client.cli import FORM_COMPOSER__DATA_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__DATA_DIR_NAME -from mephisto.client.cli import FORM_COMPOSER__FORM_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__FORM_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) diff --git a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py index 9a92102d2..cc99edbe7 100644 --- a/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_ec2_prolific.py @@ -11,10 +11,10 @@ from mephisto.abstractions.blueprints.abstract.static_task.static_blueprint import ( SharedStaticTaskState, ) -from mephisto.client.cli import FORM_COMPOSER__DATA_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__DATA_DIR_NAME -from mephisto.client.cli import FORM_COMPOSER__FORM_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__FORM_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME from mephisto.data_model.qualification import QUAL_GREATER_EQUAL from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, diff --git a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py b/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py index b382e6cd8..6595c7ec9 100644 --- a/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py +++ b/examples/form_composer_demo/run_task_dynamic_presigned_urls_ec2_prolific.py @@ -11,10 +11,10 @@ from mephisto.abstractions.blueprints.remote_procedure.remote_procedure_blueprint import ( SharedRemoteProcedureTaskState, ) -from mephisto.client.cli import FORM_COMPOSER__DATA_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__DATA_DIR_NAME -from mephisto.client.cli import FORM_COMPOSER__FORM_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__FORM_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME from mephisto.generators.form_composer.config_validation.task_data_config import ( create_extrapolated_config, ) diff --git a/mephisto/client/README.md b/mephisto/client/README.md index 645290305..c81fbd101 100644 --- a/mephisto/client/README.md +++ b/mephisto/client/README.md @@ -59,7 +59,7 @@ Run a script from `mephisto/scripts` directory. ### mephisto metrics Extension to view task health metrics via dashboard using [Prometheus](https://prometheus.io/) and [Grafana](https://grafana.com/oss/grafana/). -For more details you can consult [Docs for Metrics](docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboarding.md). +For more details you can consult [Docs for Metrics](docs/web/docs/guides/how_to_use/efficiency_organization/metrics_dashboard.md). ### mephisto review_app diff --git a/mephisto/client/cli.py b/mephisto/client/cli.py index d806d7b6f..c3b8c109b 100644 --- a/mephisto/client/cli.py +++ b/mephisto/client/cli.py @@ -4,64 +4,23 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -import os -import subprocess -from typing import Any -from typing import List -from typing import Optional - import rich_click as click # type: ignore -from flask.cli import pass_script_info -from flask.cli import ScriptInfo from rich.markdown import Markdown from rich_click import RichCommand from rich_click import RichGroup -import mephisto.scripts.form_composer.rebuild_all_apps as rebuild_all_apps_form_composer -import mephisto.scripts.heroku.initialize_heroku as initialize_heroku -import mephisto.scripts.local_db.clear_worker_onboarding as clear_worker_onboarding_local_db -import mephisto.scripts.local_db.load_data_to_mephisto_db as load_data_local_db -import mephisto.scripts.local_db.remove_accepted_tip as remove_accepted_tip_local_db -import mephisto.scripts.local_db.review_feedback_for_task as review_feedback_local_db -import mephisto.scripts.local_db.review_tips_for_task as review_tips_local_db -import mephisto.scripts.metrics.shutdown_metrics as shutdown_metrics -import mephisto.scripts.metrics.view_metrics as view_metrics -import mephisto.scripts.mturk.cleanup as cleanup_mturk -import mephisto.scripts.mturk.identify_broken_units as identify_broken_units_mturk -import mephisto.scripts.mturk.launch_makeup_hits as launch_makeup_hits_mturk -import mephisto.scripts.mturk.print_outstanding_hit_status as soft_block_workers_by_mturk_id_mturk -from mephisto.client.cli_commands import get_wut_arguments from mephisto.client.cli_db_commands import db_cli -from mephisto.generators.form_composer.config_validation.separate_token_values_config import ( - update_separate_token_values_config_with_file_urls, -) -from mephisto.generators.form_composer.config_validation.task_data_config import ( - create_extrapolated_config, -) -from mephisto.generators.form_composer.config_validation.task_data_config import ( - verify_form_composer_configs, -) -from mephisto.generators.form_composer.config_validation.token_sets_values_config import ( - update_token_sets_values_config_with_premutated_data, -) -from mephisto.generators.form_composer.config_validation.utils import is_s3_url -from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var -from mephisto.generators.form_composer.config_validation.utils import ( - set_custom_validators_js_env_var, -) +from mephisto.client.cli_form_composer_commands import form_composer +from mephisto.client.cli_form_composer_commands import form_composer_config +from mephisto.client.cli_metrics_commands import metrics_cli +from mephisto.client.cli_review_app_commands import review_app +from mephisto.client.cli_scripts_commands import run_script +from mephisto.client.cli_wut_commands import run_wut from mephisto.operations.registry import get_valid_provider_types -from mephisto.scripts.local_db import auto_generate_all_docs_reference_md -from mephisto.tools.scripts import build_custom_bundle from mephisto.utils.console_writer import ConsoleWriter from mephisto.utils.rich import console from mephisto.utils.rich import create_table -FORM_COMPOSER__DATA_DIR_NAME = "data" -FORM_COMPOSER__DATA_CONFIG_NAME = "task_data.json" -FORM_COMPOSER__FORM_CONFIG_NAME = "form_config.json" -FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME = "token_sets_values_config.json" -FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME = "separate_token_values_config.json" - logger = ConsoleWriter() @@ -216,413 +175,14 @@ def register_provider(args): click.echo(str(e)) -@cli.command("wut", cls=RichCommand, context_settings={"ignore_unknown_options": True}) -@click.argument("args", nargs=-1) -def run_wut(args): - """Discover the configuration arguments for different abstractions""" - get_wut_arguments(args) - - -@cli.command("scripts", cls=RichCommand, context_settings={"ignore_unknown_options": True}) -@click.argument("script_type", required=False, nargs=1) -@click.argument("script_name", required=False, nargs=1) -@click.argument("args", nargs=-1) # Allow arguments for low level commands -def run_script(script_type, script_name, args: Optional[Any] = None): - """Run one of the many mephisto scripts.""" - - def print_non_markdown_list(items: List[str]): - res = "" - for item in items: - res += "\n * " + item - return res - - VALID_SCRIPT_TYPES = ["local_db", "heroku", "metrics", "mturk", "form_composer"] - if script_type is None or script_type.strip() not in VALID_SCRIPT_TYPES: - logger.info("") - raise click.UsageError( - "You must specify a valid script_type from below. \n\nValid script types are:" - + print_non_markdown_list(VALID_SCRIPT_TYPES) - ) - script_type = script_type.strip() - LOCAL_DB_VALID_SCRIPTS_NAMES = [ - "review_tips", - "remove_tip", - "review_feedback", - "load_data", - "clear_worker_onboarding", - "auto_generate_all_docs_reference_md", - ] - HEROKU_VALID_SCRIPTS_NAMES = ["initialize"] - METRICS_VALID_SCRIPTS_NAMES = ["view", "shutdown"] - MTURK_VALID_SCRIPTS_NAMES = [ - "cleanup", - "identify_broken_units", - "launch_makeup_hits", - "print_outstanding_hit_status", - "soft_block_workers_by_mturk_id", - ] - FORM_COMPOSER_VALID_SCRIPTS_NAMES = [ - "rebuild_all_apps", - ] - script_type_to_scripts_data = { - "local_db": { - "valid_script_names": LOCAL_DB_VALID_SCRIPTS_NAMES, - "scripts": { - LOCAL_DB_VALID_SCRIPTS_NAMES[0]: review_tips_local_db.main, - LOCAL_DB_VALID_SCRIPTS_NAMES[1]: remove_accepted_tip_local_db.main, - LOCAL_DB_VALID_SCRIPTS_NAMES[2]: review_feedback_local_db.main, - LOCAL_DB_VALID_SCRIPTS_NAMES[3]: load_data_local_db.main, - LOCAL_DB_VALID_SCRIPTS_NAMES[4]: clear_worker_onboarding_local_db.main, - LOCAL_DB_VALID_SCRIPTS_NAMES[5]: auto_generate_all_docs_reference_md.main, - }, - }, - "heroku": { - "valid_script_names": HEROKU_VALID_SCRIPTS_NAMES, - "scripts": {HEROKU_VALID_SCRIPTS_NAMES[0]: initialize_heroku.main}, - }, - "metrics": { - "valid_script_names": METRICS_VALID_SCRIPTS_NAMES, - "scripts": { - METRICS_VALID_SCRIPTS_NAMES[0]: view_metrics.launch_servers, - METRICS_VALID_SCRIPTS_NAMES[1]: shutdown_metrics.shutdown_servers, - }, - }, - "mturk": { - "valid_script_names": MTURK_VALID_SCRIPTS_NAMES, - "scripts": { - MTURK_VALID_SCRIPTS_NAMES[0]: cleanup_mturk.main, - MTURK_VALID_SCRIPTS_NAMES[1]: identify_broken_units_mturk.main, - MTURK_VALID_SCRIPTS_NAMES[2]: launch_makeup_hits_mturk.main, - MTURK_VALID_SCRIPTS_NAMES[3]: rebuild_all_apps_form_composer.main, - MTURK_VALID_SCRIPTS_NAMES[4]: soft_block_workers_by_mturk_id_mturk.main, - }, - }, - "form_composer": { - "valid_script_names": FORM_COMPOSER_VALID_SCRIPTS_NAMES, - "scripts": { - FORM_COMPOSER_VALID_SCRIPTS_NAMES[0]: rebuild_all_apps_form_composer.main, - }, - }, - } - - if script_name is None or ( - script_name not in script_type_to_scripts_data[script_type]["valid_script_names"] - ): - logger.info("") - raise click.UsageError( - "You must specify a valid script_name from below. \n\nValid script names are:" - + print_non_markdown_list( - script_type_to_scripts_data[script_type]["valid_script_names"] - ) - ) - # runs the script - script_type_to_scripts_data[script_type]["scripts"][script_name]() - - -@cli.command("metrics", cls=RichCommand, context_settings={"ignore_unknown_options": True}) -@click.argument("args", nargs=-1) -def metrics_cli(args): - from mephisto.utils.metrics import ( - launch_servers_and_wait, - metrics_are_installed, - run_install_script, - METRICS_DIR, - shutdown_prometheus_server, - shutdown_grafana_server, - ) - - if len(args) == 0 or args[0] not in ["install", "view", "cleanup"]: - logger.error("\n[red]Usage: mephisto metrics [/red]") - metrics_table = create_table(["Property", "Value"], "Metrics Arguments") - metrics_table.add_row("install", f"Installs Prometheus and Grafana to {METRICS_DIR}") - metrics_table.add_row( - "view", - "Launches a Prometheus and Grafana server, and shuts down on exit", - ) - metrics_table.add_row( - "cleanup", - "Shuts down Prometheus and Grafana resources that may have persisted", - ) - console.print(metrics_table) - return - command = args[0] - if command == "install": - if metrics_are_installed(): - click.echo(f"Metrics are already installed! See {METRICS_DIR}") - return - run_install_script() - elif command == "view": - if not metrics_are_installed(): - click.echo(f"Metrics aren't installed! Use `mephisto metrics install` first.") - return - click.echo(f"Servers launching - use ctrl-C to shutdown") - launch_servers_and_wait() - else: # command == 'cleanup': - if not metrics_are_installed(): - click.echo(f"Metrics aren't installed! Use `mephisto metrics install` first.") - return - click.echo(f"Cleaning up existing servers if they exist") - shutdown_prometheus_server() - shutdown_grafana_server() - - -@cli.command("review_app", cls=RichCommand) -@click.option("-h", "--host", type=str, default="127.0.0.1") -@click.option("-p", "--port", type=int, default=5000) -@click.option("-d", "--debug", type=bool, default=False, is_flag=True) -@click.option("-f", "--force-rebuild", type=bool, default=False, is_flag=True) -@click.option("-s", "--skip-build", type=bool, default=False, is_flag=True) -@pass_script_info -def review_app( - info: ScriptInfo, - host: Optional[str], - port: Optional[int], - debug: bool = False, - force_rebuild: bool = False, - skip_build: bool = False, -): - """ - Launch a local review server. - Custom implementation of `flask run ` command (`flask.cli.run_command`) - """ - from flask.cli import show_server_banner - from flask.helpers import get_debug_flag - from mephisto.review_app.server import create_app - from werkzeug.serving import run_simple - - # Set env variables for Review App - app_url = f"http://{host}:{port}" - os.environ["HOST"] = host - os.environ["PORT"] = str(port) - - logger.info(f'[green]Review APP will start on "{app_url}" address.[/green]') - - # Set up Review App Client - if not skip_build: - review_app_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "review_app", - ) - client_dir = "client" - client_path = os.path.join(review_app_path, client_dir) - - # Install JS requirements - if os.path.exists(os.path.join(client_path, "node_modules")): - logger.info(f"[blue]JS requirements are already installed.[/blue]") - else: - logger.info(f"[blue]Installing JS requirements started.[/blue]") - app_started = subprocess.call(["npm", "install"], cwd=client_path) - if app_started != 0: - raise Exception( - "Please make sure npm is installed, " - "otherwise view the above error for more info." - ) - logger.info(f"[blue]Installing JS requirements finished.[/blue]") - - if os.path.exists(os.path.join(client_path, "build", "index.html")) and not force_rebuild: - logger.info(f"[blue]React bundle is already built.[/blue]") - else: - logger.info(f"[blue]Building React bundle started.[/blue]") - build_custom_bundle( - review_app_path, - force_rebuild=force_rebuild, - webapp_name=client_dir, - build_command="build", - ) - logger.info(f"[blue]Building React bundle finished.[/blue]") - - # Set debug - debug = debug if debug is not None else get_debug_flag() - reload = debug - debugger = debug - - # Show Flask banner - show_server_banner(debug, info.app_import_path) - - # Init Flask App - app = create_app(debug=debug) - - # Run Flask server - run_simple( - host, - port, - app, - use_reloader=reload, - use_debugger=debugger, - ) - - -def _get_form_composer_app_path() -> str: - app_path = os.path.join( - os.path.dirname(os.path.dirname(os.path.abspath(__file__))), - "generators", - "form_composer", - ) - return app_path - - -@cli.command("form_composer", cls=RichCommand) -@click.option("-o", "--task-data-config-only", type=bool, default=True, is_flag=True) -def form_composer(task_data_config_only: bool = True): - # Get app path to run Python script from there (instead of the current file's directory). - # This is necessary, because the whole infrastructure is built relative to the location - # of the called command-line script. - # The other parts of the logic are inside `form_composer/run***.py` script - app_path = _get_form_composer_app_path() - app_data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME) - - task_data_config_path = os.path.join(app_data_path, FORM_COMPOSER__DATA_CONFIG_NAME) - - # Change dir to app dir - os.chdir(app_path) - - # Set env var for `custom_validators.js` - set_custom_validators_js_env_var(app_data_path) - # Set env var for `custom_triggers.js` - set_custom_triggers_js_env_var(app_data_path) - - verify_form_composer_configs( - task_data_config_path=task_data_config_path, - task_data_config_only=task_data_config_only, - ) - - # Start the process - process = subprocess.Popen("python ./run.py", shell=True, cwd=app_path) - - # Kill subprocess when we interrupt the main process - try: - process.wait() - except (KeyboardInterrupt, Exception): - try: - process.terminate() - except OSError: - pass - process.wait() - - -@cli.command("form_composer_config", cls=RichCommand) -@click.option("-v", "--verify", type=bool, default=False, is_flag=True) -@click.option("-f", "--update-file-location-values", type=str, default=None) -@click.option("-e", "--extrapolate-token-sets", type=bool, default=False, is_flag=True) -@click.option("-p", "--permutate-separate-tokens", type=bool, default=False, is_flag=True) -@click.option("-d", "--directory", type=str, default=None) -@click.option("-u", "--use-presigned-urls", type=bool, default=False, is_flag=True) -def form_composer_config( - verify: Optional[bool] = False, - update_file_location_values: Optional[str] = None, - extrapolate_token_sets: Optional[bool] = False, - permutate_separate_tokens: Optional[bool] = False, - directory: Optional[str] = None, - use_presigned_urls: Optional[bool] = False, -): - """ - Prepare (parts of) config for the `form_composer` command. - Note that each parameter is essentially a separate command, and they cannot be mixed. - - :param verify: Validate all JSON configs currently present in the form builder config directory - :param update_file_location_values: Update existing separate-token values config - with file URLs automatically taken from a location (e.g. an S3 folder) - :param extrapolate_token_sets: Generate form versions based on extrapolated values of token sets - :param permutate_separate_tokens: Create tokens sets as all possible permutations of - values lists defined in separate-token values config - :param directory: Path to the directory where form and token configs are located. - By default, it's the `data` directory of `form_composer` generator - :param use_presigned_urls: a modifier for `--update_file_location_values` parameter. - Wraps every S3 URL with a standard handler that presigns these URLs during form rendering - when we use `--update_file_location_values` command - """ - - # Substitute defaults for missing param values - if directory: - app_data_path = directory - else: - app_path = _get_form_composer_app_path() - app_data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME) - logger.info(f"[blue]Using config directory: {app_data_path}[/blue]") - - # Validate param values - if not os.path.exists(app_data_path): - logger.error(f"[red]Directory '{app_data_path}' does not exist[/red]") - return None - - if use_presigned_urls and not update_file_location_values: - logger.error( - f"[red]Parameter `--use-presigned-urls` can be used " - f"only with `--update-file-location-values` option[/red]" - ) - return None - - # Check files and create `data.json` config with tokens data before running a task - full_path = lambda data_file: os.path.join(app_data_path, data_file) - task_data_config_path = full_path(FORM_COMPOSER__DATA_CONFIG_NAME) - form_config_path = full_path(FORM_COMPOSER__FORM_CONFIG_NAME) - token_sets_values_config_path = full_path(FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME) - separate_token_values_config_path = full_path(FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME) - - # Run the command - if verify: - logger.info(f"Started configs verification") - verify_form_composer_configs( - task_data_config_path=task_data_config_path, - form_config_path=form_config_path, - token_sets_values_config_path=token_sets_values_config_path, - separate_token_values_config_path=separate_token_values_config_path, - task_data_config_only=False, - data_path=app_data_path, - ) - logger.info(f"Finished configs verification") - - elif update_file_location_values: - logger.info( - f"[green]Started updating '{FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME}' " - f"with file URLs from '{update_file_location_values}'[/green]" - ) - if is_s3_url(update_file_location_values): - update_separate_token_values_config_with_file_urls( - url=update_file_location_values, - separate_token_values_config_path=separate_token_values_config_path, - use_presigned_urls=use_presigned_urls, - ) - logger.info(f"[green]Finished successfully[/green]") - else: - logger.info("`--update-file-location-values` must be a valid S3 URL") - - elif permutate_separate_tokens: - logger.info( - f"[green]Started updating '{FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME}' " - f"with permutated separate-token values[/green]" - ) - update_token_sets_values_config_with_premutated_data( - separate_token_values_config_path=separate_token_values_config_path, - token_sets_values_config_path=token_sets_values_config_path, - ) - logger.info(f"[green]Finished successfully[/green]") - - elif extrapolate_token_sets: - logger.info( - f"[green]Started extrapolating token sets values " - f"from '{FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME}' [/green]" - ) - create_extrapolated_config( - form_config_path=form_config_path, - token_sets_values_config_path=token_sets_values_config_path, - task_data_config_path=task_data_config_path, - data_path=app_data_path, - ) - logger.info(f"[green]Finished successfully[/green]") - - else: - logger.error( - f"[red]" - f"This command must have one of following parameters:" - f"\n-v/--verify" - f"\n-f/--update-file-location-value" - f"\n-e/--extrapolate-token-set" - f"\n-p/--permutate-separate-tokens" - f"[/red]" - ) - - +cli.command("scripts", cls=RichCommand, context_settings={"ignore_unknown_options": True})( + run_script +) +cli.command("wut", cls=RichCommand, context_settings={"ignore_unknown_options": True})(run_wut) +cli.command("review_app", cls=RichCommand)(review_app) +cli.command("form_composer", cls=RichCommand)(form_composer) +cli.command("form_composer_config", cls=RichCommand)(form_composer_config) +cli.add_command(metrics_cli) cli.add_command(db_cli) diff --git a/mephisto/client/cli_db_commands.py b/mephisto/client/cli_db_commands.py index bb8c6c549..0db1f5ece 100644 --- a/mephisto/client/cli_db_commands.py +++ b/mephisto/client/cli_db_commands.py @@ -38,7 +38,7 @@ def _print_used_options_for_running_command_message(ctx: click.Context, options: @click.group(name="db", context_settings=dict(help_option_names=["-h", "--help"])) def db_cli(): - """Operations with Mephisto DB and provider-specific datastores.""" + """Operations with Mephisto DB and provider-specific datastores""" pass diff --git a/mephisto/client/cli_form_composer_commands.py b/mephisto/client/cli_form_composer_commands.py new file mode 100644 index 000000000..4d8d771dd --- /dev/null +++ b/mephisto/client/cli_form_composer_commands.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os +import subprocess +from typing import Optional + +import click + +from mephisto.generators.form_composer.config_validation.separate_token_values_config import ( + update_separate_token_values_config_with_file_urls, +) +from mephisto.generators.form_composer.config_validation.task_data_config import ( + create_extrapolated_config, +) +from mephisto.generators.form_composer.config_validation.task_data_config import ( + verify_form_composer_configs, +) +from mephisto.generators.form_composer.config_validation.token_sets_values_config import ( + update_token_sets_values_config_with_premutated_data, +) +from mephisto.generators.form_composer.config_validation.utils import is_s3_url +from mephisto.generators.form_composer.config_validation.utils import set_custom_triggers_js_env_var +from mephisto.generators.form_composer.config_validation.utils import ( + set_custom_validators_js_env_var, +) +from mephisto.utils.console_writer import ConsoleWriter + +FORM_COMPOSER__DATA_DIR_NAME = "data" +FORM_COMPOSER__DATA_CONFIG_NAME = "task_data.json" +FORM_COMPOSER__FORM_CONFIG_NAME = "form_config.json" +FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME = "token_sets_values_config.json" +FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME = "separate_token_values_config.json" + +logger = ConsoleWriter() + + +def _get_form_composer_app_path() -> str: + app_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "generators", + "form_composer", + ) + return app_path + + +@click.option("-o", "--task-data-config-only", type=bool, default=True, is_flag=True) +def form_composer(task_data_config_only: bool = True): + """ + Generator of form-based Tasks with clean cross-platform Bootstrap forms + with client-side form validation. + """ + + # Get app path to run Python script from there (instead of the current file's directory). + # This is necessary, because the whole infrastructure is built relative to the location + # of the called command-line script. + # The other parts of the logic are inside `form_composer/run***.py` script + app_path = _get_form_composer_app_path() + app_data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME) + + task_data_config_path = os.path.join(app_data_path, FORM_COMPOSER__DATA_CONFIG_NAME) + + # Change dir to app dir + os.chdir(app_path) + + # Set env var for `custom_validators.js` + set_custom_validators_js_env_var(app_data_path) + # Set env var for `custom_triggers.js` + set_custom_triggers_js_env_var(app_data_path) + + verify_form_composer_configs( + task_data_config_path=task_data_config_path, + task_data_config_only=task_data_config_only, + ) + + # Start the process + process = subprocess.Popen("python ./run.py", shell=True, cwd=app_path) + + # Kill subprocess when we interrupt the main process + try: + process.wait() + except (KeyboardInterrupt, Exception): + try: + process.terminate() + except OSError: + pass + process.wait() + + +@click.option("-v", "--verify", type=bool, default=False, is_flag=True) +@click.option("-f", "--update-file-location-values", type=str, default=None) +@click.option("-e", "--extrapolate-token-sets", type=bool, default=False, is_flag=True) +@click.option("-p", "--permutate-separate-tokens", type=bool, default=False, is_flag=True) +@click.option("-d", "--directory", type=str, default=None) +@click.option("-u", "--use-presigned-urls", type=bool, default=False, is_flag=True) +def form_composer_config( + verify: Optional[bool] = False, + update_file_location_values: Optional[str] = None, + extrapolate_token_sets: Optional[bool] = False, + permutate_separate_tokens: Optional[bool] = False, + directory: Optional[str] = None, + use_presigned_urls: Optional[bool] = False, +): + """ + Prepare (parts of) config for the `form_composer` command. + Note that each parameter is essentially a separate command, and they cannot be mixed. + + :param verify: Validate all JSON configs currently present in the form builder config directory + :param update_file_location_values: Update existing separate-token values config + with file URLs automatically taken from a location (e.g. an S3 folder) + :param extrapolate_token_sets: Generate form versions based on extrapolated values of token sets + :param permutate_separate_tokens: Create tokens sets as all possible permutations of + values lists defined in separate-token values config + :param directory: Path to the directory where form and token configs are located. + By default, it's the `data` directory of `form_composer` generator + :param use_presigned_urls: a modifier for `--update_file_location_values` parameter. + Wraps every S3 URL with a standard handler that presigns these URLs during form rendering + when we use `--update_file_location_values` command + """ + + # Substitute defaults for missing param values + if directory: + app_data_path = directory + else: + app_path = _get_form_composer_app_path() + app_data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME) + logger.info(f"[blue]Using config directory: {app_data_path}[/blue]") + + # Validate param values + if not os.path.exists(app_data_path): + logger.error(f"[red]Directory '{app_data_path}' does not exist[/red]") + return None + + if use_presigned_urls and not update_file_location_values: + logger.error( + f"[red]Parameter `--use-presigned-urls` can be used " + f"only with `--update-file-location-values` option[/red]" + ) + return None + + # Check files and create `data.json` config with tokens data before running a task + full_path = lambda data_file: os.path.join(app_data_path, data_file) + task_data_config_path = full_path(FORM_COMPOSER__DATA_CONFIG_NAME) + form_config_path = full_path(FORM_COMPOSER__FORM_CONFIG_NAME) + token_sets_values_config_path = full_path(FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME) + separate_token_values_config_path = full_path(FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME) + + # Run the command + if verify: + logger.info(f"Started configs verification") + verify_form_composer_configs( + task_data_config_path=task_data_config_path, + form_config_path=form_config_path, + token_sets_values_config_path=token_sets_values_config_path, + separate_token_values_config_path=separate_token_values_config_path, + task_data_config_only=False, + data_path=app_data_path, + ) + logger.info(f"Finished configs verification") + + elif update_file_location_values: + logger.info( + f"[green]Started updating '{FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME}' " + f"with file URLs from '{update_file_location_values}'[/green]" + ) + if is_s3_url(update_file_location_values): + update_separate_token_values_config_with_file_urls( + url=update_file_location_values, + separate_token_values_config_path=separate_token_values_config_path, + use_presigned_urls=use_presigned_urls, + ) + logger.info(f"[green]Finished successfully[/green]") + else: + logger.info("`--update-file-location-values` must be a valid S3 URL") + + elif permutate_separate_tokens: + logger.info( + f"[green]Started updating '{FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME}' " + f"with permutated separate-token values[/green]" + ) + update_token_sets_values_config_with_premutated_data( + separate_token_values_config_path=separate_token_values_config_path, + token_sets_values_config_path=token_sets_values_config_path, + ) + logger.info(f"[green]Finished successfully[/green]") + + elif extrapolate_token_sets: + logger.info( + f"[green]Started extrapolating token sets values " + f"from '{FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME}' [/green]" + ) + create_extrapolated_config( + form_config_path=form_config_path, + token_sets_values_config_path=token_sets_values_config_path, + task_data_config_path=task_data_config_path, + data_path=app_data_path, + ) + logger.info(f"[green]Finished successfully[/green]") + + else: + logger.error( + f"[red]" + f"This command must have one of following parameters:" + f"\n-v/--verify" + f"\n-f/--update-file-location-value" + f"\n-e/--extrapolate-token-set" + f"\n-p/--permutate-separate-tokens" + f"[/red]" + ) diff --git a/mephisto/client/cli_metrics_commands.py b/mephisto/client/cli_metrics_commands.py new file mode 100644 index 000000000..5f48e757b --- /dev/null +++ b/mephisto/client/cli_metrics_commands.py @@ -0,0 +1,100 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import click +from rich_click import RichCommand + +from mephisto.utils.console_writer import ConsoleWriter +from mephisto.utils.metrics import cleanup_metrics +from mephisto.utils.metrics import launch_servers_and_wait +from mephisto.utils.metrics import metrics_are_installed +from mephisto.utils.metrics import METRICS_DIR +from mephisto.utils.metrics import remove_metrics_files +from mephisto.utils.metrics import run_install_script + +VERBOSITY_HELP = "write more informative messages about progress (Default 0. Values: 0, 1)" +VERBOSITY_DEFAULT_VALUE = 0 + +logger = ConsoleWriter() + + +@click.group(name="metrics", context_settings=dict(help_option_names=["-h", "--help"])) +def metrics_cli(): + """View task health and status with Mephisto Metrics""" + pass + + +# --- INSTALL --- +@metrics_cli.command("install", cls=RichCommand) +@click.pass_context +@click.option("-v", "--verbosity", type=int, default=VERBOSITY_DEFAULT_VALUE, help=VERBOSITY_HELP) +def install(ctx: click.Context, **options: dict): + """Installs Prometheus and Grafana to `METRICS_DIR`""" + + if metrics_are_installed(): + click.echo(f"Metrics are already installed! See {METRICS_DIR}") + return + + run_install_script() + + +# --- REINSTALL --- +@metrics_cli.command("reinstall", cls=RichCommand) +@click.pass_context +@click.option("-v", "--verbosity", type=int, default=VERBOSITY_DEFAULT_VALUE, help=VERBOSITY_HELP) +def reinstall(ctx: click.Context, **options: dict): + """Cleanup, remove and install Prometheus and Grafana from scratch""" + + cleanup_metrics() + + if metrics_are_installed(): + remove_metrics_files() + + run_install_script() + + +# --- VIEW --- +@metrics_cli.command("view", cls=RichCommand) +@click.pass_context +@click.option("-v", "--verbosity", type=int, default=VERBOSITY_DEFAULT_VALUE, help=VERBOSITY_HELP) +def view(ctx: click.Context, **options: dict): + """Launches a Prometheus and Grafana server, and shuts down on exit""" + + if not metrics_are_installed(): + click.echo(f"Metrics aren't installed! Use `mephisto metrics install` first.") + return + + click.echo(f"Servers launching - use CTRL-C to shutdown") + launch_servers_and_wait() + + +# --- CLEANUP --- +@metrics_cli.command("cleanup", cls=RichCommand) +@click.pass_context +@click.option("-v", "--verbosity", type=int, default=VERBOSITY_DEFAULT_VALUE, help=VERBOSITY_HELP) +def cleanup(ctx: click.Context, **options: dict): + """Shuts down Prometheus and Grafana resources that may have persisted""" + + if not metrics_are_installed(): + click.echo(f"Metrics aren't installed! Use `mephisto metrics install` first.") + return + + cleanup_metrics() + + +# --- REMOVE --- +@metrics_cli.command("remove", cls=RichCommand) +@click.pass_context +@click.option("-v", "--verbosity", type=int, default=VERBOSITY_DEFAULT_VALUE, help=VERBOSITY_HELP) +def remove(ctx: click.Context, **options: dict): + """Remove Prometheus and Grafana files""" + + if not metrics_are_installed(): + click.echo(f"Metrics aren't installed. Nothing to remove") + return + + cleanup_metrics() + remove_metrics_files() diff --git a/mephisto/client/cli_review_app_commands.py b/mephisto/client/cli_review_app_commands.py new file mode 100644 index 000000000..44d2c678e --- /dev/null +++ b/mephisto/client/cli_review_app_commands.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 + +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +import os +import subprocess +from typing import Optional + +import click +from flask.cli import pass_script_info +from flask.cli import ScriptInfo + +from mephisto.tools.scripts import build_custom_bundle +from mephisto.utils.console_writer import ConsoleWriter + +logger = ConsoleWriter() + + +@click.option("-h", "--host", type=str, default="127.0.0.1") +@click.option("-p", "--port", type=int, default=5000) +@click.option("-d", "--debug", type=bool, default=False, is_flag=True) +@click.option("-f", "--force-rebuild", type=bool, default=False, is_flag=True) +@click.option("-s", "--skip-build", type=bool, default=False, is_flag=True) +@pass_script_info +def review_app( + info: ScriptInfo, + host: Optional[str], + port: Optional[int], + debug: bool = False, + force_rebuild: bool = False, + skip_build: bool = False, +): + """ + Launch a local review server. + Custom implementation of `flask run ` command (`flask.cli.run_command`) + """ + from flask.cli import show_server_banner + from flask.helpers import get_debug_flag + from mephisto.review_app.server import create_app + from werkzeug.serving import run_simple + + # Set env variables for Review App + app_url = f"http://{host}:{port}" + os.environ["HOST"] = host + os.environ["PORT"] = str(port) + + logger.info(f'[green]Review APP will start on "{app_url}" address.[/green]') + + # Set up Review App Client + if not skip_build: + review_app_path = os.path.join( + os.path.dirname(os.path.dirname(os.path.abspath(__file__))), + "review_app", + ) + client_dir = "client" + client_path = os.path.join(review_app_path, client_dir) + + # Install JS requirements + if os.path.exists(os.path.join(client_path, "node_modules")): + logger.info(f"[blue]JS requirements are already installed.[/blue]") + else: + logger.info(f"[blue]Installing JS requirements started.[/blue]") + app_started = subprocess.call(["npm", "install"], cwd=client_path) + if app_started != 0: + raise Exception( + "Please make sure npm is installed, " + "otherwise view the above error for more info." + ) + logger.info(f"[blue]Installing JS requirements finished.[/blue]") + + if os.path.exists(os.path.join(client_path, "build", "index.html")) and not force_rebuild: + logger.info(f"[blue]React bundle is already built.[/blue]") + else: + logger.info(f"[blue]Building React bundle started.[/blue]") + build_custom_bundle( + review_app_path, + force_rebuild=force_rebuild, + webapp_name=client_dir, + build_command="build", + ) + logger.info(f"[blue]Building React bundle finished.[/blue]") + + # Set debug + debug = debug if debug is not None else get_debug_flag() + reload = debug + debugger = debug + + # Show Flask banner + show_server_banner(debug, info.app_import_path) + + # Init Flask App + app = create_app(debug=debug) + + # Run Flask server + run_simple( + host, + port, + app, + use_reloader=reload, + use_debugger=debugger, + ) diff --git a/mephisto/client/cli_scripts_commands.py b/mephisto/client/cli_scripts_commands.py new file mode 100644 index 000000000..21e405b4d --- /dev/null +++ b/mephisto/client/cli_scripts_commands.py @@ -0,0 +1,136 @@ +# Copyright (c) Meta Platforms and its affiliates. +# This source code is licensed under the MIT license found in the +# LICENSE file in the root directory of this source tree. + +from typing import Any +from typing import List +from typing import Optional + +import click + +import mephisto.scripts.form_composer.rebuild_all_apps as rebuild_all_apps_form_composer +import mephisto.scripts.heroku.initialize_heroku as initialize_heroku +import mephisto.scripts.local_db.clear_worker_onboarding as clear_worker_onboarding_local_db +import mephisto.scripts.local_db.load_data_to_mephisto_db as load_data_local_db +import mephisto.scripts.local_db.remove_accepted_tip as remove_accepted_tip_local_db +import mephisto.scripts.local_db.review_feedback_for_task as review_feedback_local_db +import mephisto.scripts.local_db.review_tips_for_task as review_tips_local_db +import mephisto.scripts.metrics.shutdown_metrics as shutdown_metrics +import mephisto.scripts.metrics.view_metrics as view_metrics +import mephisto.scripts.mturk.cleanup as cleanup_mturk +import mephisto.scripts.mturk.identify_broken_units as identify_broken_units_mturk +import mephisto.scripts.mturk.launch_makeup_hits as launch_makeup_hits_mturk +import mephisto.scripts.mturk.print_outstanding_hit_status as soft_block_workers_by_mturk_id_mturk +from mephisto.scripts.local_db import auto_generate_all_docs_reference_md +from mephisto.utils.console_writer import ConsoleWriter + +FORM_COMPOSER_VALID_SCRIPTS_NAMES = [ + "rebuild_all_apps", +] +HEROKU_VALID_SCRIPTS_NAMES = [ + "initialize", +] +LOCAL_DB_VALID_SCRIPTS_NAMES = [ + "review_tips", + "remove_tip", + "review_feedback", + "load_data", + "clear_worker_onboarding", + "auto_generate_all_docs_reference_md", +] +METRICS_VALID_SCRIPTS_NAMES = [ + "view", + "shutdown", +] +MTURK_VALID_SCRIPTS_NAMES = [ + "cleanup", + "identify_broken_units", + "launch_makeup_hits", + "print_outstanding_hit_status", + "soft_block_workers_by_mturk_id", +] +VALID_SCRIPT_TYPES = [ + "local_db", + "heroku", + "metrics", + "mturk", + "form_composer", +] + +logger = ConsoleWriter() + + +@click.argument("script_type", required=False, nargs=1) +@click.argument("script_name", required=False, nargs=1) +@click.argument("args", nargs=-1) # Allow arguments for low level commands +def run_script(script_type, script_name, args: Optional[Any] = None): + """Run one of the many mephisto scripts.""" + + def print_non_markdown_list(items: List[str]): + res = "" + for item in items: + res += "\n * " + item + return res + + if script_type is None or script_type.strip() not in VALID_SCRIPT_TYPES: + logger.info("") + raise click.UsageError( + "You must specify a valid script_type from below. \n\nValid script types are:" + + print_non_markdown_list(VALID_SCRIPT_TYPES) + ) + script_type = script_type.strip() + + script_type_to_scripts_data = { + "local_db": { + "valid_script_names": LOCAL_DB_VALID_SCRIPTS_NAMES, + "scripts": { + LOCAL_DB_VALID_SCRIPTS_NAMES[0]: review_tips_local_db.main, + LOCAL_DB_VALID_SCRIPTS_NAMES[1]: remove_accepted_tip_local_db.main, + LOCAL_DB_VALID_SCRIPTS_NAMES[2]: review_feedback_local_db.main, + LOCAL_DB_VALID_SCRIPTS_NAMES[3]: load_data_local_db.main, + LOCAL_DB_VALID_SCRIPTS_NAMES[4]: clear_worker_onboarding_local_db.main, + LOCAL_DB_VALID_SCRIPTS_NAMES[5]: auto_generate_all_docs_reference_md.main, + }, + }, + "heroku": { + "valid_script_names": HEROKU_VALID_SCRIPTS_NAMES, + "scripts": {HEROKU_VALID_SCRIPTS_NAMES[0]: initialize_heroku.main}, + }, + "metrics": { + "valid_script_names": METRICS_VALID_SCRIPTS_NAMES, + "scripts": { + METRICS_VALID_SCRIPTS_NAMES[0]: view_metrics.launch_servers, + METRICS_VALID_SCRIPTS_NAMES[1]: shutdown_metrics.shutdown_servers, + }, + }, + "mturk": { + "valid_script_names": MTURK_VALID_SCRIPTS_NAMES, + "scripts": { + MTURK_VALID_SCRIPTS_NAMES[0]: cleanup_mturk.main, + MTURK_VALID_SCRIPTS_NAMES[1]: identify_broken_units_mturk.main, + MTURK_VALID_SCRIPTS_NAMES[2]: launch_makeup_hits_mturk.main, + MTURK_VALID_SCRIPTS_NAMES[3]: rebuild_all_apps_form_composer.main, + MTURK_VALID_SCRIPTS_NAMES[4]: soft_block_workers_by_mturk_id_mturk.main, + }, + }, + "form_composer": { + "valid_script_names": FORM_COMPOSER_VALID_SCRIPTS_NAMES, + "scripts": { + FORM_COMPOSER_VALID_SCRIPTS_NAMES[0]: rebuild_all_apps_form_composer.main, + }, + }, + } + + if script_name is None or ( + script_name not in script_type_to_scripts_data[script_type]["valid_script_names"] + ): + logger.info("") + raise click.UsageError( + "You must specify a valid script_name from below. \n\nValid script names are:" + + print_non_markdown_list( + script_type_to_scripts_data[script_type]["valid_script_names"] + ) + ) + + # runs the script + script_type_to_scripts_data[script_type]["scripts"][script_name]() diff --git a/mephisto/client/cli_commands.py b/mephisto/client/cli_wut_commands.py similarity index 78% rename from mephisto/client/cli_commands.py rename to mephisto/client/cli_wut_commands.py index bcbb78ce9..11397c89b 100644 --- a/mephisto/client/cli_commands.py +++ b/mephisto/client/cli_wut_commands.py @@ -9,6 +9,7 @@ def get_wut_arguments(args): """Display information about hydra config properties""" + from mephisto.operations.registry import ( get_blueprint_from_type, get_crowd_provider_from_type, @@ -23,20 +24,36 @@ def get_wut_arguments(args): abstractions_table = create_table(["Abstraction", "Description"], "\n\n[b]Abstractions[/b]") abstractions_table.add_row( "blueprint", - f"The blueprint contains all of the related code required to set up a task run. \nValid blueprints types are [b]{get_valid_blueprint_types()}[/b]", + ( + f"The blueprint contains all of the related code required to set up a task run.\n" + f"Valid blueprints types are [b]{get_valid_blueprint_types()}[/b]" + ), ) abstractions_table.add_row( "architect", - f"Architect's contain the logic surrounding deploying a server that workers will be able to access. \nValid architects types are [b]{get_valid_architect_types()}[/b]", + ( + f"Architect's contain the logic surrounding " + f"deploying a server that workers will be able to access.\n" + f"Valid architects types are [b]{get_valid_architect_types()}[/b]" + ), ) abstractions_table.add_row( "requester", - f"The requester is an account for a crowd provider. Requesters are used as the identity that launches the task. \nValid requester types types are [b]{get_valid_provider_types()}[/b]. \n" - "\nUse `mephisto requesters` to see registered requesters, and `mephisto register ` to register.", + ( + f"The requester is an account for a crowd provider. " + f"Requesters are used as the identity that launches the task.\n" + f"Valid requester types types are [b]{get_valid_provider_types()}[/b].\n\n" + "Use `mephisto requesters` to see registered requesters, " + "and `mephisto register ` to register." + ), ) abstractions_table.add_row( "provider", - f"The crowd provider is responsible for standardizing Mephisto's interaction with external crowds. \nValid provider types are [b]{get_valid_provider_types()}[/b]", + ( + f"The crowd provider is responsible for standardizing Mephisto's " + f"interaction with external crowds.\n" + f"Valid provider types are [b]{get_valid_provider_types()}[/b]" + ), ) console.print(abstractions_table) return @@ -55,7 +72,9 @@ def get_wut_arguments(args): if abstraction not in VALID_ABSTRACTIONS: print( - f"[red]Given abstraction {abstraction} not in valid abstractions {VALID_ABSTRACTIONS}][/red]" + f"[red]" + f"Given abstraction {abstraction} not in valid abstractions {VALID_ABSTRACTIONS}]" + f"[/red]" ) return @@ -65,7 +84,7 @@ def get_wut_arguments(args): target_class = TaskRun else: if len(abstraction_equal_split) == 1: - # querying about the general abstraction + # Querying about the general abstraction if abstraction == "blueprint": click.echo("The blueprint determines the task content.\n") valid_blueprints_text = """**Valid blueprints are:**""" @@ -79,7 +98,8 @@ def get_wut_arguments(args): elif abstraction == "requester": click.echo( f"The requester is an account for a crowd provider. \n" - "Use `mephisto requesters` to see registered requesters, and `mephisto register ` to register.\n" + "Use `mephisto requesters` to see registered requesters, " + "and `mephisto register ` to register.\n" ) valid_requester_text = """**Valid requesters are:**""" print_out_valid_options(valid_requester_text, get_valid_provider_types()) @@ -114,6 +134,7 @@ def get_wut_arguments(args): target_class = get_crowd_provider_from_type(abstract_value).RequesterClass except: valid = get_valid_provider_types() + if valid is not None: print(f"\n[b]The valid types for {abstraction} are:[/b]") valid_options_text = """""" @@ -121,7 +142,8 @@ def get_wut_arguments(args): print(f"[red]'{abstract_value}' not found[/red]\n") return - # These are values that do not convert to a string well or are incorrect, so they need to be hardcoded + # These are values that do not convert to a string well or are incorrect, + # so they need to be hardcoded argument_overrides = { "tips_location": ("default", "path_to_task/assets/tips.csv"), "heroku_config_args": ("default", "{}"), @@ -130,9 +152,11 @@ def get_wut_arguments(args): arg_dict = get_extra_argument_dicts(target_class)[0] click.echo(arg_dict["desc"]) + checking_args = arg_dict["args"] if len(args) > 1: checking_args = {k: v for k, v in checking_args.items() if k in args[1:]} + checking_args_keys = list(checking_args.keys()) if len(checking_args_keys) > 0: first_arg = checking_args_keys[0] @@ -141,6 +165,7 @@ def get_wut_arguments(args): first_arg_keys, "\n[b]{abstraction} Arguments[/b]".format(abstraction=abstraction.capitalize()), ) + for arg in checking_args: if arg in argument_overrides: checking_args[arg][argument_overrides[arg][0]] = argument_overrides[arg][1] @@ -155,10 +180,13 @@ def get_wut_arguments(args): state_args = get_task_state_dicts(target_class)[0]["args"] if len(args) > 1: state_args = {k: v for k, v in state_args.items() if k in args[1:]} + state_args_keys = list(state_args.keys()) if len(state_args_keys) > 0: click.echo( - f"\n\nAdditional SharedTaskState args from {target_class.SharedStateClass.__name__}, which may be configured in your run script" + f"\n\n" + f"Additional SharedTaskState args from {target_class.SharedStateClass.__name__}, " + f"which may be configured in your run script" ) first_state_arg = state_args_keys[0] first_arg_keys = list(state_args[first_state_arg].keys()) @@ -173,3 +201,9 @@ def get_wut_arguments(args): state_args_table.add_row(*arg_values) console.print(state_args_table) return [arg_dict, state_args] + + +@click.argument("args", nargs=-1) +def run_wut(args): + """Discover the configuration arguments for different abstractions""" + get_wut_arguments(args) diff --git a/mephisto/scripts/form_composer/rebuild_all_apps.py b/mephisto/scripts/form_composer/rebuild_all_apps.py index a7f57b19a..b5d32fd8a 100644 --- a/mephisto/scripts/form_composer/rebuild_all_apps.py +++ b/mephisto/scripts/form_composer/rebuild_all_apps.py @@ -138,7 +138,7 @@ def _build_examples_form_composer_demo(repo_path: str): print(f"[blue]Building '{app_path}'[/blue]") # Set env var for `custom_validators.js` - from mephisto.client.cli import FORM_COMPOSER__DATA_DIR_NAME + from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME, "dynamic") set_custom_validators_js_env_var(data_path) @@ -171,7 +171,7 @@ def _build_generators_form_composer(repo_path: str): print(f"[blue]Building '{app_path}'[/blue]") # Set env var for `custom_validators.js` - from mephisto.client.cli import FORM_COMPOSER__DATA_DIR_NAME + from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_DIR_NAME data_path = os.path.join(app_path, FORM_COMPOSER__DATA_DIR_NAME) set_custom_validators_js_env_var(data_path) diff --git a/mephisto/scripts/local_db/gh_actions/auto_generate_architect.py b/mephisto/scripts/local_db/gh_actions/auto_generate_architect.py index eca8fd34c..249ec5a58 100644 --- a/mephisto/scripts/local_db/gh_actions/auto_generate_architect.py +++ b/mephisto/scripts/local_db/gh_actions/auto_generate_architect.py @@ -7,7 +7,7 @@ from mdutils.mdutils import MdUtils -from mephisto.client.cli_commands import get_wut_arguments +from mephisto.client.cli_wut_commands import get_wut_arguments from mephisto.operations.registry import get_valid_architect_types from mephisto.scripts.local_db.gh_actions.auto_generate_blueprint import ( add_object_args_table_info, diff --git a/mephisto/scripts/local_db/gh_actions/auto_generate_blueprint.py b/mephisto/scripts/local_db/gh_actions/auto_generate_blueprint.py index 7611c53f7..068bc7a74 100644 --- a/mephisto/scripts/local_db/gh_actions/auto_generate_blueprint.py +++ b/mephisto/scripts/local_db/gh_actions/auto_generate_blueprint.py @@ -9,7 +9,7 @@ from mdutils.mdutils import MdUtils -from mephisto.client.cli_commands import get_wut_arguments +from mephisto.client.cli_wut_commands import get_wut_arguments from mephisto.operations.registry import get_valid_blueprint_types diff --git a/mephisto/scripts/local_db/gh_actions/auto_generate_provider.py b/mephisto/scripts/local_db/gh_actions/auto_generate_provider.py index 687a33b5c..650654089 100644 --- a/mephisto/scripts/local_db/gh_actions/auto_generate_provider.py +++ b/mephisto/scripts/local_db/gh_actions/auto_generate_provider.py @@ -7,7 +7,7 @@ from mdutils.mdutils import MdUtils -from mephisto.client.cli_commands import get_wut_arguments +from mephisto.client.cli_wut_commands import get_wut_arguments from mephisto.operations.registry import ( get_valid_provider_types, ) diff --git a/mephisto/scripts/local_db/gh_actions/auto_generate_requester.py b/mephisto/scripts/local_db/gh_actions/auto_generate_requester.py index d94c01fbd..b381c215a 100644 --- a/mephisto/scripts/local_db/gh_actions/auto_generate_requester.py +++ b/mephisto/scripts/local_db/gh_actions/auto_generate_requester.py @@ -7,7 +7,7 @@ from mdutils.mdutils import MdUtils -from mephisto.client.cli_commands import get_wut_arguments +from mephisto.client.cli_wut_commands import get_wut_arguments from mephisto.operations.registry import ( get_valid_provider_types, ) diff --git a/mephisto/scripts/local_db/gh_actions/auto_generate_task.py b/mephisto/scripts/local_db/gh_actions/auto_generate_task.py index 94f2ae26a..aa5e35fa9 100644 --- a/mephisto/scripts/local_db/gh_actions/auto_generate_task.py +++ b/mephisto/scripts/local_db/gh_actions/auto_generate_task.py @@ -7,7 +7,7 @@ from mdutils.mdutils import MdUtils -from mephisto.client.cli_commands import get_wut_arguments +from mephisto.client.cli_wut_commands import get_wut_arguments from mephisto.scripts.local_db.gh_actions.auto_generate_blueprint import ( add_object_args_table_info, ) diff --git a/mephisto/scripts/metrics/install_metrics.sh b/mephisto/scripts/metrics/install_metrics.sh index 6a46163f0..1dd872e0d 100755 --- a/mephisto/scripts/metrics/install_metrics.sh +++ b/mephisto/scripts/metrics/install_metrics.sh @@ -2,10 +2,15 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. +command_exists() { + type "$1" &> /dev/null ; +} + +# 1. Check OS type if [ -z "$OSTYPE" ] then OSTYPE=$(uname) - echo "Checking OS type returned ${OSTYPE}" + echo "Checking OS type returned ${OSTYPE}..." fi case "$OSTYPE" in darwin* | Darwin*) @@ -20,51 +25,70 @@ case "$OSTYPE" in ;; esac +# 2. Check if CURL installed. +# Some systems may have not installed CURL. In Docker we install it on building image +if ! command_exists "curl" +then + echo "Please install 'curl'" + exit 0 +fi + +# 3. Check if metrics were already installed if [ -d "prometheus" ] then - echo "Prometheus directory already exists, skipping install" - exit 0 + echo "Prometheus directory already exists, skipping install\n" + exit 0 fi -# Download and unpack grafana +# 4. Download and unpack Grafana +echo "\nInstalling Grafana..." + curl -O "https://dl.grafana.com/oss/release/grafana-8.4.2.$platform-amd64.tar.gz" tar -zxvf "grafana-8.4.2.$platform-amd64.tar.gz" > /dev/null 2>&1 mv grafana-8.4.2 grafana rm "grafana-8.4.2.$platform-amd64.tar.gz" cp resources/grafana_defaults.ini grafana/conf/defaults.ini -# # Download and unpack prometheus +echo "Installing Grafana finished!\n" + + +# 5. Download and unpack Prometheus +echo "\nInstalling Prometheus..." + curl -L -O "https://github.com/prometheus/prometheus/releases/download/v2.33.4/prometheus-2.33.4.$platform-amd64.tar.gz" tar xvfz prometheus-*.tar.gz > /dev/null 2>&1 mv "prometheus-2.33.4.$platform-amd64" prometheus rm "prometheus-2.33.4.$platform-amd64.tar.gz" cp resources/mephisto-prometheus-config.yml prometheus/prometheus.yml -# Run grafana in background to receive the desired defaults +echo "Installing Prometheus finished!\n" + +# 6. Run grafana in background to receive the desired defaults cd grafana ./bin/grafana-server > /dev/null 2>&1 & GRAFANA_PID=$! cd .. -until $(curl --output /dev/null --silent --head --fail http://localhost:3032); do - printf '.' - sleep 1 +grafana_url=http://$GRAFANA_HOST:$GRAFANA_PORT +until curl --output /dev/null --silent --head --fail $grafana_url; do + printf '.' + sleep 1 done -# Copy over the Mephisto datasource -curl -X "POST" "http://localhost:3032/api/datasources" \ --H "Content-Type: application/json" \ - --user admin:admin \ - --data-binary @resources/mephisto_source.json +# 7. Copy over the Mephisto datasource +curl -X "POST" "${grafana_url}/api/datasources" \ + -H "Content-Type: application/json" \ + --user $GRAFANA_USER:$GRAFANA_PASSWORD \ + --data-binary @resources/mephisto_source.json -# Copy over the mephisto default dashboard -curl --fail -k -X "POST" "http://localhost:3032/api/dashboards/db" \ - -H "Content-Type: application/json" \ - -H "Accept: application/json" \ - --user admin:admin \ - --data-binary @resources/default_mephisto_dash.json +# 8. Copy over the mephisto default dashboard +curl --fail -k -X "POST" "${grafana_url}/api/dashboards/db" \ + -H "Content-Type: application/json" \ + -H "Accept: application/json" \ + --user $GRAFANA_USER:$GRAFANA_PASSWORD \ + --data-binary @resources/default_mephisto_dash.json -# Close grafana +# 9. Close grafana kill $GRAFANA_PID sleep 3 diff --git a/mephisto/scripts/metrics/resources/grafana_defaults.ini b/mephisto/scripts/metrics/resources/grafana_defaults.ini index 2713b6389..d12653c30 100644 --- a/mephisto/scripts/metrics/resources/grafana_defaults.ini +++ b/mephisto/scripts/metrics/resources/grafana_defaults.ini @@ -35,10 +35,10 @@ protocol = http http_addr = # The http port to use -http_port = 3032 +http_port = ${GRAFANA_PORT} # The public facing domain name used to access grafana from a browser -domain = localhost +domain = ${GRAFANA_HOST} # Redirect to correct domain if host header does not match domain # Prevents DNS rebinding attacks @@ -229,10 +229,10 @@ application_insights_endpoint_url = disable_initial_admin_creation = false # default admin user, created on startup -admin_user = admin +admin_user = ${GRAFANA_USER} # default admin password, can be changed before first start of grafana, or in profile settings -admin_password = admin +admin_password = ${GRAFANA_PASSWORD} # used for signing secret_key = SW2YcwTIb9zpOOhoPsMm diff --git a/mephisto/scripts/metrics/view_metrics.py b/mephisto/scripts/metrics/view_metrics.py index eefbd7391..babd5d1ac 100644 --- a/mephisto/scripts/metrics/view_metrics.py +++ b/mephisto/scripts/metrics/view_metrics.py @@ -5,32 +5,42 @@ # LICENSE file in the root directory of this source tree. import time -from mephisto.utils.metrics import ( - launch_grafana_server, - launch_prometheus_server, - get_dash_url, -) + +from mephisto.utils.console_writer import ConsoleWriter from mephisto.utils.logger_core import set_mephisto_log_level +from mephisto.utils.metrics import get_default_dashboard_url +from mephisto.utils.metrics import launch_grafana_server +from mephisto.utils.metrics import launch_prometheus_server + +logger = ConsoleWriter() def launch_servers(): """ - Launches a prometheus and grafana server instances and print the address as well as shutdown instructions + Launches a prometheus and grafana server instances and print the address + as well as shutdown instructions """ - print("Launching servers") + logger.info("Launching servers") + set_mephisto_log_level(level="info") + if not launch_grafana_server(): - print("Issue launching grafana, see above") + logger.info("Issue launching grafana, see above") return + if not launch_prometheus_server(): - print("Issue launching prometheus, see above") + logger.info("Issue launching prometheus, see above") return - print(f"Waiting for grafana server to come up.") + + logger.info(f"Waiting for grafana server to come up.") + time.sleep(3) - dash_url = get_dash_url() - print(f"Dashboard is now running, you can access it at {dash_url}") - print( - f"Once you're no longer using it, and no jobs need it anymore, you can shutdown with `shutdown_metrics.py`" + + default_dashboard_url = get_default_dashboard_url() + logger.info(f"Dashboard is now running, you can access it at {default_dashboard_url}") + logger.info( + f"Once you're no longer using it, and no jobs need it anymore, " + f"you can shutdown with `shutdown_metrics.py`" ) diff --git a/mephisto/utils/metrics.py b/mephisto/utils/metrics.py index 9dbd6c85c..c8465f719 100644 --- a/mephisto/utils/metrics.py +++ b/mephisto/utils/metrics.py @@ -4,41 +4,51 @@ # This source code is licensed under the MIT license found in the # LICENSE file in the root directory of this source tree. -# Note, all of the functions in this file take an `args` option which currently +# Note, all the functions in this file take an `args` option which currently # goes unused, but is planned for making the system more configurable in 1.1 -import os import errno -import requests -from requests.auth import HTTPBasicAuth +import os +import shutil import subprocess import time - -from mephisto.utils.dirs import get_mephisto_tmp_dir, get_root_dir -from mephisto.utils.logger_core import get_logger, warn_once -from prometheus_client import start_http_server -from omegaconf import DictConfig from typing import Optional +from urllib.parse import urljoin +import click +import requests +from omegaconf import DictConfig +from prometheus_client import start_http_server +from requests.auth import HTTPBasicAuth -logger = get_logger(name=__name__) +from mephisto.utils.dirs import get_mephisto_tmp_dir +from mephisto.utils.dirs import get_root_dir +from mephisto.utils.logger_core import get_logger +from mephisto.utils.logger_core import warn_once +METRICS_DIR = os.path.join(get_root_dir(), "mephisto", "scripts", "metrics") -PROMETHEUS_PID_FILE = os.path.join(get_mephisto_tmp_dir(), "PROMETHEUS_PID.txt") +GRAFANA_HOST = os.environ.get("GRAFANA_HOST", "localhost") +GRAFANA_PORT = os.environ.get("GRAFANA_PORT", 3032) +GRAFANA_USER = os.environ.get("GRAFANA_USER", "admin") +GRAFANA_PASSWORD = os.environ.get("GRAFANA_PASSWORD", "admin") +GRAFANA_DIR = os.path.join(METRICS_DIR, "grafana") +GRAFANA_EXECUTABLE = os.path.join(GRAFANA_DIR, "bin", "grafana-server") GRAFANA_PID_FILE = os.path.join(get_mephisto_tmp_dir(), "GRAFANA_PID.txt") -METRICS_DIR = os.path.join(get_root_dir(), "mephisto", "scripts", "metrics") + PROMETHEUS_DIR = os.path.join(METRICS_DIR, "prometheus") -PROMETHEUS_EXECUTABLE = os.path.join(PROMETHEUS_DIR, "prometheus") PROMETHEUS_CONFIG = os.path.join(PROMETHEUS_DIR, "prometheus.yml") -GRAFANA_DIR = os.path.join(METRICS_DIR, "grafana") -GRAFANA_EXECUTABLE = os.path.join(GRAFANA_DIR, "bin", "grafana-server") +PROMETHEUS_EXECUTABLE = os.path.join(PROMETHEUS_DIR, "prometheus") +PROMETHEUS_PID_FILE = os.path.join(get_mephisto_tmp_dir(), "PROMETHEUS_PID.txt") + +logger = get_logger(name=__name__) class InaccessiblePrometheusServer(Exception): pass -def _server_process_running(pid): +def _server_process_running(pid) -> bool: """Check on the existing process id""" try: os.kill(pid, 0) @@ -55,15 +65,19 @@ def _server_process_running(pid): return True -def _get_pid_from_file(fn): +def _get_pid_from_file(fn) -> int: """Get the PID from the given file""" with open(fn) as pid_file: pid = int(pid_file.read().strip()) return pid +def _get_grafana_url() -> str: + return f"http://{GRAFANA_HOST}:{GRAFANA_PORT}/" + + def run_install_script() -> bool: - """Run the install script from METRICS_DIR""" + """Run the installation script from METRICS_DIR""" res = subprocess.check_call( [ "sh", @@ -74,7 +88,7 @@ def run_install_script() -> bool: return res == 0 -def metrics_are_installed(): +def metrics_are_installed() -> bool: """Return whether metrics are installed""" return os.path.exists(PROMETHEUS_EXECUTABLE) and os.path.exists(GRAFANA_EXECUTABLE) @@ -87,20 +101,27 @@ def launch_servers_and_wait(): """ try: print("Servers launching...") + if not launch_grafana_server(): print("Issue launching grafana, see above") return + if not launch_prometheus_server(): print("Issue launching prometheus, see above") return + print(f"Waiting for grafana server to come up.") + time.sleep(3) - dash_url = get_dash_url() - print(f"Dashboard is now running, you can access it at http://{dash_url}") - print(f"===========================") - print(f"| Default username: admin |") - print(f"| Default password: admin |") - print(f"===========================") + default_dashboard_url = get_default_dashboard_url() + + print( + f"Dashboard is now running, you can access it at http://{default_dashboard_url}\n" + f"===========================\n" + f"| Default username: {GRAFANA_USER} |\n" + f"| Default password: {GRAFANA_PASSWORD} |\n" + f"===========================\n" + ) while True: # Relaunch the server in case it's shut down by a @@ -108,6 +129,7 @@ def launch_servers_and_wait(): time.sleep(5) if not os.path.exists(PROMETHEUS_PID_FILE): launch_prometheus_server() + except KeyboardInterrupt: print("Caught Ctrl-C, shutting down servers") finally: @@ -126,8 +148,12 @@ def start_metrics_server(args: Optional["DictConfig"] = None): start_http_server(3031) except Exception as e: logger.exception( - "Could not launch prometheus metrics client, perhaps a process is already running on 3031? " - "Mephisto metrics currently only supports one Operator class at a time at the moment", + ( + "Could not launch prometheus metrics client, " + "perhaps a process is already running on 3031? " + "Mephisto metrics currently only supports " + "one Operator class at a time at the moment" + ), exc_info=True, ) @@ -144,11 +170,14 @@ def launch_prometheus_server(args: Optional["DictConfig"] = None) -> bool: is_ok = r.ok except requests.exceptions.ConnectionError: is_ok = False + if not is_ok: logger.warning("Prometheus PID existed, but server doesn't appear to be up.") + if _server_process_running(_get_pid_from_file(PROMETHEUS_PID_FILE)): logger.warning( - "Prometheus server appears to be running though! exiting as unsure what to do..." + "Prometheus server appears to be running though! " + "exiting as unsure what to do..." ) raise InaccessiblePrometheusServer("Prometheus server running but inaccessible") else: @@ -159,13 +188,16 @@ def launch_prometheus_server(args: Optional["DictConfig"] = None) -> bool: else: logger.debug("Prometheus server appears to be running at 9090") return True + if not os.path.exists(PROMETHEUS_EXECUTABLE): warn_once( - f"Mephisto supports rich run-time metrics visualization through Prometheus and Grafana. " + f"Mephisto supports rich run-time metrics visualization " + f"through Prometheus and Grafana. " f"If you'd like to use this feature, use `mephisto metrics install` to install." f"(Current install dir is '{METRICS_DIR}')'" ) return False + proc = subprocess.Popen( [ f"./prometheus", @@ -175,6 +207,7 @@ def launch_prometheus_server(args: Optional["DictConfig"] = None) -> bool: stderr=subprocess.DEVNULL, cwd=f"{PROMETHEUS_DIR}", ) + with open(PROMETHEUS_PID_FILE, "w+") as pid_file: pid_file.write(str(proc.pid)) @@ -182,16 +215,18 @@ def launch_prometheus_server(args: Optional["DictConfig"] = None) -> bool: return True -def launch_grafana_server(args: Optional["DictConfig"] = None) -> bool: +def launch_grafana_server() -> bool: """ Launch a grafana server if one is not already running (based on having an expected PID) """ if os.path.exists(GRAFANA_PID_FILE): try: - r = requests.get("http://localhost:3032/") + grafana_url = _get_grafana_url() + r = requests.get(grafana_url) is_ok = r.ok except requests.exceptions.ConnectionError: is_ok = False + if not is_ok: logger.warning("Grafana PID existed, but server doesn't appear to be up.") if _server_process_running(_get_pid_from_file(GRAFANA_PID_FILE)): @@ -207,19 +242,23 @@ def launch_grafana_server(args: Optional["DictConfig"] = None) -> bool: else: logger.debug("Grafana server appears to be running at 3032") return True + if not os.path.exists(GRAFANA_EXECUTABLE): warn_once( - f"Mephisto supports rich run-time metrics visualization through Prometheus and Grafana. " + f"Mephisto supports rich run-time metrics visualization " + f"through Prometheus and Grafana. " f"If you'd like to use this feature, use `mephisto metrics install` to install." f"(Current install dir is '{METRICS_DIR}')'" ) return False + proc = subprocess.Popen( [f"./bin/grafana-server"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=f"{GRAFANA_DIR}", ) + with open(GRAFANA_PID_FILE, "w+") as pid_file: pid_file.write(str(proc.pid)) @@ -227,18 +266,23 @@ def launch_grafana_server(args: Optional["DictConfig"] = None) -> bool: return True -def get_dash_url(args: Optional["DictConfig"] = None): +def get_default_dashboard_url() -> str: """ Return the url to the default Mephisto dashboard. Requires a running grafana server """ headers_dict = {"Accept": "application/json"} + grafana_url = _get_grafana_url() + url = urljoin(grafana_url, "/api/search?query=Default%20Mephisto%20Monitorin") + r = requests.get( - "http://localhost:3032/api/search?query=Default%20Mephisto%20Monitoring", + url, headers=headers_dict, - auth=HTTPBasicAuth("admin", "admin"), + auth=HTTPBasicAuth(GRAFANA_USER, GRAFANA_PASSWORD), ) output = r.json() - return f"localhost:3032{output[0]['url']}" + + default_dashboard_url = f"{GRAFANA_HOST}:{GRAFANA_PORT}{output[0]['url']}" + return default_dashboard_url def shutdown_prometheus_server(args: Optional["DictConfig"] = None, expect_exists=False): @@ -269,3 +313,19 @@ def shutdown_grafana_server(args: Optional["DictConfig"] = None, expect_exists=F f"No PID file at {GRAFANA_PID_FILE}... Check lsof -i :3032 to find the " "process if it still exists and interrupt it yourself." ) + + +def remove_metrics_files(): + click.echo(f"Removing Grafana files...") + shutil.rmtree(os.path.join(METRICS_DIR, "grafana")) + click.echo(f"Removing Grafana files finished") + + click.echo(f"Removing Prometheus files...") + shutil.rmtree(os.path.join(METRICS_DIR, "prometheus")) + click.echo(f"Removing Prometheus files finished") + + +def cleanup_metrics(): + click.echo(f"Cleaning up existing servers if they exist") + shutdown_prometheus_server() + shutdown_grafana_server() diff --git a/test/generators/form_composer/config_validation/test_separate_token_values_config.py b/test/generators/form_composer/config_validation/test_separate_token_values_config.py index 459daabba..0ab417ba4 100644 --- a/test/generators/form_composer/config_validation/test_separate_token_values_config.py +++ b/test/generators/form_composer/config_validation/test_separate_token_values_config.py @@ -7,7 +7,9 @@ from botocore.exceptions import NoCredentialsError -from mephisto.client.cli import FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import ( + FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME, +) from mephisto.generators.form_composer.config_validation.separate_token_values_config import ( update_separate_token_values_config_with_file_urls, ) diff --git a/test/generators/form_composer/config_validation/test_task_data_config.py b/test/generators/form_composer/config_validation/test_task_data_config.py index ae8868602..8014613c6 100644 --- a/test/generators/form_composer/config_validation/test_task_data_config.py +++ b/test/generators/form_composer/config_validation/test_task_data_config.py @@ -8,10 +8,12 @@ from copy import deepcopy from unittest.mock import patch -from mephisto.client.cli import FORM_COMPOSER__DATA_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__FORM_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__DATA_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__FORM_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import ( + FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME, +) +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME from mephisto.generators.form_composer.config_validation.task_data_config import ( _collect_form_config_items_to_extrapolate, ) diff --git a/test/generators/form_composer/config_validation/test_token_sets_values_config.py b/test/generators/form_composer/config_validation/test_token_sets_values_config.py index 2dc03b3bf..1b8f57c91 100644 --- a/test/generators/form_composer/config_validation/test_token_sets_values_config.py +++ b/test/generators/form_composer/config_validation/test_token_sets_values_config.py @@ -4,8 +4,10 @@ import tempfile import unittest -from mephisto.client.cli import FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME -from mephisto.client.cli import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME +from mephisto.client.cli_form_composer_commands import ( + FORM_COMPOSER__SEPARATE_TOKEN_VALUES_CONFIG_NAME, +) +from mephisto.client.cli_form_composer_commands import FORM_COMPOSER__TOKEN_SETS_VALUES_CONFIG_NAME from mephisto.generators.form_composer.config_validation.token_sets_values_config import ( _premutate_separate_tokens, )