From 84e66579502ceccd194129906fa48f7472726652 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 23 Oct 2024 14:56:47 -0400 Subject: [PATCH 01/12] Fix galaxy-manage-db for the installed context --- lib/galaxy/model/orm/scripts.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index e48d639e92f6..77dc660d69f2 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -28,6 +28,8 @@ DEFAULT_CONFIG_NAMES = ["galaxy", "universe_wsgi"] DEFAULT_CONFIG_PREFIX = "" DEFAULT_DATABASE = "galaxy" +ALEMBIC_CONFIG = os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "migrations", "alembic.ini")) + DATABASE = { "galaxy": { @@ -163,6 +165,9 @@ def _insert_x_argument(key, value): _insert_x_argument("tsi_url", tsi_config.url) _insert_x_argument("gxy_url", gxy_config.url) + sys.argv.insert(1, "--config") + sys.argv.insert(2, ALEMBIC_CONFIG) + if "heads" in sys.argv and "upgrade" in sys.argv: i = sys.argv.index("heads") sys.argv[i] = f"{GXY}@head" From 8f87afb807aae73d0dec97655f851a546560556c Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 23 Oct 2024 14:57:15 -0400 Subject: [PATCH 02/12] Don't require galaxy.yml to start with defaults --- lib/galaxy/util/properties.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 7cf3b8710a75..72a53dabd911 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -103,7 +103,7 @@ def load_app_properties( config_section = config_section or ini_section # read from file or init w/no file - if config_file: + if config_file and os.path.exists(config_file): properties = read_properties_from_file(config_file, config_section) else: properties = {"__file__": None} @@ -208,6 +208,8 @@ def get_data_dir(properties): if data_dir is None: if running_from_source: data_dir = "./database" + elif properties["__file__"] is None: + data_dir = "./data" else: config_dir = properties.get("config_dir", os.path.dirname(properties["__file__"])) data_dir = os.path.join(config_dir, "data") From 3021508b978f9b23baeeae1b1bde0041a8f27bd9 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 23 Oct 2024 14:57:32 -0400 Subject: [PATCH 03/12] Add mercurial as a dep to app package for tool installs --- packages/app/setup.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/app/setup.cfg b/packages/app/setup.cfg index 285e9b8c7bf0..8864f0c693d7 100644 --- a/packages/app/setup.cfg +++ b/packages/app/setup.cfg @@ -57,6 +57,7 @@ install_requires = Mako Markdown MarkupSafe + mercurial packaging paramiko!=2.9.0,!=2.9.1 pebble From 0a6d7b8f91beadbabb75f4fe1c5f9db92ec4d6de Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 23 Oct 2024 15:01:32 -0400 Subject: [PATCH 04/12] Add a script for pip installing the packages in dev mode --- packages/package-dev-install.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100755 packages/package-dev-install.sh diff --git a/packages/package-dev-install.sh b/packages/package-dev-install.sh new file mode 100755 index 000000000000..19801a74d1b9 --- /dev/null +++ b/packages/package-dev-install.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# +# Install packages in development mode, for running installed Galaxy from the source +# +set -euo pipefail + +up_to="${1:-}" + +if [ -n "$up_to" -a ! -d "$up_to" ]; then + echo "ERROR: package does not exist: $up_to" + exit 1 +fi + +while read package; do + [ -n "$package" ] || continue + pushd $package + pip install -e . + popd + [ "$package" != "$up_to" ] || exit +done < packages_by_dep_dag.txt From 3b2d81d33b3ecbc856959f38ed0c6d7092f89489 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 23 Oct 2024 15:50:38 -0400 Subject: [PATCH 05/12] Add simple command line interface to launch uvicorn --- lib/galaxy/webapps/galaxy/script.py | 30 +++++++++++++++++++++++++++++ packages/web_apps/setup.cfg | 4 ++++ 2 files changed, 34 insertions(+) create mode 100644 lib/galaxy/webapps/galaxy/script.py diff --git a/lib/galaxy/webapps/galaxy/script.py b/lib/galaxy/webapps/galaxy/script.py new file mode 100644 index 000000000000..59708d945924 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/script.py @@ -0,0 +1,30 @@ +import os +from argparse import ArgumentParser + +import uvicorn + + +def main() -> None: + parser = ArgumentParser( + prog="galaxy-web", + description="Run Galaxy Web Server", + ) + parser.add_argument("--config", "-c", help="Galaxy config file") + parser.add_argument("--single-user", "-s", help="Single user mode as user SINGLE_USER") + parser.add_argument("--bind", "-b", default="localhost:8080", help="Bind to :") + parser.add_argument( + "--client-path", "-n", default="node_modules/@galaxyproject/galaxy-client", help="Path to Galaxy client" + ) + args = parser.parse_args() + env = os.environ.copy() + if args.config: + env["GALAXY_CONFIG_FILE"] = args.config + if args.single_user: + env["GALAXY_CONFIG_SINGLE_USER"] = args.single_user + env["GALAXY_CONFIG_ADMIN_USERS"] = args.single_user + uvicorn.run( + "galaxy.webapps.galaxy.fast_factory:factory", + factory=True, + host=args.bind.split(":")[0], + port=int(args.bind.split(":")[1]), + ) diff --git a/packages/web_apps/setup.cfg b/packages/web_apps/setup.cfg index 914d43a895d2..bcf88a282c43 100644 --- a/packages/web_apps/setup.cfg +++ b/packages/web_apps/setup.cfg @@ -68,6 +68,10 @@ install_requires = packages = find: python_requires = >=3.8 +[options.entry_points] +console_scripts = + galaxy-web = galaxy.webapps.galaxy.script:main + [options.packages.find] exclude = tests* From 90caa402a5cf42768c8c959ad87cf43d485e0219 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 24 Oct 2024 10:05:41 -0400 Subject: [PATCH 06/12] Disable conda_auto_init default = true when not running from source --- doc/source/admin/galaxy_options.rst | 3 ++- lib/galaxy/config/__init__.py | 3 +++ lib/galaxy/config/sample/galaxy.yml.sample | 3 ++- lib/galaxy/config/schemas/config_schema.yml | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 45386d72d861..393b54cb92a6 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -509,7 +509,8 @@ :Description: Set to true to instruct Galaxy to install Conda from the web automatically if it cannot find a local copy and conda_exec is not - configured. + configured. The default is true if running Galaxy from source, and + false if running from installed packages. :Default: ``true`` :Type: bool diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 80d57ab04bf6..812ca96fe6b9 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -982,6 +982,9 @@ def _process_config(self, kwargs: Dict[str, Any]) -> None: # self, populate config_dict self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] + if not running_from_source and kwargs.get("conda_auto_init") is None: + self.config_dict["conda_auto_init"] = False + if self.container_resolvers_config_file: self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file) diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index d6718cf176e7..dca66e799347 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -589,7 +589,8 @@ galaxy: # Set to true to instruct Galaxy to install Conda from the web # automatically if it cannot find a local copy and conda_exec is not - # configured. + # configured. The default is true if running Galaxy from source, and + # false if running from installed packages. #conda_auto_init: true # You must set this to true if conda_prefix and job_working_directory diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 3ed0c7455e74..06963d364186 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -386,7 +386,8 @@ mapping: required: false desc: | Set to true to instruct Galaxy to install Conda from the web automatically - if it cannot find a local copy and conda_exec is not configured. + if it cannot find a local copy and conda_exec is not configured. The default is + true if running Galaxy from source, and false if running from installed packages. conda_copy_dependencies: type: bool From 6b14e9d0f133409b6d0be679f4376331ad3b8f42 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 24 Oct 2024 13:16:12 -0400 Subject: [PATCH 07/12] Update package-dev-install.sh to make the editable flag optional --- packages/package-dev-install.sh | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/packages/package-dev-install.sh b/packages/package-dev-install.sh index 19801a74d1b9..b22d5460e72c 100755 --- a/packages/package-dev-install.sh +++ b/packages/package-dev-install.sh @@ -4,6 +4,18 @@ # set -euo pipefail +editable=false + +while getopts ':e' OPTION +do + case $OPTION in + e) + editable=true + ;; + esac +done +shift $((OPTIND - 1)) + up_to="${1:-}" if [ -n "$up_to" -a ! -d "$up_to" ]; then @@ -14,7 +26,11 @@ fi while read package; do [ -n "$package" ] || continue pushd $package - pip install -e . + if $editable; then + pip install -e . + else + pip install . + fi popd [ "$package" != "$up_to" ] || exit done < packages_by_dep_dag.txt From bbc8dd64b6fb5a165c079ee6ae44ac917628886c Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 24 Oct 2024 13:17:21 -0400 Subject: [PATCH 08/12] Correct mistake in galaxy-web entry point where env is not actually set --- lib/galaxy/webapps/galaxy/script.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/script.py b/lib/galaxy/webapps/galaxy/script.py index 59708d945924..4ca9d050bf7e 100644 --- a/lib/galaxy/webapps/galaxy/script.py +++ b/lib/galaxy/webapps/galaxy/script.py @@ -16,12 +16,12 @@ def main() -> None: "--client-path", "-n", default="node_modules/@galaxyproject/galaxy-client", help="Path to Galaxy client" ) args = parser.parse_args() - env = os.environ.copy() if args.config: - env["GALAXY_CONFIG_FILE"] = args.config + os.environ["GALAXY_CONFIG_FILE"] = args.config if args.single_user: - env["GALAXY_CONFIG_SINGLE_USER"] = args.single_user - env["GALAXY_CONFIG_ADMIN_USERS"] = args.single_user + os.environ["GALAXY_CONFIG_SINGLE_USER"] = args.single_user + os.environ["GALAXY_CONFIG_ADMIN_USERS"] = args.single_user + os.environ["GALAXY_CONFIG_STATIC_DIR"] = args.client_path uvicorn.run( "galaxy.webapps.galaxy.fast_factory:factory", factory=True, From 235e4ce5c33ecc72520716812d8f744e6acb6628 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Thu, 24 Oct 2024 13:17:40 -0400 Subject: [PATCH 09/12] Update static handling for welcome.html to automatically load sample if the non-sample is not present, and for installed Galaxy. Fix robots.txt and favicon which were not served at all due to middleware. --- lib/galaxy/web/framework/middleware/static.py | 4 +-- lib/galaxy/webapps/base/webapp.py | 31 ++++++++++++++----- scripts/common_startup.sh | 1 - ...elcome.html.sample => welcome.sample.html} | 0 4 files changed, 24 insertions(+), 12 deletions(-) rename static/{welcome.html.sample => welcome.sample.html} (100%) diff --git a/lib/galaxy/web/framework/middleware/static.py b/lib/galaxy/web/framework/middleware/static.py index d46ff6af592c..a5d9f17d36fe 100644 --- a/lib/galaxy/web/framework/middleware/static.py +++ b/lib/galaxy/web/framework/middleware/static.py @@ -19,9 +19,7 @@ def __init__(self, directory, cache_seconds=None, directory_per_host=None): def __call__(self, environ, start_response): path_info = environ.get("PATH_INFO", "") script_name = environ.get("SCRIPT_NAME", "") - if script_name == "/robots.txt" or script_name == "/favicon.ico": - filename = script_name.replace("/", "") - elif not path_info: + if not path_info: # See if this is a static file hackishly mapped. if os.path.exists(self.directory) and os.path.isfile(self.directory): app = FileApp(self.directory) diff --git a/lib/galaxy/webapps/base/webapp.py b/lib/galaxy/webapps/base/webapp.py index 75a14f04e301..f2be00448f20 100644 --- a/lib/galaxy/webapps/base/webapp.py +++ b/lib/galaxy/webapps/base/webapp.py @@ -1156,19 +1156,34 @@ def build_url_map(app, global_conf, **local_conf): # Send to dynamic app by default urlmap["/"] = app - def get_static_from_config(option_name, default_path): - config_val = conf.get(option_name, default_url_path(default_path)) + def get_static_from_config(option_name, default_path, sample=None): + config_val = conf.get(option_name) + default = default_url_path(default_path) + if not config_val: + if not os.path.exists(default) and sample: + config_val = os.path.abspath(f"{sample}") + else: + config_val = default per_host_config_option = f"{option_name}_by_host" per_host_config = conf.get(per_host_config_option) return Static(config_val, cache_time, directory_per_host=per_host_config) # Define static mappings from config - urlmap["/static"] = get_static_from_config("static_dir", "static/") - urlmap["/images"] = get_static_from_config("static_images_dir", "static/images") - urlmap["/static/scripts"] = get_static_from_config("static_scripts_dir", "static/scripts/") - urlmap["/static/welcome.html"] = get_static_from_config("static_welcome_html", "static/welcome.html") - urlmap["/favicon.ico"] = get_static_from_config("static_favicon_dir", "static/favicon.ico") - urlmap["/robots.txt"] = get_static_from_config("static_robots_txt", "static/robots.txt") + static_dir = get_static_from_config("static_dir", "static/") + static_dir_bare = static_dir.directory.rstrip("/") + urlmap["/static"] = static_dir + urlmap["/images"] = get_static_from_config("static_images_dir", f"{static_dir_bare}/images") + urlmap["/static/scripts"] = get_static_from_config("static_scripts_dir", f"{static_dir_bare}/scripts/") + + urlmap["/static/welcome.html"] = get_static_from_config( + "static_welcome_html", f"{static_dir_bare}/welcome.html", sample=default_url_path("static/welcome.sample.html") + ) + urlmap["/favicon.ico"] = get_static_from_config( + "static_favicon_dir", f"{static_dir_bare}/favicon.ico", sample=default_url_path("static/favicon.ico") + ) + urlmap["/robots.txt"] = get_static_from_config( + "static_robots_txt", f"{static_dir_bare}/robots.txt", sample=default_url_path("static/robots.txt") + ) if "static_local_dir" in conf: urlmap["/static_local"] = Static(conf["static_local_dir"], cache_time) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index e1fa46fdbac6..20a9b0ea29c1 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -34,7 +34,6 @@ done SAMPLES=" tool-data/shared/ucsc/builds.txt.sample tool-data/shared/ucsc/manual_builds.txt.sample - static/welcome.html.sample " RMFILES=" diff --git a/static/welcome.html.sample b/static/welcome.sample.html similarity index 100% rename from static/welcome.html.sample rename to static/welcome.sample.html From 88a90c7646d5351a7698ff5dfc97915f50eabba2 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 25 Oct 2024 09:58:57 -0400 Subject: [PATCH 10/12] Lint --- lib/galaxy/web/framework/middleware/static.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/web/framework/middleware/static.py b/lib/galaxy/web/framework/middleware/static.py index a5d9f17d36fe..03e636e186bf 100644 --- a/lib/galaxy/web/framework/middleware/static.py +++ b/lib/galaxy/web/framework/middleware/static.py @@ -18,7 +18,6 @@ def __init__(self, directory, cache_seconds=None, directory_per_host=None): def __call__(self, environ, start_response): path_info = environ.get("PATH_INFO", "") - script_name = environ.get("SCRIPT_NAME", "") if not path_info: # See if this is a static file hackishly mapped. if os.path.exists(self.directory) and os.path.isfile(self.directory): From d622d2d8f14c845aff0a23200b273b84a112d409 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 28 Oct 2024 14:50:01 -0400 Subject: [PATCH 11/12] Remove conda_auto_init default from schema as suggested --- doc/source/admin/galaxy_options.rst | 4 ++-- lib/galaxy/config/sample/galaxy.yml.sample | 4 ++-- lib/galaxy/config/schemas/config_schema.yml | 1 - 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 393b54cb92a6..c29b7952ff04 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -511,7 +511,7 @@ automatically if it cannot find a local copy and conda_exec is not configured. The default is true if running Galaxy from source, and false if running from installed packages. -:Default: ``true`` +:Default: ``None`` :Type: bool @@ -5125,7 +5125,7 @@ 'deferred' are "materialized" (or undeferred) by the workflow scheduler. This might be a lengthy process. Setting this to 'True' will place the invocation back in the queue after materialization - before scheduling the workflow so it is less likely to starve + before scheduling the workflow so it is less likely to starve other workflow scheduling. Ideally, Galaxy would allow more fine grain control of handlers but until then, this provides a way to tip the balance between "doing more work" and "being more fair". diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index dca66e799347..394189c6d68f 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -591,7 +591,7 @@ galaxy: # automatically if it cannot find a local copy and conda_exec is not # configured. The default is true if running Galaxy from source, and # false if running from installed packages. - #conda_auto_init: true + #conda_auto_init: false # You must set this to true if conda_prefix and job_working_directory # are not on the same volume, or some conda dependencies will fail to @@ -2753,7 +2753,7 @@ galaxy: # 'deferred' are "materialized" (or undeferred) by the workflow # scheduler. This might be a lengthy process. Setting this to 'True' # will place the invocation back in the queue after materialization - # before scheduling the workflow so it is less likely to starve other + # before scheduling the workflow so it is less likely to starve other # workflow scheduling. Ideally, Galaxy would allow more fine grain # control of handlers but until then, this provides a way to tip the # balance between "doing more work" and "being more fair". The default diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 06963d364186..f0ee832e34ec 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -382,7 +382,6 @@ mapping: conda_auto_init: type: bool - default: true required: false desc: | Set to true to instruct Galaxy to install Conda from the web automatically From 17e228f333bc009f0ee68e16555783a1c3805ca5 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Mon, 28 Oct 2024 14:51:08 -0400 Subject: [PATCH 12/12] Update lib/galaxy/config/__init__.py Co-authored-by: Nicola Soranzo --- lib/galaxy/config/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 812ca96fe6b9..d4b1170673a1 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -982,8 +982,8 @@ def _process_config(self, kwargs: Dict[str, Any]) -> None: # self, populate config_dict self.config_dict["conda_mapping_files"] = [self.local_conda_mapping_file, _default_mapping] - if not running_from_source and kwargs.get("conda_auto_init") is None: - self.config_dict["conda_auto_init"] = False + if kwargs.get("conda_auto_init") is None: + self.config_dict["conda_auto_init"] = running_from_source if self.container_resolvers_config_file: self.container_resolvers_config_file = self._in_config_dir(self.container_resolvers_config_file)