From 7223f69e599b72cbe861d6bcf5ce57e10fe280ba Mon Sep 17 00:00:00 2001 From: Harshad Reddy Nalla Date: Wed, 25 Sep 2024 19:27:36 -0400 Subject: [PATCH] set ELYRA_FILE_BASE_PATH var for using the script embedded in runtime images Signed-off-by: Harshad Reddy Nalla --- ...running-elyra-in-air-gapped-environment.md | 5 ++++- elyra/pipeline/kfp/processor_kfp.py | 21 ++++++++++++------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/docs/source/recipes/running-elyra-in-air-gapped-environment.md b/docs/source/recipes/running-elyra-in-air-gapped-environment.md index 5be08150b..5e2b299ec 100644 --- a/docs/source/recipes/running-elyra-in-air-gapped-environment.md +++ b/docs/source/recipes/running-elyra-in-air-gapped-environment.md @@ -62,5 +62,8 @@ During pipeline execution in the Kubeflow Pipelines or Apache Airflow environmen - `ELYRA_REQUIREMENTS_URL` (URL of `.../etc/generic/requirements-elyra.txt`) - For Apache Airflow: - `ELYRA_BOOTSTRAP_SCRIPT_URL` (URL of `.../elyra/airflow/bootstrapper.py`) - - `ELYRA_REQUIREMENTS_URL` (URL of `.../etc/generic/requirements-elyra.txt`) + - `ELYRA_REQUIREMENTS_URL` (URL of `.../etc/generic/requirements-elyra.txt`) + + Alternatively, for Kubeflow Pipeline users can store the bootstrapper.py, pip.conf and requirements-elyra.txt files in the runtime container itself, so it could used without having to set the above variables.The Path of these files needs to be configured, so elyra execution can pick it up. + - `ELYRA_FILE_BASE_PATH` (default `/opt/app-root/bin/utils`) - **S3-compatible cloud storage for [generic components](../user_guide/pipeline-components.html#generic-components)**: When processing pipeline nodes that are implemented using [generic components](../user_guide/pipeline-components.html#generic-components), Elyra downloads the pipeline artifacts that were uploaded when the pipeline was exported or submitted. diff --git a/elyra/pipeline/kfp/processor_kfp.py b/elyra/pipeline/kfp/processor_kfp.py index 9133c09ec..2942c4404 100644 --- a/elyra/pipeline/kfp/processor_kfp.py +++ b/elyra/pipeline/kfp/processor_kfp.py @@ -1088,19 +1088,24 @@ def _compose_container_command_args( common_curl_options = "--fail -H 'Cache-Control: no-cache'" + # Verify utils files are present in the container + # if not, then download them as per the provided URLs. + file_base=os.getenv("ELYRA_FILE_BASE_PATH",f"/opt/app-root/bin/utils") + command_args = [ - f"mkdir -p {container_work_dir} && cd {container_work_dir}", - f"echo 'Downloading {elyra_bootstrap_script_url}' && " - f"curl {common_curl_options} -L {elyra_bootstrap_script_url} --output bootstrapper.py", - f"echo 'Downloading {elyra_requirements_url}' && " - f"curl {common_curl_options} -L {elyra_requirements_url} --output requirements-elyra.txt", - ] + f"mkdir -p {container_work_dir} && cd {container_work_dir}", + f"[[ -e '{file_base}/bootstrapper.py' ]] && (echo 'bootstrapper.py file already exists'; cp '{file_base}/bootstrapper.py' .) || " + f"(echo 'Downloading {elyra_bootstrap_script_url}'; curl {common_curl_options} -L '{elyra_bootstrap_script_url}' --output bootstrapper.py)", + f"[[ -e '{file_base}/requirements-elyra.txt' ]] && (echo 'requirements-elyra.txt file already exists'; cp '{file_base}/requirements-elyra.txt' .) || " + f"(echo 'Downloading {elyra_requirements_url}'; curl {common_curl_options} -L '{elyra_requirements_url}' --output requirements-elyra.txt)", + ] if is_crio_runtime: command_args.append( f"mkdir {container_python_path} && cd {container_python_path} && " - f"echo 'Downloading {python_pip_config_url}' && " - f"curl {common_curl_options} -L {python_pip_config_url} --output pip.conf && cd .. && " + f"[[ -e '{file_base}/pip.conf' ]] && (echo 'pip.conf file already exists'; cp $file_base/pip.conf .) ||" + f"(echo 'Downloading {python_pip_config_url}'; curl {common_curl_options} -L {python_pip_config_url} --output pip.conf)", + f"&& cd .. && " ) bootstrapper_command = [