Support for builds on mac+podman (#22)

* applying changes to build the dev image for mac

* applying changes to build the dev image for mac

* Updating Dockerfile to compile elyra on mac

* implementing changes for the build process to work on a mac + podman

* implementing changes for the build process to work on a mac + podman

* Fixing dockerfile to work on mac

* implementing changes for the build process to work on a mac + podman

* Fixing dockerfile to work on mac

* fixing linter

* Upgrading to Node 20

* Undoing yarn.lock change

---------

Co-authored-by: Eder Ignatowicz <ignatowicz@gmail.com>

elyra/kfp/bootstrapper.py

@@ -472,7 +472,7 @@ def execute(self) -> None:
         """Execute the Python script and upload results to object storage"""
         python_script = os.path.basename(self.filepath)
         python_script_name = python_script.replace(".py", "")
-        # python_script_output = f"{python_script_name}.log"
+        python_script_output = f"{python_script_name}.log"
 
         try:
             OpUtil.log_operation_info(
@@ -483,16 +483,16 @@ def execute(self) -> None:
             run_args = ["python3", python_script]
             if self.parameter_pass_method == "env":
                 self.set_parameters_in_env()
-            
+
             logger.info("----------------------Python logs start----------------------")
             # Removing support for the s3 storage of python script logs
             # with open(python_script_output, "w") as log_file:
             #     process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
             process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
-            for line in iter(process.stdout.readline, b''):
+            for line in iter(process.stdout.readline, b""):
                 sys.stdout.write(line.decode())
-                
+
             process.stdout.close()
             return_code = process.wait()
             logger.info("----------------------Python logs ends----------------------")
@@ -519,7 +519,7 @@ def execute(self) -> None:
         """Execute the R script and upload results to object storage"""
         r_script = os.path.basename(self.filepath)
         r_script_name = r_script.replace(".r", "")
-        # r_script_output = f"{r_script_name}.log"
+        r_script_output = f"{r_script_name}.log"
 
         try:
             OpUtil.log_operation_info(f"executing R script using 'Rscript {r_script}' to '{r_script_output}'")
@@ -535,9 +535,9 @@ def execute(self) -> None:
             #     process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
             process = subprocess.Popen(run_args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 
-            for line in iter(process.stdout.readline, b''):
+            for line in iter(process.stdout.readline, b""):
                 sys.stdout.write(line.decode())
-                
+
             process.stdout.close()
             return_code = process.wait()
             logger.info("----------------------R script logs ends----------------------")

elyra/pipeline/kfp/processor_kfp.py

@@ -43,7 +43,7 @@
 from traitlets import default
 from traitlets import Unicode
 
-RUN_ID_PLACEHOLDER = '{{workflow.uid}}'
+RUN_ID_PLACEHOLDER = "{{workflow.uid}}"
 
 from elyra._version import __version__
 from elyra.metadata.schemaspaces import RuntimeImages
@@ -74,6 +74,7 @@
 from elyra.util.kubernetes import sanitize_label_value
 from elyra.util.path import get_absolute_path
 
+
 @unique
 class WorkflowEngineType(Enum):
     """
@@ -1057,7 +1058,8 @@ def _compose_container_command_args(
         account whether the container will run in a CRI-O environment.
         """
 
-        # NOTE: The default organization and default branch are meant for the opendatahub-io/Data-science-pipelines v2 use case 
+        # NOTE: The default organization and default branch are
+        # meant for the opendatahub-io/Data-science-pipelines v2 use case
         # and should not be opened as PR against upstream Elyra
         elyra_github_org = os.getenv("ELYRA_GITHUB_ORG", "opendatahub-io")
         elyra_github_branch = os.getenv("ELYRA_GITHUB_BRANCH", "dspv2" if "dev" in __version__ else __version__)

elyra/pipeline/processor.py

@@ -431,8 +431,8 @@ def _upload_dependencies_to_object_store(
             raise ex from ex
 
     def _verify_cos_connectivity(self, runtime_configuration) -> None:
-        if not runtime_configuration.metadata['cos_endpoint'].startswith("https://"):
-            runtime_configuration.metadata['cos_endpoint'] = "https://" + runtime_configuration.metadata['cos_endpoint']
+        if not runtime_configuration.metadata["cos_endpoint"].startswith("https://"):
+            runtime_configuration.metadata["cos_endpoint"] = "https://" + runtime_configuration.metadata["cos_endpoint"]
         self.log.debug(
             "Verifying cloud storage connectivity using runtime configuration "
             f"'{runtime_configuration.display_name}'."

elyra/tests/metadata/test_metadata.py

@@ -704,7 +704,7 @@ def test_validation_performance():
     print(
         f"\nMemory: {diff:,} kb, Start: {memory_start.rss / 1024 / 1024:,.3f} mb, "
         f"End: {memory_end.rss / 1024 / 1024:,.3f} mb., "
-        f"Elapsed time: {t1-t0:.3f}s over {iterations} iterations."
+        f"Elapsed time: {t1 - t0:.3f}s over {iterations} iterations."
     )
 
 

elyra/tests/util/test_kubernetes.py

@@ -87,7 +87,7 @@ def test_is_valid_label_key_invalid_input():
     assert not is_valid_label_key(key="/n")  # prefix too short
     assert not is_valid_label_key(key="p/")  # name too short
     assert not is_valid_label_key(key="a" * 254)  # name too long
-    assert not is_valid_label_key(key=f"d/{'b'*64}")  # name too long
+    assert not is_valid_label_key(key=f"d / {'b' * 64}")  # name too long
     # test first character violations (not alphanum)
     assert not is_valid_label_key(key="-a")
     assert not is_valid_label_key(key=".b")
@@ -116,7 +116,7 @@ def test_is_valid_label_key_valid_input():
     assert is_valid_label_key(key="p/n")
     assert is_valid_label_key(key="prefix/you.2")
     assert is_valid_label_key(key="how.sad/to-see")
-    assert is_valid_label_key(key=f"{'d'*253}/{'n'*63}")
+    assert is_valid_label_key(key=f"{'d' * 253} / {'n' * 63}")
 
 
 def test_is_valid_label_value_invalid_input():
@@ -175,7 +175,7 @@ def test_is_valid_annotation_key_invalid_input():
     assert not is_valid_annotation_key(key="/n")  # prefix too short
     assert not is_valid_annotation_key(key="p/")  # name too short
     assert not is_valid_annotation_key(key="a" * 254)  # name too long
-    assert not is_valid_annotation_key(key=f"d/{'b'*64}")  # name too long
+    assert not is_valid_annotation_key(key=f"d / {'b' * 64}")  # name too long
     # test first character violations (not alphanum)
     assert not is_valid_annotation_key(key="-a")
     assert not is_valid_annotation_key(key=".b")
@@ -204,7 +204,7 @@ def test_is_valid_annotation_key_valid_input():
     assert is_valid_annotation_key(key="p/n")
     assert is_valid_annotation_key(key="prefix/you.2")
     assert is_valid_annotation_key(key="how.sad/to-see")
-    assert is_valid_annotation_key(key=f"{'d'*253}/{'n'*63}")
+    assert is_valid_annotation_key(key=f"{'d' * 253}/{'n' * 63}")
 
 
 def test_is_valid_annotation_value_invalid_input():

etc/docker/elyra_development/Dockerfile

@@ -1,36 +1,63 @@
-FROM ubuntu:20.04
+FROM ubuntu:latest
+
+ARG TARGETARCH
+
+ENV NODE_VERSION 20
+ENV TINI_VERSION v0.19.0
+
+RUN echo ">> Building for TARGETARCH=${TARGETARCH}, NODE_VERSION=${NODE_VERSION}, TINI_VERSION=${TINI_VERSION}"
 
 ENV LANG=C.UTF-8 LC_ALL=C.UTF-8
 ENV PATH /opt/conda/bin:$PATH
 
-RUN apt-get update --fix-missing && \
-    apt-get install -y make wget bzip2 ca-certificates curl git && \
-    apt-get clean && \
+RUN <<EOF
+    apt-get update --fix-missing 
+    apt-get install -y make wget bzip2 ca-certificates curl git 
+    apt-get clean 
     rm -rf /var/lib/apt/lists/*
+EOF
 
 COPY requirements.yml requirements.yml
 
-RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-4.5.11-Linux-x86_64.sh -O ~/miniconda.sh && \
-    /bin/bash ~/miniconda.sh -b -p /opt/conda && \
-    rm ~/miniconda.sh && \
-    /opt/conda/bin/conda clean -tipsy && \
-    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
-    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
-    echo "conda activate elyra-env" >> ~/.bashrc && \
+RUN /bin/bash <<EOF
+    if [ "${TARGETARCH}" == "arm64" ]; then 
+        CONDA_ARCH="aarch64"
+    else
+        CONDA_ARCH=${TARGETARCH:-"x86_64"}
+    fi
+    # CONDA_ARCH expansion needs to be escaped to avoid docker expanding the variable before calling the shell
+    echo "wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-\${CONDA_ARCH}.sh -O ~/miniconda.sh"
+    wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-py310_24.5.0-0-Linux-\${CONDA_ARCH}.sh -O ~/miniconda.sh 
+    /bin/bash ~/miniconda.sh -b -p /opt/conda 
+    rm ~/miniconda.sh 
+    /opt/conda/bin/conda clean -tipy 
+    ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh 
+    echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc 
+    echo "conda activate elyra-env" >> ~/.bashrc 
     /opt/conda/bin/conda env create --file requirements.yml
+EOF
 
 # update nodejs
-RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+RUN curl -fsSL https://deb.nodesource.com/setup_${NODE_VERSION}.x | bash - && \
     apt-get install -y nodejs
 
 # install yarn
 RUN curl -sL https://dl.yarnpkg.com/debian/pubkey.gpg | gpg --dearmor | tee /usr/share/keyrings/yarnkey.gpg >/dev/null && \
     echo "deb [signed-by=/usr/share/keyrings/yarnkey.gpg] https://dl.yarnpkg.com/debian stable main" | tee /etc/apt/sources.list.d/yarn.list && \
-    apt-get update && apt-get install yarn
+    apt-get update && apt-get install yarn && apt-get install -y build-essential
 
-ENV TINI_VERSION v0.16.1
-ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini /usr/bin/tini
-RUN chmod +x /usr/bin/tini
+# add tini
+RUN /bin/bash <<EOF 
+    if [ "${TARGETARCH}" == "x86_64" ]; then 
+        TINI_FILE=tini
+    else
+        TINI_FILE=tini-${TARGETARCH} 
+    fi
+    # TINI_FILE expansion needs to be escaped to avoid docker expanding the variable before calling the shell
+    echo "wget --quiet https://github.com/krallin/tini/releases/download/${TINI_VERSION}/\${TINI_FILE} -O /usr/bin/tini"
+    wget --quiet https://github.com/krallin/tini/releases/download/${TINI_VERSION}/\${TINI_FILE} -O /usr/bin/tini   
+    chmod +x /usr/bin/tini
+EOF
 
 RUN mkdir -p /dev/elyra