WIP Konflux mode internal hermetic, prefetch

doozer/doozerlib/backend/konflux_client.py

@@ -14,6 +14,8 @@
 
 from doozerlib import constants
 
+from doozerlib.image import ImageMetadata
+
 yaml = YAML(typ="safe")
 LOGGER = logging.getLogger(__name__)
 
@@ -305,7 +307,7 @@ async def _get_pipelinerun_template(self, template_url: str):
 
     async def _new_pipelinerun_for_image_build(self, generate_name: str, namespace: Optional[str], application_name: str, component_name: str,
                                                git_url: str, commit_sha: str, target_branch: str, output_image: str,
-                                               build_platforms: Sequence[str], git_auth_secret: str = "pipelines-as-code-secret",
+                                               build_platforms: Sequence[str], image_metadata: ImageMetadata, git_auth_secret: str = "pipelines-as-code-secret",
                                                additional_tags: Optional[Sequence[str]] = None, skip_checks: bool = False,
                                                pipelinerun_template_url: str = constants.KONFLUX_DEFAULT_IMAGE_BUILD_PLR_TEMPLATE_URL) -> dict:
         if additional_tags is None:
@@ -359,6 +361,58 @@ def _modify_param(params: List, name: str, value: Union[str, bool, list[str]]):
         _modify_param(params, "image-expires-after", "6w")
         _modify_param(params, "build-platforms", list(build_platforms))
 
+        prefetch_params = []
+        package_managers = image_metadata.config.get("content", {}).get("source", {}).get("pkg_managers", [])
+        if "pip" in package_managers:
+            pip_configs = image_metadata.config.get("cachito", {}).get("packages", {}).get("pip", [])
+            if pip_configs:
+                for entry in pip_configs:
+                    pip_override = {"type": "pip"}
+
+                    pip_path = entry.get("path")
+                    if pip_path:
+                        pip_override["path"] = pip_path
+
+                    pip_requirements_files = entry.get("requirements_files", [])
+                    pip_requirements_build_files = entry.get("requirements_build_files", [])
+                    if pip_requirements_files or pip_requirements_build_files:
+                        pip_override["requirements_files"] = pip_requirements_files + pip_requirements_build_files
+                    prefetch_params.append(pip_override)
+            else:
+                prefetch_params.append({"type": "pip", "path": "."})
+
+        if "gomod" in package_managers:
+            gomod_configs = image_metadata.config.get("cachito", {}).get("packages", {}).get("gomod", [])
+            if gomod_configs:
+                for entry in gomod_configs:
+                    gomod_override = {"type": "gomod"}
+
+                    gomod_path = entry.get("path")
+                    if gomod_path:
+                        gomod_override["path"] = gomod_path
+
+                    prefetch_params.append({"type": "gomod", "path": gomod_path})
+            else:
+                prefetch_params.append({"type": "gomod", "path": "."})
+
+        if "npm" in package_managers:
+            npm_configs = image_metadata.config.get("cachito", {}).get("packages", {}).get("npm", [])
+            if npm_configs:
+                for entry in npm_configs:
+                    npm_override = {"type": "npm"}
+
+                    npm_path = entry.get("path")
+                    if npm_path:
+                        npm_override["path"] = npm_path
+
+                    prefetch_params.append({"type": "npm", "path": npm_path})
+            else:
+                prefetch_params.append({"type": "npm", "path": "."})
+
+        cachito_enabled = image_metadata.config.get("cachito", {}).get("enabled", False)
+        if cachito_enabled and prefetch_params:
+            _modify_param(params, "prefetch-input", prefetch_params)
+
         # See https://konflux-ci.dev/docs/how-tos/configuring/customizing-the-build/#configuring-timeouts
         obj["spec"]["timeouts"] = {"pipeline": "12h"}
 
@@ -408,6 +462,7 @@ async def start_pipeline_run_for_image_build(
         additional_tags: Sequence[str] = [],
         skip_checks: bool = False,
         pipelinerun_template_url: str = constants.KONFLUX_DEFAULT_IMAGE_BUILD_PLR_TEMPLATE_URL,
+        image_metadata: ImageMetadata = None
     ):
         """
         Start a PipelineRun for building an image.
@@ -425,6 +480,7 @@ async def start_pipeline_run_for_image_build(
         :param git_auth_secret: The git auth secret.
         :param additional_tags: Additional tags to apply to the image.
         :param skip_checks: Whether to skip checks.
+        :param image_metadata: Image metadata
         :return: The PipelineRun resource.
         """
         unsupported_arches = set(building_arches) - set(self.SUPPORTED_ARCHES)
@@ -448,6 +504,7 @@ async def start_pipeline_run_for_image_build(
             skip_checks=skip_checks,
             additional_tags=additional_tags,
             pipelinerun_template_url=pipelinerun_template_url,
+            image_metadata=image_metadata
         )
         if self.dry_run:
             fake_pipelinerun = resource.ResourceInstance(self.dyn_client, pipelinerun_manifest)

doozer/doozerlib/backend/konflux_image_builder.py

@@ -213,6 +213,7 @@ async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, bui
             skip_checks=self._config.skip_checks,
             vm_override=metadata.config.get("konflux", {}).get("vm_override"),
             pipelinerun_template_url=self._config.plr_template,
+            image_metadata=metadata
         )
 
         logger.info(f"Created PipelineRun: {self.build_pipeline_url(pipelinerun)}")

doozer/doozerlib/backend/rebaser.py

@@ -757,20 +757,32 @@ def _update_dockerfile(self, metadata: ImageMetadata, source: Optional[SourceRes
         self._update_environment_variables(metadata, source, df_path, build_update_envs=build_update_env_vars, metadata_envs=metadata_envs)
 
         # Inject build repos for Konflux
-        self._add_build_repos(dfp)
+        self._add_build_repos(dfp, metadata)
 
         self._modify_cachito_commands(metadata, df_path)
 
         self._reflow_labels(df_path)
 
-    def _add_build_repos(self, dfp: DockerfileParser):
+    def _add_build_repos(self, dfp: DockerfileParser, metadata: ImageMetadata):
         # Populating the repo file needs to happen after every FROM before the original Dockerfile can invoke yum/dnf.
+        konflux_lines = ["\n# Start Konflux-specific steps",
+                         "RUN mkdir -p /tmp/yum_temp; mv /etc/yum.repos.d/*.repo /tmp/yum_temp/ || true",
+                         f"COPY .oit/{self.repo_type}.repo /etc/yum.repos.d/",
+                         f"ADD {constants.KONFLUX_REPO_CA_BUNDLE_HOST}/{constants.KONFLUX_REPO_CA_BUNDLE_FILENAME} {constants.KONFLUX_REPO_CA_BUNDLE_TMP_PATH}"]
+
+        network_mode = metadata.config.get("konflux", {}).get("network_mode")
+
+        if network_mode == "internal-hermetic":
+            konflux_lines += [
+                "ENV NO_PROXY='localhost,127.0.0.1,::1,.redhat.com'",
+                "ENV HTTP_PROXY='http://127.0.0.1:9999'",
+                "ENV HTTPS_PROXY='http://127.0.0.1:9999'",
+            ]
+
+        konflux_lines += ["# End Konflux-specific steps\n\n"]
+
         dfp.add_lines(
-            "\n# Start Konflux-specific steps",
-            "RUN mkdir -p /tmp/yum_temp; mv /etc/yum.repos.d/*.repo /tmp/yum_temp/ || true",
-            f"COPY .oit/{self.repo_type}.repo /etc/yum.repos.d/",
-            f"ADD {constants.KONFLUX_REPO_CA_BUNDLE_HOST}/{constants.KONFLUX_REPO_CA_BUNDLE_FILENAME} {constants.KONFLUX_REPO_CA_BUNDLE_TMP_PATH}",
-            "# End Konflux-specific steps\n\n",
+            *konflux_lines,
             at_start=True,
             all_stages=True,
         )