diff --git a/doozer/doozerlib/backend/konflux_image_builder.py b/doozer/doozerlib/backend/konflux_image_builder.py index a7c2dfa8c..d143c9c3e 100644 --- a/doozer/doozerlib/backend/konflux_image_builder.py +++ b/doozer/doozerlib/backend/konflux_image_builder.py @@ -5,12 +5,10 @@ from dataclasses import dataclass from datetime import datetime, timezone from pathlib import Path -from typing import Optional, cast +from typing import Dict, Optional, cast from artcommonlib import exectools from artcommonlib.arch_util import go_arch_for_brew_arch -from packageurl import PackageURL - from artcommonlib.exectools import limit_concurrency from artcommonlib.konflux.konflux_build_record import (ArtifactType, Engine, KonfluxBuildOutcome, @@ -18,11 +16,13 @@ from artcommonlib.release_util import isolate_el_version_in_release from dockerfile_parse import DockerfileParser from kubernetes.dynamic import resource +from packageurl import PackageURL from doozerlib import constants from doozerlib.backend.build_repo import BuildRepo from doozerlib.backend.konflux_client import KonfluxClient from doozerlib.image import ImageMetadata +from doozerlib.record_logger import RecordLogger from doozerlib.source_resolver import SourceResolution LOGGER = logging.getLogger(__name__) @@ -54,16 +54,27 @@ class KonfluxImageBuilder: # https://gitlab.cee.redhat.com/konflux/docs/users/-/blob/main/topics/getting-started/multi-platform-builds.md SUPPORTED_ARCHES = { - # Only x86_64 is supported, until we are on the new cluster "x86_64": "linux/x86_64", "s390x": "linux/s390x", "ppc64le": "linux/ppc64le", "aarch64": "linux/arm64", } - def __init__(self, config: KonfluxImageBuilderConfig, logger: Optional[logging.Logger] = None) -> None: + def __init__( + self, + config: KonfluxImageBuilderConfig, + logger: Optional[logging.Logger] = None, + record_logger: Optional[RecordLogger] = None, + ): + """ Initialize the KonfluxImageBuilder. + + :param config: Options for the KonfluxImageBuilder. + :param logger: Logger to use for logging. Defaults to the module logger. + :param record_logger: Logger to use for logging build records. If None, no build records will be logged. + """ self._config = config self._logger = logger or LOGGER + self._record_logger = record_logger self._konflux_client = KonfluxClient.from_kubeconfig(default_namespace=config.namespace, config_file=config.kubeconfig, context=config.context, dry_run=config.dry_run) if self._config.image_repo == constants.KONFLUX_DEFAULT_IMAGE_REPO: @@ -76,8 +87,20 @@ async def build(self, metadata: ImageMetadata): """ Build a container image with Konflux. """ logger = self._logger.getChild(f"[{metadata.distgit_key}]") metadata.build_status = False + dest_dir = self._config.base_dir.joinpath(metadata.qualified_key) + df_path = dest_dir.joinpath("Dockerfile") + record = { + "dir": str(dest_dir.absolute()), + "dockerfile": str(df_path.absolute()), + "name": metadata.distgit_key, + "nvrs": "n/a", + "message": "Unknown failure", + "task_id": "n/a", + "task_url": "n/a", + "status": -1, # Status defaults to failure until explicitly set by success. This handles raised exceptions. + "has_olm_bundle": metadata.is_olm_operator, + } try: - dest_dir = self._config.base_dir.joinpath(metadata.qualified_key) if dest_dir.exists(): # Load exiting build source repository build_repo = await BuildRepo.from_local_dir(dest_dir, logger) @@ -97,6 +120,14 @@ async def build(self, metadata: ImageMetadata): build_repo = BuildRepo(url=source.url, branch=dest_branch, local_dir=dest_dir, logger=logger) await build_repo.ensure_source() + # Parse Dockerfile + uuid_tag, version, release = self._parse_dockerfile(metadata.distgit_key, df_path) + record["nvrs"] = f"{metadata.distgit_key}-{version}-{release}" + output_image = f"{self._config.image_repo}:{uuid_tag}" + additional_tags = [ + f"{metadata.image_name_short}-{version}-{release}" + ] + # Wait for parent members to be built parent_members = await self._wait_for_parent_members(metadata) failed_parents = [parent_member.distgit_key for parent_member in parent_members if parent_member is not None and not parent_member.build_status] @@ -110,10 +141,12 @@ async def build(self, metadata: ImageMetadata): error = None for attempt in range(retries): logger.info("Build attempt %s/%s", attempt + 1, retries) - pipelinerun = await self._start_build(metadata, build_repo, building_arches) + pipelinerun = await self._start_build(metadata, build_repo, building_arches, output_image, additional_tags) + pipelinerun_name = pipelinerun['metadata']['name'] + record["task_id"] = pipelinerun_name + record["task_url"] = self._konflux_client.build_pipeline_url(pipelinerun) await self.update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.PENDING, building_arches) - pipelinerun_name = pipelinerun['metadata']['name'] logger.info("Waiting for PipelineRun %s to complete...", pipelinerun_name) pipelinerun = await self._konflux_client.wait_for_pipelinerun(pipelinerun_name, namespace=self._config.namespace) logger.info("PipelineRun %s completed", pipelinerun_name) @@ -130,13 +163,38 @@ async def build(self, metadata: ImageMetadata): pipelinerun_name, pipelinerun) else: metadata.build_status = True + record["message"] = "Success" + record["status"] = 0 break if not metadata.build_status and error: + record["message"] = str(error) raise error finally: + if self._record_logger: + self._record_logger.add_record("image_build_konflux", **record) metadata.build_event.set() return pipelinerun_name, pipelinerun + def _parse_dockerfile(self, distgit_key: str, df_path: Path): + """ Parse the Dockerfile and return the UUID tag, version, and release. + + :param distgit_key: The distgit key of the image. + :param df_path: The path to the Dockerfile. + :return: A tuple containing the UUID tag, version, and release. + :raises ValueError: If the Dockerfile is missing the required environment variables or labels. + """ + df = DockerfileParser(str(df_path)) + uuid_tag = df.envs.get("__doozer_uuid_tag") + if not uuid_tag: + raise ValueError(f"[{distgit_key}] Dockerfile must have a '__doozer_uuid_tag' environment variable; Did you forget to run 'doozer beta:images:konflux:rebase' first?") + version = df.labels.get("version") + if not version: + raise ValueError(f"[{distgit_key}] Dockerfile must have a 'version' label.") + release = df.labels.get("release") + if not release: + raise ValueError(f"[{distgit_key}] Dockerfile must have a 'release' label.") + return uuid_tag, version, release + async def _wait_for_parent_members(self, metadata: ImageMetadata): # If this image is FROM another group member, we need to wait on that group member to be built logger = self._logger.getChild(f"[{metadata.distgit_key}]") @@ -151,7 +209,8 @@ async def _wait_for_parent_members(self, metadata: ImageMetadata): await asyncio.sleep(20) # check every 20 seconds return parent_members - async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, building_arches: list): + async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, building_arches: list[str], + output_image: str, additional_tags: list[str]): logger = self._logger.getChild(f"[{metadata.distgit_key}]") if not build_repo.commit_hash: raise IOError(f"The build branch {build_repo.branch} doesn't have any commits in the build repository {build_repo.https_url}") @@ -160,11 +219,6 @@ async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, bui git_url = build_repo.https_url git_commit = build_repo.commit_hash - df_path = build_repo.local_dir.joinpath("Dockerfile") - df = DockerfileParser(str(df_path)) - if "__doozer_uuid_tag" not in df.envs: - raise ValueError(f"[{metadata.distgit_key}] Dockerfile must have a '__doozer_uuid_tag' environment variable; Did you forget to run 'doozer beta:images:konflux:rebase' first?") - # Ensure the Application resource exists app_name = self._config.group_name.replace(".", "-") logger.info(f"Using application: {app_name}") @@ -178,22 +232,13 @@ async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, bui # A component resource name must start with a lower case letter and must be no more than 63 characters long. component_name = f"ose-{component_name.removeprefix('openshift-')}" - dest_image_repo = self._config.image_repo - dest_image_tag = df.envs["__doozer_uuid_tag"] - version = df.labels.get("version") - release = df.labels.get("release") - if not version or not release: - raise ValueError(f"[{metadata.distgit_key}] `version` and `release` labels are required.") - additional_tags = [ - f"{metadata.image_name_short}-{version}-{release}" - ] default_revision = f"art-{self._config.group_name}-assembly-test-dgk-{metadata.distgit_key}" logger.info(f"Using component: {component_name}") await self._konflux_client.ensure_component( name=component_name, application=app_name, component_name=component_name, - image_repo=dest_image_repo, + image_repo=output_image.split(":")[0], source_url=git_url, revision=default_revision, ) @@ -207,7 +252,7 @@ async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, bui git_url=git_url, commit_sha=git_commit, target_branch=git_branch, - output_image=f"{dest_image_repo}:{dest_image_tag}", + output_image=output_image, building_arches=building_arches, additional_tags=additional_tags, skip_checks=self._config.skip_checks, @@ -215,12 +260,9 @@ async def _start_build(self, metadata: ImageMetadata, build_repo: BuildRepo, bui pipelinerun_template_url=self._config.plr_template, ) - logger.info(f"Created PipelineRun: {self.build_pipeline_url(pipelinerun)}") + logger.info(f"Created PipelineRun: {self._konflux_client.build_pipeline_url(pipelinerun)}") return pipelinerun - def build_pipeline_url(self, pipelinerun): - return self._konflux_client.build_pipeline_url(pipelinerun) - async def get_installed_packages(self, image_pullspec, arches, logger) -> list: """ Example sbom: https://gist.github.com/thegreyd/6718f4e4dae9253310c03b5d492fab68 @@ -301,7 +343,7 @@ async def update_konflux_db(self, metadata, build_repo, pipelinerun, outcome, bu nvr = "-".join([component_name, version, release]) pipelinerun_name = pipelinerun['metadata']['name'] - build_pipeline_url = self.build_pipeline_url(pipelinerun) + build_pipeline_url = self._konflux_client.build_pipeline_url(pipelinerun) build_record_params = { 'name': metadata.distgit_key, diff --git a/doozer/doozerlib/backend/konflux_olm_bundler.py b/doozer/doozerlib/backend/konflux_olm_bundler.py index 1a9979716..6beed5bbb 100644 --- a/doozer/doozerlib/backend/konflux_olm_bundler.py +++ b/doozer/doozerlib/backend/konflux_olm_bundler.py @@ -22,6 +22,7 @@ from doozerlib.backend.build_repo import BuildRepo from doozerlib.backend.konflux_client import KonfluxClient, resource from doozerlib.image import ImageMetadata +from doozerlib.record_logger import RecordLogger from doozerlib.source_resolver import SourceResolution, SourceResolver _LOGGER = logging.getLogger(__name__) @@ -121,12 +122,11 @@ async def _rebase_dir(self, metadata: ImageMetadata, operator_dir: Path, bundle_ # Read the operator's Dockerfile operator_df = DockerfileParser(str(operator_dir.joinpath('Dockerfile'))) - operator_component = operator_df.labels.get('com.redhat.component') operator_version = operator_df.labels.get('version') operator_release = operator_df.labels.get('release') - if not operator_component or not operator_version or not operator_release: - raise ValueError(f"[{metadata.distgit_key}] Label 'com.redhat.component', 'version' or 'release' is not set in the operator's Dockerfile") - operator_nvr = f"{operator_component}-{operator_version}-{operator_release}" + if not operator_version or not operator_release: + raise ValueError(f"[{metadata.distgit_key}] Label 'version' or 'release' is not set in the operator's Dockerfile") + operator_nvr = f"{metadata.distgit_key}-{operator_version}-{operator_release}" # Get operator package name and channel from its package YAML # This info will be used to generate bundle's Dockerfile labels and metadata/annotations.yaml @@ -256,10 +256,11 @@ async def _replace_image_references(self, old_registry: str, content: str): csv_namespace = self._group_config.get('csv_namespace', 'openshift') for pullspec, image_info in zip(references, image_infos): image_labels = image_info['config']['config']['Labels'] - image_component = image_labels['com.redhat.component'] image_version = image_labels['version'] image_release = image_labels['release'] - image_nvr = f"{image_component}-{image_version}-{image_release}" + image_envs = image_info['config']['config']['Env'] + image_dgk = next((env.split('=')[1] for env in image_envs if env.startswith('__doozer_key='))) + image_nvr = f"{image_dgk}-{image_version}-{image_release}" namespace, image_short_name, image_tag = references[pullspec] image_sha = image_info['listDigest'] if self._group_config.operator_image_ref_mode == 'manifest-list' else image_info['contentDigest'] new_namespace = 'openshift4' if namespace == csv_namespace else namespace @@ -377,6 +378,7 @@ def __init__(self, skip_checks: bool = False, pipelinerun_template_url: str = constants.KONFLUX_DEFAULT_BUNDLE_BUILD_PLR_TEMPLATE_URL, dry_run: bool = False, + record_logger: Optional[RecordLogger] = None, logger: logging.Logger = _LOGGER) -> None: self.base_dir = base_dir self.group = group @@ -390,6 +392,7 @@ def __init__(self, self.skip_checks = skip_checks self.pipelinerun_template_url = pipelinerun_template_url self.dry_run = dry_run + self._record_logger = record_logger self._logger = logger self._konflux_client = KonfluxClient.from_kubeconfig(self.konflux_namespace, self.konflux_kubeconfig, self.konflux_context, dry_run=self.dry_run) @@ -397,72 +400,111 @@ async def build(self, metadata: ImageMetadata): """ Build a bundle with Konflux. """ logger = self._logger.getChild(f"[{metadata.distgit_key}]") konflux_client = self._konflux_client - bundle_dir = self.base_dir.joinpath(metadata.get_olm_bundle_short_name()) - if bundle_dir.exists(): - # Load exiting build source repository - logger.info("Loading existing bundle repository...") - bundle_build_repo = await BuildRepo.from_local_dir(bundle_dir, self._logger) - logger.info("Bundle repository loaded from %s", bundle_dir) - else: - source = None - if metadata.has_source(): - logger.info("Resolving source...") - source = cast(SourceResolution, await exectools.to_thread(self._source_resolver.resolve_source, metadata, no_clone=True)) - else: - raise IOError(f"Image {metadata.qualified_key} doesn't have upstream source. This is no longer supported.") - # Clone the build source repository - bundle_build_branch = "art-{group}-assembly-{assembly_name}-bundle-{distgit_key}".format_map({ - "group": self.group, - "assembly_name": self.assembly, - "distgit_key": metadata.distgit_key, - }) - logger.info("Cloning bundle repository...") - bundle_build_repo = BuildRepo(url=source.url, branch=bundle_build_branch, local_dir=bundle_dir, logger=self._logger) - await bundle_build_repo.ensure_source() - logger.info("Bundle repository cloned to %s", bundle_dir) - if not bundle_build_repo.commit_hash: - raise IOError(f"Bundle repository {bundle_build_repo.url} doesn't have any commits to build") - - logger.info("Starting Konflux bundle image build for %s...", metadata.distgit_key) - retries = 3 - for attempt in range(retries): - logger.info("Build attempt %d/%d", attempt + 1, retries) - pipelinerun, url = await self._start_build(metadata, bundle_build_repo, self.image_repo, self.konflux_namespace, self.skip_checks) - pipelinerun_name = pipelinerun.metadata.name - logger.info(f"Build started: {url}") - - # Update the Konflux DB with status PENDING - outcome = KonfluxBuildOutcome.PENDING - if not self.dry_run: - await self._update_konflux_db(metadata, bundle_build_repo, pipelinerun, outcome) - else: - logger.warning("Dry run: Would update Konflux DB for %s with outcome %s", pipelinerun_name, outcome) - - # Wait for the PipelineRun to complete - pipelinerun = await konflux_client.wait_for_pipelinerun(pipelinerun_name, self.konflux_namespace) - status = pipelinerun.status.conditions[0].status - outcome = KonfluxBuildOutcome.SUCCESS if status == "True" else KonfluxBuildOutcome.FAILURE - logger.info(f"PipelineRun {url} completed with outcome {outcome}") + df_path = bundle_dir.joinpath("Dockerfile") + + record = { + 'status': -1, # Status defaults to failure until explicitly set by success. This handles raised exceptions. + "message": "Unknown failure", + "task_id": "n/a", + "task_url": "n/a", + "operator_nvr": "n/a", + "operand_nvrs": "n/a", + "bundle_nvr": "n/a", + } - # Update the Konflux DB with the final outcome - if not self.dry_run: - await self._update_konflux_db(metadata, bundle_build_repo, pipelinerun, outcome) - else: - logger.warning("Dry run: Would update Konflux DB for %s with outcome %s", pipelinerun_name, outcome) - if status != "True": - error = KonfluxOlmBundleBuildError(f"Konflux bundle image build for {metadata.distgit_key} failed", pipelinerun_name, pipelinerun) - logger.error(f"{error}: {url}") + try: + if bundle_dir.exists(): + # Load exiting build source repository + logger.info("Loading existing bundle repository...") + bundle_build_repo = await BuildRepo.from_local_dir(bundle_dir, self._logger) + logger.info("Bundle repository loaded from %s", bundle_dir) else: - error = None - break - - if error: - raise error + source = None + if metadata.has_source(): + logger.info("Resolving source...") + source = cast(SourceResolution, await exectools.to_thread(self._source_resolver.resolve_source, metadata, no_clone=True)) + else: + raise IOError(f"Image {metadata.qualified_key} doesn't have upstream source. This is no longer supported.") + # Clone the build source repository + bundle_build_branch = "art-{group}-assembly-{assembly_name}-bundle-{distgit_key}".format_map({ + "group": self.group, + "assembly_name": self.assembly, + "distgit_key": metadata.distgit_key, + }) + logger.info("Cloning bundle repository...") + bundle_build_repo = BuildRepo(url=source.url, branch=bundle_build_branch, local_dir=bundle_dir, logger=self._logger) + await bundle_build_repo.ensure_source() + logger.info("Bundle repository cloned to %s", bundle_dir) + if not bundle_build_repo.commit_hash: + raise IOError(f"Bundle repository {bundle_build_repo.url} doesn't have any commits to build") + + # Parse bundle's Dockerfile + bundle_df = DockerfileParser(str(df_path)) + version = bundle_df.labels.get('version') + if not version: + raise IOError(f"{metadata.distgit_key}: Label 'version' is not set. Did you run rebase?") + release = bundle_df.labels.get('release') + if not release: + raise IOError(f"{metadata.distgit_key}: Label 'release' is not set. Did you run rebase?") + nvr = f"{metadata.get_olm_bundle_short_name()}-{version}-{release}" + record['bundle_nvr'] = nvr + output_image = f"{self.image_repo}:{nvr}" + + # Load olm_bundle_info.yaml to get the operator and operand NVRs + async with aiofiles.open(bundle_build_repo.local_dir / '.oit' / 'olm_bundle_info.yaml', 'r') as f: + bundle_info = yaml.safe_load(await f.read()) + operator_nvr = bundle_info['operator']['nvr'] + record['operator_nvr'] = operator_nvr + operand_nvrs = sorted({info['nvr'] for info in bundle_info['operands'].values()}) + record['operand_nvrs'] = ','.join(operand_nvrs) + + # Start the bundle build + logger.info("Starting Konflux bundle image build for %s...", metadata.distgit_key) + retries = 3 + for attempt in range(retries): + logger.info("Build attempt %d/%d", attempt + 1, retries) + pipelinerun, url = await self._start_build(metadata, bundle_build_repo, output_image, self.konflux_namespace, self.skip_checks) + pipelinerun_name = pipelinerun.metadata.name + record["task_id"] = pipelinerun_name + record["task_url"] = url + + # Update the Konflux DB with status PENDING + outcome = KonfluxBuildOutcome.PENDING + if not self.dry_run: + await self._update_konflux_db(metadata, bundle_build_repo, pipelinerun, outcome, operator_nvr, operand_nvrs) + else: + logger.warning("Dry run: Would update Konflux DB for %s with outcome %s", pipelinerun_name, outcome) + + # Wait for the PipelineRun to complete + pipelinerun = await konflux_client.wait_for_pipelinerun(pipelinerun_name, self.konflux_namespace) + status = pipelinerun.status.conditions[0].status + outcome = KonfluxBuildOutcome.SUCCESS if status == "True" else KonfluxBuildOutcome.FAILURE + logger.info(f"PipelineRun {url} completed with outcome {outcome}") + + # Update the Konflux DB with the final outcome + if not self.dry_run: + await self._update_konflux_db(metadata, bundle_build_repo, pipelinerun, outcome, operator_nvr, operand_nvrs) + else: + logger.warning("Dry run: Would update Konflux DB for %s with outcome %s", pipelinerun_name, outcome) + if status != "True": + error = KonfluxOlmBundleBuildError(f"Konflux bundle image build for {metadata.distgit_key} failed", pipelinerun_name, pipelinerun) + logger.error(f"{error}: {url}") + else: + error = None + record["message"] = "Success" + record['status'] = 0 + break + if error: + record['message'] = str(error) + raise error + finally: + if self._record_logger: + self._record_logger.add_record("build_olm_bundle_konflux", **record) return pipelinerun_name, pipelinerun @limit_concurrency(limit=constants.MAX_KONFLUX_BUILD_QUEUE_SIZE) - async def _start_build(self, metadata: ImageMetadata, bundle_build_repo: BuildRepo, image_repo: str, namespace: str, + async def _start_build(self, metadata: ImageMetadata, bundle_build_repo: BuildRepo, output_image: str, namespace: str, skip_checks: bool = False, additional_tags: Optional[Sequence[str]] = None): """ Start a build with Konflux. """ if not bundle_build_repo.commit_hash: @@ -482,7 +524,7 @@ async def _start_build(self, metadata: ImageMetadata, bundle_build_repo: BuildRe # Openshift doesn't allow dots or underscores in any of its fields, so we replace them with dashes component_name = f"{app_name}-{bundle_name}".replace(".", "-").replace("_", "-") logger.info(f"Creating Konflux component: {component_name}") - dest_image_repo = image_repo + dest_image_repo = output_image.split(":")[0] await konflux_client.ensure_component( name=component_name, application=app_name, @@ -492,20 +534,7 @@ async def _start_build(self, metadata: ImageMetadata, bundle_build_repo: BuildRe revision=target_branch, ) logger.info(f"Konflux component {component_name} created") - # Read the bundle's Dockerfile - bundle_df = DockerfileParser(str(bundle_build_repo.local_dir.joinpath('Dockerfile'))) # Start a PipelineRun - component_name = bundle_df.labels.get('com.redhat.component') - if not component_name: - raise IOError(f"{metadata.distgit_key}: Label 'com.redhat.component' is not set. Did you run rebase?") - version = bundle_df.labels.get('version') - if not version: - raise IOError(f"{metadata.distgit_key}: Label 'version' is not set. Did you run rebase?") - release = bundle_df.labels.get('release') - if not release: - raise IOError(f"{metadata.distgit_key}: Label 'release' is not set. Did you run rebase?") - nvr = f"{component_name}-{version}-{release}" - logger.info(f"Building bundle {nvr}...") pipelinerun = await konflux_client.start_pipeline_run_for_image_build( generate_name=f"{component_name}-", namespace=namespace, @@ -514,7 +543,7 @@ async def _start_build(self, metadata: ImageMetadata, bundle_build_repo: BuildRe git_url=bundle_build_repo.https_url, commit_sha=bundle_build_repo.commit_hash, target_branch=target_branch, - output_image=f"{dest_image_repo}:{nvr}", + output_image=output_image, vm_override={}, building_arches=["x86_64"], # We always build bundles on x86_64 additional_tags=list(additional_tags), @@ -526,7 +555,8 @@ async def _start_build(self, metadata: ImageMetadata, bundle_build_repo: BuildRe return pipelinerun, url async def _update_konflux_db(self, metadata: ImageMetadata, build_repo: BuildRepo, - pipelinerun: resource.ResourceInstance, outcome: KonfluxBuildOutcome): + pipelinerun: resource.ResourceInstance, outcome: KonfluxBuildOutcome, + operator_nvr: str, operand_nvrs: list[str]): logger = self._logger.getChild(f"[{metadata.distgit_key}]") db = self._db if not db or db.record_cls != KonfluxBundleBuildRecord: @@ -550,12 +580,6 @@ async def _update_konflux_db(self, metadata: ImageMetadata, build_repo: BuildRep pipelinerun_name = pipelinerun.metadata.name build_pipeline_url = KonfluxClient.build_pipeline_url(pipelinerun) - # Load .oit files - async with aiofiles.open(build_repo.local_dir / '.oit' / 'olm_bundle_info.yaml', 'r') as f: - bundle_info = yaml.safe_load(await f.read()) - operator_nvr = bundle_info['operator']['nvr'] - operand_nvrs = sorted({info['nvr'] for info in bundle_info['operands'].values()}) - build_record_params = { 'name': metadata.get_olm_bundle_short_name(), 'version': version, diff --git a/doozer/doozerlib/cli/images_konflux.py b/doozer/doozerlib/cli/images_konflux.py index c1c4b9677..300cf7549 100644 --- a/doozer/doozerlib/cli/images_konflux.py +++ b/doozer/doozerlib/cli/images_konflux.py @@ -164,7 +164,7 @@ async def run(self): dry_run=self.dry_run, plr_template=self.plr_template ) - builder = KonfluxImageBuilder(config=config) + builder = KonfluxImageBuilder(config=config, record_logger=runtime.record_logger) tasks = [] for image_meta in metas: tasks.append(asyncio.create_task(builder.build(image_meta))) @@ -184,7 +184,7 @@ async def run(self): @cli.command("beta:images:konflux:build", short_help="Build images for the group.") @click.option('--konflux-kubeconfig', metavar='PATH', help='Path to the kubeconfig file to use for Konflux cluster connections.') @click.option('--konflux-context', metavar='CONTEXT', help='The name of the kubeconfig context to use for Konflux cluster connections.') -@click.option('--konflux-namespace', metavar='NAMESPACE', required=True, help='The namespace to use for Konflux cluster connections.') +@click.option('--konflux-namespace', metavar='NAMESPACE', default=constants.KONFLUX_DEFAULT_NAMESPACE, help='The namespace to use for Konflux cluster connections.') @click.option('--image-repo', default=constants.KONFLUX_DEFAULT_IMAGE_REPO, help='Push images to the specified repo.') @click.option('--skip-checks', default=False, is_flag=True, help='Skip all post build checks') @click.option('--dry-run', default=False, is_flag=True, help='Do not build anything, but only print build operations.') @@ -352,6 +352,7 @@ async def run(self): skip_checks=self.skip_checks, pipelinerun_template_url=self.plr_template, dry_run=self.dry_run, + record_logger=runtime.record_logger, ) tasks = [] @@ -379,7 +380,7 @@ async def run(self): help='Do not push to build repo or build anything, but print what would be done.') @click.option('--konflux-kubeconfig', metavar='PATH', help='Path to the kubeconfig file to use for Konflux cluster connections.') @click.option('--konflux-context', metavar='CONTEXT', help='The name of the kubeconfig context to use for Konflux cluster connections.') -@click.option('--konflux-namespace', metavar='NAMESPACE', required=True, help='The namespace to use for Konflux cluster connections.') +@click.option('--konflux-namespace', metavar='NAMESPACE', default=constants.KONFLUX_DEFAULT_NAMESPACE, help='The namespace to use for Konflux cluster connections.') @click.option('--image-repo', default=constants.KONFLUX_DEFAULT_IMAGE_REPO, help='Push images to the specified repo.') @click.option('--skip-checks', default=False, is_flag=True, help='Skip all post build checks') @click.option("--release", metavar='RELEASE', help="Release string to populate in bundle's Dockerfiles.") diff --git a/doozer/doozerlib/constants.py b/doozer/doozerlib/constants.py index 02803e09e..2f3389b2c 100644 --- a/doozer/doozerlib/constants.py +++ b/doozer/doozerlib/constants.py @@ -38,6 +38,7 @@ ART_PROD_IMAGE_REPO = "quay.io/openshift-release-dev/ocp-v4.0-art-dev" KONFLUX_UI_HOST = "https://konflux.apps.kflux-ocp-p01.7ayg.p1.openshiftapps.com" KONFLUX_UI_DEFAULT_WORKSPACE = "ocp-art" # associated with ocp-art-tenant +KONFLUX_DEFAULT_NAMESPACE = f"{KONFLUX_UI_DEFAULT_WORKSPACE}-tenant" MAX_KONFLUX_BUILD_QUEUE_SIZE = 25 # how many concurrent Konflux pipeline can we spawn per OCP version? KONFLUX_DEFAULT_IMAGE_BUILD_PLR_TEMPLATE_URL = "https://github.com/openshift-priv/art-konflux-template/raw/refs/heads/main/.tekton/art-konflux-template-push.yaml" KONFLUX_DEFAULT_BUNDLE_BUILD_PLR_TEMPLATE_URL = "https://github.com/openshift-priv/art-konflux-template/raw/refs/heads/main/.tekton/art-bundle-konflux-template-push.yaml" diff --git a/doozer/tests/backend/test_konflux_olm_bundler.py b/doozer/tests/backend/test_konflux_olm_bundler.py index c6ed27074..90fb3c559 100644 --- a/doozer/tests/backend/test_konflux_olm_bundler.py +++ b/doozer/tests/backend/test_konflux_olm_bundler.py @@ -84,9 +84,12 @@ async def test_replace_image_references(self, mock_oc_image_info): 'config': { 'config': { 'Labels': { - 'com.redhat.component': 'test-component', + 'com.redhat.component': 'test-brew-component', 'version': '1.0', 'release': '1', + }, + 'Env': { + '__doozer_key=test-component' } } }, @@ -388,7 +391,7 @@ async def test_rebase_dir(self, mock_create_oit_files, mock_create_container_yam 'operators.operatorframework.io.bundle.package.v1': 'test-package', }, input_release) mock_create_container_yaml.assert_called_once_with(Path("/path/to/bundle/dir/container.yaml")) - mock_create_oit_files.assert_called_once_with(bundle_dir, 'test-component-1.0-1', { + mock_create_oit_files.assert_called_once_with(bundle_dir, 'test-distgit-key-1.0-1', { 'image': ('old_pullspec', 'new_pullspec', 'test-component-1.0-1') }) @@ -481,7 +484,7 @@ async def test_rebase_dir_no_dockerfile_labels(self, mock_dockerfile_parser, _): with self.assertRaises(ValueError) as context: await self.rebaser._rebase_dir(metadata, operator_dir, bundle_dir, input_release) - self.assertIn("Label 'com.redhat.component', 'version' or 'release' is not set in the operator's Dockerfile", str(context.exception)) + self.assertIn("Label 'version' or 'release' is not set in the operator's Dockerfile", str(context.exception)) @patch("pathlib.Path.iterdir", return_value=iter([])) async def test_rebase_dir_no_files_in_bundle_dir(self, _): @@ -536,7 +539,7 @@ def setUp(self): ) @patch("doozerlib.backend.konflux_olm_bundler.DockerfileParser") - async def test_start_build_success(self, mock_dockerfile_parser): + async def test_start_build(self, mock_dockerfile_parser): metadata = MagicMock() metadata.distgit_key = "test-distgit-key" metadata.get_olm_bundle_short_name.return_value = "test-bundle" @@ -559,7 +562,7 @@ async def test_start_build_success(self, mock_dockerfile_parser): self.konflux_client.start_pipeline_run_for_image_build.return_value = pipelinerun self.konflux_client.build_pipeline_url = MagicMock(return_value="https://example.com/pipelinerun") - pipelinerun, url = await self.builder._start_build(metadata, bundle_build_repo, self.image_repo, self.konflux_namespace, self.skip_checks, additional_tags) + pipelinerun, url = await self.builder._start_build(metadata, bundle_build_repo, f"{self.image_repo}:test-component-1.0-1", self.konflux_namespace, self.skip_checks, additional_tags) self.konflux_client.ensure_application.assert_called_once_with(name="test-group", display_name="test-group") self.konflux_client.ensure_component.assert_called_once_with( @@ -571,10 +574,10 @@ async def test_start_build_success(self, mock_dockerfile_parser): revision=bundle_build_repo.commit_hash, ) self.konflux_client.start_pipeline_run_for_image_build.assert_called_once_with( - generate_name="test-component-", + generate_name="test-group-test-bundle-", namespace=self.konflux_namespace, application_name="test-group", - component_name="test-component", + component_name='test-group-test-bundle', git_url=bundle_build_repo.https_url, commit_sha=bundle_build_repo.commit_hash, target_branch=bundle_build_repo.commit_hash, @@ -587,66 +590,6 @@ async def test_start_build_success(self, mock_dockerfile_parser): ) self.assertEqual(url, "https://example.com/pipelinerun") - @patch("doozerlib.backend.konflux_olm_bundler.DockerfileParser") - async def test_start_build_missing_component_label(self, mock_dockerfile_parser): - metadata = MagicMock() - metadata.distgit_key = "test-distgit-key" - bundle_build_repo = MagicMock() - bundle_build_repo.commit_hash = "test-commit-hash" - bundle_build_repo.branch = None - bundle_build_repo.https_url = "https://example.com/repo.git" - - mock_dockerfile = MagicMock() - mock_dockerfile.labels = { - 'version': '1.0', - 'release': '1', - } - mock_dockerfile_parser.return_value = mock_dockerfile - - with self.assertRaises(IOError) as context: - await self.builder._start_build(metadata, bundle_build_repo, self.image_repo, self.konflux_namespace) - self.assertIn("Label 'com.redhat.component' is not set. Did you run rebase?", str(context.exception)) - - @patch("doozerlib.backend.konflux_olm_bundler.DockerfileParser") - async def test_start_build_missing_version_label(self, mock_dockerfile_parser): - metadata = MagicMock() - metadata.distgit_key = "test-distgit-key" - bundle_build_repo = MagicMock() - bundle_build_repo.commit_hash = "test-commit-hash" - bundle_build_repo.branch = None - bundle_build_repo.https_url = "https://example.com/repo.git" - - mock_dockerfile = MagicMock() - mock_dockerfile.labels = { - 'com.redhat.component': 'test-component', - 'release': '1', - } - mock_dockerfile_parser.return_value = mock_dockerfile - - with self.assertRaises(IOError) as context: - await self.builder._start_build(metadata, bundle_build_repo, self.image_repo, self.konflux_namespace) - self.assertIn("Label 'version' is not set. Did you run rebase?", str(context.exception)) - - @patch("doozerlib.backend.konflux_olm_bundler.DockerfileParser") - async def test_start_build_missing_release_label(self, mock_dockerfile_parser): - metadata = MagicMock() - metadata.distgit_key = "test-distgit-key" - bundle_build_repo = MagicMock() - bundle_build_repo.commit_hash = "test-commit-hash" - bundle_build_repo.branch = None - bundle_build_repo.https_url = "https://example.com/repo.git" - - mock_dockerfile = MagicMock() - mock_dockerfile.labels = { - 'com.redhat.component': 'test-component', - 'version': '1.0', - } - mock_dockerfile_parser.return_value = mock_dockerfile - - with self.assertRaises(IOError) as context: - await self.builder._start_build(metadata, bundle_build_repo, self.image_repo, self.konflux_namespace) - self.assertIn("Label 'release' is not set. Did you run rebase?", str(context.exception)) - async def test_start_build_no_commit_hash(self): metadata = MagicMock() metadata.distgit_key = "test-distgit-key" @@ -703,7 +646,8 @@ async def test_update_konflux_db_success(self, mock_build_pipeline_url, mock_doc } }) - await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.SUCCESS) + await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.SUCCESS, + 'test-operator-1.0-1', ["operand1-1.0-1", "operand2-1.0-1"]) self.db.add_build.assert_called_once() build_record = self.db.add_build.call_args[0][0] @@ -770,7 +714,8 @@ async def test_update_konflux_db_failure(self, mock_build_pipeline_url, mock_doc } }) - await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.FAILURE) + await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.FAILURE, + 'test-operator-1.0-1', ["operand1-1.0-1", "operand2-1.0-1"]) self.db.add_build.assert_called_once() build_record = self.db.add_build.call_args[0][0] @@ -842,7 +787,8 @@ async def test_update_konflux_db_no_db_connection(self, mock_build_pipeline_url, self.builder._db = None with self.assertLogs(self.builder._logger, level='WARNING') as cm: - await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.SUCCESS) + await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.SUCCESS, + 'test-operator-1.0-1', ["operand1-1.0-1", "operand2-1.0-1"]) self.assertIn('Konflux DB connection is not initialized, not writing build record to the Konflux DB.', cm.output[0]) @patch("aiofiles.open") @@ -893,5 +839,6 @@ async def test_update_konflux_db_exception(self, mock_build_pipeline_url, mock_d self.db.add_build.side_effect = Exception("Test exception") with self.assertLogs(self.builder._logger, level='ERROR') as cm: - await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.SUCCESS) + await self.builder._update_konflux_db(metadata, build_repo, pipelinerun, KonfluxBuildOutcome.SUCCESS, + 'test-operator-1.0-1', ["operand1-1.0-1", "operand2-1.0-1"]) self.assertIn('Failed writing record to the konflux DB: Test exception', cm.output[0]) diff --git a/pyartcd/pyartcd/__main__.py b/pyartcd/pyartcd/__main__.py index 3b8e65cb4..f88d51887 100644 --- a/pyartcd/pyartcd/__main__.py +++ b/pyartcd/pyartcd/__main__.py @@ -5,7 +5,8 @@ build_microshift_bootc, build_microshift, check_bugs, gen_assembly, ocp4_konflux, prepare_release, promote, rebuild, review_cvp, tarball_sources, build_sync, build_rhcos, ocp4_scan, ocp4_scan_konflux, images_health, operator_sdk_sync, olm_bundle, ocp4, scan_for_kernel_bugs, tag_rpms, advisory_drop, cleanup_locks, brew_scan_osh, - sigstore_sign, update_golang, rebuild_golang_rpms, scan_fips, quay_doomsday_backup + sigstore_sign, update_golang, rebuild_golang_rpms, scan_fips, quay_doomsday_backup, + olm_bundle_konflux ) from pyartcd.pipelines.scheduled import schedule_ocp4_scan, schedule_ocp4_scan_konflux diff --git a/pyartcd/pyartcd/constants.py b/pyartcd/pyartcd/constants.py index e0ecfef54..03d42bf52 100644 --- a/pyartcd/pyartcd/constants.py +++ b/pyartcd/pyartcd/constants.py @@ -49,4 +49,5 @@ GITHUB_OWNER = "openshift-eng" -KONFLUX_IMAGE_BUILD_PLR_TEMPLATE_URL_FORMAT = "https://raw.githubusercontent.com/{owner}/art-konflux-template/refs/heads/{branch_name}/.tekton/art-konflux-template-push.yaml" # Konflux PipelineRun (PLR) template +KONFLUX_IMAGE_BUILD_PLR_TEMPLATE_URL_FORMAT = "https://raw.githubusercontent.com/{owner}/art-konflux-template/refs/heads/{branch_name}/.tekton/art-konflux-template-push.yaml" # Konflux PipelineRun (PLR) template for image builds +KONFLUX_BUNDLE_BUILD_PLR_TEMPLATE_URL_FORMAT = "https://raw.githubusercontent.com/{owner}/art-konflux-template/refs/heads/{branch_name}/.tekton/art-bundle-konflux-template-push.yaml" # Konflux PipelineRun (PLR) template for bundle builds diff --git a/pyartcd/pyartcd/jenkins.py b/pyartcd/pyartcd/jenkins.py index 33884ba4d..699660e33 100644 --- a/pyartcd/pyartcd/jenkins.py +++ b/pyartcd/pyartcd/jenkins.py @@ -32,6 +32,7 @@ class Jobs(Enum): OCP4_SCAN_KONFLUX = 'aos-cd-builds/build%2Focp4-scan-konflux' RHCOS = 'aos-cd-builds/build%2Frhcos' OLM_BUNDLE = 'aos-cd-builds/build%2Folm_bundle' + OLM_BUNDLE_KONFLUX = 'aos-cd-builds/build%2Folm_bundle_konflux' SYNC_FOR_CI = 'scheduled-builds/sync-for-ci' MICROSHIFT_SYNC = 'aos-cd-builds/build%2Fmicroshift_sync' CINCINNATI_PRS = 'aos-cd-builds/build%2Fcincinnati-prs' @@ -292,7 +293,7 @@ def start_ocp4_konflux(build_version: str, assembly: str, image_list: list, params['LIMIT_ARCHES'] = ','.join(limit_arches) return start_build( - job=Jobs.OCP4_KONFLUX, + job=Jobs.OLM_BUNDLE_KONFLUX, params=params, **kwargs ) @@ -393,6 +394,27 @@ def start_olm_bundle(build_version: str, assembly: str, operator_nvrs: list, ) +def start_olm_bundle_konflux( + build_version: str, assembly: str, operator_nvrs: list, + doozer_data_path: str = constants.OCP_BUILD_DATA_URL, + doozer_data_gitref: str = '', **kwargs) -> Optional[str]: + if not operator_nvrs: + logger.warning('Empty operator NVR received: skipping olm-bundle') + return + + return start_build( + job=Jobs.OLM_BUNDLE_KONFLUX, + params={ + 'BUILD_VERSION': build_version, + 'ASSEMBLY': assembly, + 'DOOZER_DATA_PATH': doozer_data_path, + 'DOOZER_DATA_GITREF': doozer_data_gitref, + 'OPERATOR_NVRS': ','.join(operator_nvrs) + }, + **kwargs + ) + + def start_sync_for_ci(version: str, **kwargs): return start_build( job=Jobs.SYNC_FOR_CI, diff --git a/pyartcd/pyartcd/locks.py b/pyartcd/pyartcd/locks.py index fa845ce21..c59ddbf0e 100644 --- a/pyartcd/pyartcd/locks.py +++ b/pyartcd/pyartcd/locks.py @@ -11,6 +11,7 @@ # Defines the pipeline locks managed by Redis class Lock(enum.Enum): OLM_BUNDLE = 'lock:olm-bundle:{version}' + OLM_BUNDLE_KONFLUX = 'lock:olm-bundle-konflux:{version}' MIRRORING_RPMS = 'lock:mirroring-rpms:{version}' PLASHET = 'lock:compose:{assembly}:{version}' BUILD = 'lock:build:{version}' @@ -42,6 +43,11 @@ class Keys(enum.Enum): 'retry_delay_min': 0.1, 'lock_timeout': DEFAULT_LOCK_TIMEOUT }, + Lock.OLM_BUNDLE_KONFLUX: { + 'retry_count': 36000, + 'retry_delay_min': 0.1, + 'lock_timeout': DEFAULT_LOCK_TIMEOUT + }, Lock.MIRRORING_RPMS: { 'retry_count': 36000, 'retry_delay_min': 0.1, diff --git a/pyartcd/pyartcd/pipelines/ocp4_konflux.py b/pyartcd/pyartcd/pipelines/ocp4_konflux.py index 251031d73..37e24097e 100644 --- a/pyartcd/pyartcd/pipelines/ocp4_konflux.py +++ b/pyartcd/pyartcd/pipelines/ocp4_konflux.py @@ -2,6 +2,7 @@ import json import logging import os +from pathlib import Path import shutil from enum import Enum from typing import Optional, Tuple @@ -15,6 +16,7 @@ from pyartcd.cli import cli, click_coroutine, pass_runtime from pyartcd.locks import Lock from pyartcd.runtime import Runtime +from pyartcd import record as record_util LOGGER = logging.getLogger(__name__) @@ -49,13 +51,16 @@ def default(self, obj): class KonfluxOcp4Pipeline: def __init__(self, runtime: Runtime, assembly: str, data_path: Optional[str], image_build_strategy: Optional[str], image_list: Optional[str], version: str, data_gitref: Optional[str], - kubeconfig: Optional[str], skip_rebase: bool, arches: Tuple[str, ...], plr_template: str): + kubeconfig: Optional[str], skip_rebase: bool, skip_bundle_build: bool, arches: Tuple[str, ...], plr_template: str): self.runtime = runtime self.assembly = assembly self.version = version + self.data_path = data_path + self.data_gitref = data_gitref self.kubeconfig = kubeconfig self.arches = arches self.skip_rebase = skip_rebase + self.skip_bundle_build = skip_bundle_build self.plr_template = plr_template group_param = f'--group=openshift-{version}' @@ -209,6 +214,27 @@ async def clean_up(self): self.runtime.cleanup_sources('konflux_build_sources'), ]) + def trigger_bundle_build(self): + record_log_path = Path(self.runtime.doozer_working, 'record.log') + if not record_log_path.exists(): + LOGGER.warning('record.log not found, skipping bundle build') + return + with record_log_path.open('r') as file: + record_log: dict = record_util.parse_record_log(file) + records = record_log.get('image_build_konflux', []) + operator_nvrs = [] + for record in records: + if record['has_olm_bundle'] == '1' and record['status'] == '0' and record.get('nvrs', None): + operator_nvrs.append(record['nvrs'].split(',')[0]) + if operator_nvrs: + jenkins.start_olm_bundle_konflux( + build_version=self.version, + assembly=self.assembly, + operator_nvrs=operator_nvrs, + doozer_data_path=self.data_path or '', + doozer_data_gitref=self.data_gitref or '', + ) + async def run(self): await self.initialize() @@ -222,9 +248,17 @@ async def run(self): await self.rebase(version, input_release) LOGGER.info(f"Building images for OCP {self.version} with release {input_release}") - await self.build() - - await self.clean_up() + try: + await self.build() + finally: + if self.skip_bundle_build: + LOGGER.warning("Skipping bundle build step because --skip-bundle-build flag is set") + else: + try: + self.trigger_bundle_build() + except Exception as e: + LOGGER.exception(f"Failed to trigger bundle build: {e}") + await self.clean_up() @cli.command("beta:ocp4-konflux", help="A pipeline to build images with Konflux for OCP 4") @@ -244,6 +278,7 @@ async def run(self): @click.option('--ignore-locks', is_flag=True, default=False, help='Do not wait for other builds in this version to complete (use only if you know they will not conflict)') @click.option("--skip-rebase", is_flag=True, help="(For testing) Skip the rebase step") +@click.option("--skip-bundle-build", is_flag=True, help="(For testing) Skip the bundle build step") @click.option("--arch", "arches", metavar="TAG", multiple=True, help="(Optional) [MULTIPLE] Limit included arches to this list") @click.option('--plr-template', required=False, default='', @@ -252,7 +287,7 @@ async def run(self): @click_coroutine async def ocp4(runtime: Runtime, image_build_strategy: str, image_list: Optional[str], assembly: str, data_path: Optional[str], version: str, data_gitref: Optional[str], kubeconfig: Optional[str], - ignore_locks: bool, skip_rebase: bool, arches: Tuple[str, ...], plr_template: str): + ignore_locks: bool, skip_rebase: bool, skip_bundle_build: bool, arches: Tuple[str, ...], plr_template: str): if not kubeconfig: kubeconfig = os.environ.get('KONFLUX_SA_KUBECONFIG') @@ -270,6 +305,7 @@ async def ocp4(runtime: Runtime, image_build_strategy: str, image_list: Optional data_gitref=data_gitref, kubeconfig=kubeconfig, skip_rebase=skip_rebase, + skip_bundle_build=skip_bundle_build, arches=arches, plr_template=plr_template) diff --git a/pyartcd/pyartcd/pipelines/olm_bundle.py b/pyartcd/pyartcd/pipelines/olm_bundle.py index db4ba42d1..1fc735abb 100644 --- a/pyartcd/pyartcd/pipelines/olm_bundle.py +++ b/pyartcd/pyartcd/pipelines/olm_bundle.py @@ -40,7 +40,7 @@ async def olm_bundle(runtime: Runtime, version: str, assembly: str, data_path: s cmd = [ 'doozer', f'--assembly={assembly}', - '--working-dir=doozer_working', + f'--working-dir={runtime.doozer_working}', f'--group=openshift-{version}@{data_gitref}' if data_gitref else f'--group=openshift-{version}', f'--data-path={data_path}' ] @@ -73,7 +73,7 @@ async def olm_bundle(runtime: Runtime, version: str, assembly: str, data_path: s ) # Parse doozer record.log - with open('doozer_working/record.log') as file: + with open(f'{runtime.doozer_working}/record.log') as file: record_log = parse_record_log(file) records = record_log.get('build_olm_bundle', []) bundle_nvrs = [] diff --git a/pyartcd/pyartcd/pipelines/olm_bundle_konflux.py b/pyartcd/pyartcd/pipelines/olm_bundle_konflux.py new file mode 100644 index 000000000..61fb6f75d --- /dev/null +++ b/pyartcd/pyartcd/pipelines/olm_bundle_konflux.py @@ -0,0 +1,108 @@ +import os +from pathlib import Path + +import click +from aioredlock import LockError + +from artcommonlib import exectools +from pyartcd import constants, jenkins +from pyartcd.cli import cli, pass_runtime, click_coroutine +from pyartcd import locks +from pyartcd.locks import Lock +from pyartcd.record import parse_record_log +from pyartcd.runtime import Runtime + + +@cli.command('olm-bundle-konflux') +@click.option('--version', required=True, help='OCP version') +@click.option('--assembly', required=True, help='Assembly name') +@click.option('--data-path', required=False, default=constants.OCP_BUILD_DATA_URL, + help='ocp-build-data fork to use (e.g. assembly definition in your own fork)') +@click.option('--data-gitref', required=False, + help='(Optional) Doozer data path git [branch / tag / sha] to use') +@click.option('--nvrs', required=False, + help='(Optional) List **only** the operator NVRs you want to build bundles for, everything else ' + 'gets ignored. The operators should not be mode:disabled/wip in ocp-build-data') +@click.option('--only', required=False, + help='(Optional) List **only** the operators you want to build, everything else gets ignored.\n' + 'Format: Comma and/or space separated list of brew packages (e.g.: cluster-nfd-operator-container)\n' + 'Leave empty to build all (except EXCLUDE, if defined)') +@click.option('--exclude', required=False, + help='(Optional) List the operators you **don\'t** want to build, everything else gets built.\n' + 'Format: Comma and/or space separated list of brew packages (e.g.: cluster-nfd-operator-container)\n' + 'Leave empty to build all (or ONLY, if defined)') +@click.option('--force', is_flag=True, + help='Rebuild bundle containers, even if they already exist for given operator NVRs') +@click.option("--kubeconfig", required=False, help="Path to kubeconfig file to use for Konflux cluster connections") +@click.option('--plr-template', required=False, default='', + help='Override the Pipeline Run template commit from openshift-priv/art-konflux-template; format: @') +@pass_runtime +@click_coroutine +async def olm_bundle_konflux( + runtime: Runtime, version: str, assembly: str, data_path: str, data_gitref: str, + nvrs: str, only: bool, exclude: str, force: bool, kubeconfig: str, plr_template: str): + # Create Doozer invocation + cmd = [ + 'doozer', + f'--assembly={assembly}', + f'--working-dir={runtime.doozer_working}', + f'--group=openshift-{version}@{data_gitref}' if data_gitref else f'--group=openshift-{version}', + f'--data-path={data_path}' + ] + if only: + cmd.append(f'--images={only}') + if exclude: + cmd.append(f'--exclude={exclude}') + cmd.append('beta:images:konflux:bundle') + if force: + cmd.append('--force') + if kubeconfig: + cmd.extend(['--konflux-kubeconfig', kubeconfig]) + if plr_template: + plr_template_owner, plr_template_branch = plr_template.split("@") if plr_template else ["openshift-priv", "main"] + plr_template_url = constants.KONFLUX_BUNDLE_BUILD_PLR_TEMPLATE_URL_FORMAT.format(owner=plr_template_owner, branch_name=plr_template_branch) + cmd.extend(['--plr-template', plr_template_url]) + if runtime.dry_run: + cmd.append('--dry-run') + if nvrs: + cmd.append('--') + cmd.extend(nvrs.split(',')) + + lock = Lock.OLM_BUNDLE_KONFLUX + lock_name = lock.value.format(version=version) + lock_identifier = jenkins.get_build_path() + if not lock_identifier: + runtime.logger.warning('Env var BUILD_URL has not been defined: a random identifier will be used for the locks') + + try: + # Build bundles + await locks.run_with_lock( + coro=exectools.cmd_assert_async(cmd), + lock=lock, + lock_name=lock_name, + lock_id=lock_identifier + ) + + # Parse doozer record.log + bundle_nvrs = [] + record_log_path = Path(runtime.doozer_working, 'record.log') + if record_log_path.exists(): + with record_log_path.open() as file: + record_log = parse_record_log(file) + records = record_log.get('build_olm_bundle_konflux', []) + for record in records: + if record['status'] != '0': + raise RuntimeError('record.log includes unexpected build_olm_bundle_konflux ' + f'record with error message: {record["message"]}') + bundle_nvrs.append(record['bundle_nvr']) + + runtime.logger.info(f'Successfully built:\n{", ".join(bundle_nvrs)}') + + except (ChildProcessError, RuntimeError) as e: + runtime.logger.error('Encountered error: %s', e) + if not runtime.dry_run and assembly != 'test': + # slack_client = runtime.new_slack_client() + # slack_client.bind_channel(version) + # await slack_client.say('*:heavy_exclamation_mark: olm_bundle failed*\n' + # f'buildvm job: {os.environ["BUILD_URL"]}') + raise