Define metadata.get_latest_konflux_build()

artcommon/artcommonlib/konflux/konflux_db.py

@@ -210,7 +210,7 @@ async def get_latest_build(
             name: str,
             group: str,
             outcome: KonfluxBuildOutcome = KonfluxBuildOutcome.SUCCESS,
-            assembly: str = 'stream',
+            assembly: typing.Optional[str] = None,
             el_target: typing.Optional[str] = None,
             artifact_type: typing.Optional[ArtifactType] = None,
             engine: typing.Optional[Engine] = None,
@@ -224,7 +224,7 @@ async def get_latest_build(
         :param name: component name, e.g. 'ironic'
         :param group: e.g. 'openshift-4.18'
         :param outcome: 'success' | 'failure'
-        :param assembly: non-standard assembly name, defaults to 'stream' if omitted
+        :param assembly: assembly name, if omitted any assembly is matched
         :param el_target: e.g. 'el8'
         :param artifact_type: 'rpm' | 'image'
         :param engine: 'brew' | 'konflux'
@@ -242,8 +242,10 @@ async def get_latest_build(
             Column('name', String) == name,
             Column('group', String) == group,
             Column('outcome', String) == str(outcome),
-            Column('assembly', String) == assembly,
         ]
+        if assembly:
+            base_clauses.append(Column('assembly', String) == assembly)
+
         order_by_clause = Column('start_time', quote=True).desc()
 
         if completed_before:

artcommon/tests/test_konflux_db.py

@@ -185,6 +185,15 @@ async def test_get_latest_build(self, query_mock, datetime_mock):
         datetime_mock.now.return_value = now
 
         await self.db.get_latest_build(name='ironic', group='openshift-4.18', outcome='success')
+        query_mock.assert_called_once_with(f"SELECT * FROM `{constants.BUILDS_TABLE_ID}` WHERE name = 'ironic' "
+                                           "AND `group` = 'openshift-4.18' AND outcome = 'success' "
+                                           f"AND start_time >= '{str(lower_bound)}' "
+                                           f"AND start_time < '{now}' "
+                                           "ORDER BY `start_time` DESC LIMIT 1")
+
+        query_mock.reset_mock()
+        await self.db.get_latest_build(name='ironic', group='openshift-4.18',
+                                       outcome='success', assembly='stream')
         query_mock.assert_called_once_with(f"SELECT * FROM `{constants.BUILDS_TABLE_ID}` WHERE name = 'ironic' "
                                            "AND `group` = 'openshift-4.18' AND outcome = 'success' "
                                            "AND assembly = 'stream' "
@@ -193,7 +202,8 @@ async def test_get_latest_build(self, query_mock, datetime_mock):
                                            "ORDER BY `start_time` DESC LIMIT 1")
 
         query_mock.reset_mock()
-        await self.db.get_latest_build(name='ironic', group='openshift-4.18', outcome='success', completed_before=now)
+        await self.db.get_latest_build(name='ironic', group='openshift-4.18', outcome='success',
+                                       completed_before=now, assembly='stream')
         query_mock.assert_called_once_with(f"SELECT * FROM `{constants.BUILDS_TABLE_ID}` WHERE name = 'ironic' "
                                            "AND `group` = 'openshift-4.18' AND outcome = 'success' "
                                            "AND assembly = 'stream' AND end_time IS NOT NULL "
@@ -204,7 +214,8 @@ async def test_get_latest_build(self, query_mock, datetime_mock):
 
         query_mock.reset_mock()
         like = {'release': 'b45ea65'}
-        await self.db.get_latest_build(name='ironic', group='openshift-4.18', outcome='success', extra_patterns=like)
+        await self.db.get_latest_build(name='ironic', group='openshift-4.18', outcome='success',
+                                       extra_patterns=like, assembly='stream')
         query_mock.assert_called_once_with(f"SELECT * FROM `{constants.BUILDS_TABLE_ID}` WHERE name = 'ironic' "
                                            "AND `group` = 'openshift-4.18' AND outcome = 'success' "
                                            "AND assembly = 'stream' "

doozer/doozerlib/cli/scan_sources_konflux.py

@@ -60,13 +60,6 @@ def __init__(self, runtime: Runtime, ci_kubeconfig: str, as_yaml: bool,
         self.assessment_reason = dict()  # maps metadata qualified_key => message describing change
         self.issues = list()  # tracks issues that arose during the scan, which did not interrupt the job
 
-        # Common params for all queries
-        self.base_search_params = {
-            'group': self.runtime.group,
-            'assembly': self.runtime.assembly,  # to let test ocp4-scan in non-stream assemblies, e.g. 'test'
-            'engine': Engine.KONFLUX.value,
-        }
-
         self.latest_image_build_records_map: typing.Dict[str, KonfluxBuildRecord] = {}
         self.latest_rpm_build_records_map: typing.Dict[str, typing.Dict[str, KonfluxBuildRecord]] = {}
         self.image_tree = {}
@@ -292,31 +285,25 @@ async def find_latest_rpms_builds(self):
 
         self.logger.info('Gathering latest RPM build records information...')
 
-        async def _find_target_build(rpm_name, el_target):
-            search_params = {
-                'name': rpm_name,
-                'el_target': el_target,
-                **self.base_search_params,
-                'engine': 'brew'  # for RPMs we only have Brew build engine
-            }
-            build_record = await self.runtime.konflux_db.get_latest_build(**search_params)
+        async def _find_target_build(rpm_meta, el_target):
+            rpm_name = rpm_meta.rpm_name
+            build_record = await rpm_meta.get_latest_build(el_target=el_target)
             if not self.latest_rpm_build_records_map.get(rpm_name):
                 self.latest_rpm_build_records_map[rpm_name] = {}
             self.latest_rpm_build_records_map[rpm_name][el_target] = build_record
 
         tasks = []
         for rpm in self.runtime.rpm_metas():
             tasks.extend([
-                _find_target_build(rpm.rpm_name, f'el{target}')
+                _find_target_build(rpm, f'el{target}')
                 for target in rpm.determine_rhel_targets()
             ])
         await asyncio.gather(*tasks)
 
     async def find_latest_image_builds(self, image_names: typing.List[str]):
         self.logger.info('Gathering latest image build records information...')
-        latest_image_builds = await self.runtime.konflux_db.get_latest_builds(
-            names=image_names,
-            **self.base_search_params)
+        latest_image_builds = await asyncio.gather(*[
+            self.runtime.image_map[name].get_latest_build(engine=Engine.KONFLUX.value) for name in image_names])
         self.latest_image_build_records_map.update((zip(
             image_names, latest_image_builds)))
 
@@ -421,10 +408,9 @@ async def scan_for_upstream_changes(self, image_meta: ImageMetadata):
 
         # Scan for any build in this assembly which includes the git commit.
         upstream_commit_hash = self.find_upstream_commit_hash(image_meta)
-        upstream_commit_build_record = await self.runtime.konflux_db.get_latest_build(
-            **self.base_search_params,
-            name=image_meta.distgit_key,
-            extra_patterns={'commitish': upstream_commit_hash}  # WHERE commitish LIKE '%{upstream_commit_hash}%'
+        upstream_commit_build_record = await image_meta.get_latest_build(
+            engine=Engine.KONFLUX.value,
+            extra_patterns={'commitish': upstream_commit_hash}
         )
 
         # No build from latest upstream commit: handle accordingly
@@ -455,9 +441,7 @@ async def handle_missing_upstream_commit_build(self, image_meta: ImageMetadata,
         now = datetime.now(timezone.utc)
 
         # Check whether a build attempt with this commit has failed before.
-        failed_commit_build_record = await self.runtime.konflux_db.get_latest_build(
-            **self.base_search_params,
-            name=image_meta.distgit_key,
+        failed_commit_build_record = await image_meta.get_latest_build(
             extra_patterns={'commitish': upstream_commit_hash},
             outcome=KonfluxBuildOutcome.FAILURE
         )
@@ -641,7 +625,11 @@ async def get_builder_build_start_time(self, builder_build_nvr: str) -> typing.O
 
         build = await anext(self.runtime.konflux_db.search_builds_by_fields(
             start_search=start_search,
-            where={'nvr': builder_build_nvr, **self.base_search_params},
+            where={
+                'nvr': builder_build_nvr,
+                'group': self.runtime.group,
+                'assembly': self.runtime.assembly,
+            },
             limit=1,
         ), None)
         if build:
@@ -745,15 +733,11 @@ async def check_rpm_target(rpm_meta: RPMMetadata, el_target):
 
             # Scan for any build in this assembly which includes the git commit.
             upstream_commit_hash = await find_rpm_commit_hash(rpm_meta)
-            search_params = {
-                'name': rpm_name,
-                'extra_patterns': {'commitish': upstream_commit_hash},
-                **self.base_search_params,
-                'engine': 'brew'  # for RPMs we only have Brew build engine
-            }
-            if el_target:
-                search_params['el_target'] = el_target
-            upstream_commit_build_record = await self.runtime.konflux_db.get_latest_build(**search_params)
+            upstream_commit_build_record = await rpm_meta.get_latest_build(
+                el_target=el_target,
+                extra_patterns={'commitish': upstream_commit_hash},
+                engine=Engine.BREW.value
+            )
 
             if not upstream_commit_build_record:
                 self.logger.warning('No build for RPM %s from upstream commit %s',

doozer/doozerlib/metadata.py

@@ -21,6 +21,7 @@
 import doozerlib
 from artcommonlib import logutil, exectools, assertion
 from artcommonlib.assembly import assembly_basis_event, assembly_metadata_config
+from artcommonlib.konflux.konflux_build_record import KonfluxBuildRecord, KonfluxBuildOutcome, Engine
 from artcommonlib.pushd import Dir
 from artcommonlib.model import Missing, Model
 from doozerlib.brew import BuildStates
@@ -401,11 +402,121 @@ async def get_latest_build(self, **kwargs):
             return await self.get_latest_brew_build_async(**kwargs)
 
         elif self.runtime.build_system == 'konflux':
-            raise NotImplementedError
+            return await self.get_latest_konflux_build(**kwargs)
 
         else:
             raise ValueError(f'Invalid value for --build-system: {self.runtime.build_system}')
 
+    async def get_pinned_konflux_build(self, el_target: int):
+        """
+        Return the assembly pinned NVR.
+        Under 'is' for RPMs, we expect 'el7' and/or 'el8', etc. For images, just 'nvr'.
+        """
+
+        is_config = self.config['is']
+
+        if self.meta_type == 'rpm':
+            if el_target is None:
+                raise ValueError(
+                    f'Expected el_target to be set when querying a pinned RPM component {self.distgit_key}')
+            is_nvr = is_config[f'el{el_target}']
+            if not is_nvr:
+                self.logger.warning('No pinned NVR found for RPM %s and target %s', self.rpm_name, el_target)
+                return None
+
+        else:
+            # The image metadata (or, more likely, the current assembly) has the image
+            # pinned. Return only the pinned NVR. When a child image is being rebased,
+            # it uses get_latest_build to find the parent NVR to use (if it is not
+            # included in the "-i" doozer argument). We need it to find the pinned NVR
+            # to place in its Dockerfile.
+            # Pinning also informs gen-payload when attempting to assemble a release.
+            is_nvr = is_config.nvr
+            if not is_nvr:
+                raise ValueError(f'Did not find nvr field in pinned Image component {self.distgit_key}')
+
+        # strict means raise an exception if not found.
+        return await self.runtime.konflux_db.get_build_record_by_nvr(is_nvr, strict=True)
+
+    async def get_latest_konflux_build(self, assembly: Optional[str] = None,
+                                       outcome: KonfluxBuildOutcome = KonfluxBuildOutcome.SUCCESS,
+                                       el_target: Optional[int] = None,
+                                       honor_is: bool = True,
+                                       completed_before: Optional[datetime] = None,
+                                       extra_patterns: dict = {},
+                                       **kwargs) -> Optional[KonfluxBuildRecord]:
+        """
+        :param assembly: A non-default assembly name to search relative to. If not specified, runtime.assembly
+                         will be used. If runtime.assembly is also None, the search will return true latest.
+                         If the assembly parameter is set to '', this search will also return true latest.
+        :param outcome: one in KonfluxBuildOutcome[FAILURE, SUCCESS, PENDING]
+        :param el_target: specify 'el7', 'el8', etc.
+                          For an RPM, leave as None if you want the true latest.
+                          For an image, leaving as none will default to the RHEL version in the branch
+                          associated with the metadata.
+        :param honor_is: If True, and an assembly component specifies 'is', that nvr will be returned.
+        :param completed_before: cut off timestamp for builds completion time
+        :param extra_patterns: e.g. {'release': 'b45ea65'} will result in adding "AND release LIKE '%b45ea65%'" to the query
+        """
+
+        assert self.runtime.konflux_db is not None, 'Konflux DB must be initialized with GCP credentials'
+        self.runtime.konflux_db.bind(KonfluxBuildRecord)
+
+        # Is the component pinned in config?
+        if honor_is and self.config['is']:
+            if outcome != KonfluxBuildOutcome.SUCCESS:
+                # If this component is defined by 'is', history failures, etc, do not matter.
+                return None
+            return await self.get_pinned_konflux_build(el_target=el_target)
+
+        # If it's not pinned, fetch the build from the Konflux DB
+        base_search_params = {
+            'name': self.distgit_key if self.meta_type == 'image' else self.rpm_name,
+            'group': self.runtime.group,
+            'outcome': outcome,
+            'completed_before': completed_before,
+            'engine': self.runtime.build_system,
+            'extra_patterns': extra_patterns,
+            **kwargs
+        }
+        if self.meta_type == 'rpm':
+            # For RPMs, if rhel target is not set fetch true latest
+            if el_target:
+                base_search_params['el_target'] = el_target
+        else:
+            # For images, if rhel target is not set default to the rhel version in this group
+            base_search_params['el_target'] = el_target if el_target else f'el{self.branch_el_target()}'
+
+        assembly = assembly if assembly else self.runtime.assembly
+        if not assembly:
+            # if assembly is '' (by parameter) or still None after runtime.assembly, get true latest
+            build_record = await self.runtime.konflux_db.get_latest_build(**base_search_params)
+
+        else:
+            basis_event = assembly_basis_event(self.runtime.get_releases_config(), assembly=assembly)
+            if basis_event:
+                # If an assembly has a basis event, its latest images can only be sourced from
+                # "is:" or the stream assembly. We've already checked for "is" above.
+                assembly = 'stream'
+
+            # Search by matching the assembly as well
+            build_record = await self.runtime.konflux_db.get_latest_build(
+                **base_search_params,
+                assembly=assembly,
+            )
+
+            # If not builds were found and assembly != stream, look for 'stream' builds
+            if build_record is None and assembly != 'stream':
+                build_record = await self.runtime.konflux_db.get_latest_build(
+                    **base_search_params,
+                    assembly='stream',
+                )
+
+        if not build_record:
+            self.logger.warning('No build found for %s in group and %s assembly %s',
+                                self.distgit_key, self.runtime.group, self.runtime.assembly)
+        return build_record
+
     async def get_latest_brew_build_async(self, **kwargs):
         return await asyncio.to_thread(self.get_latest_brew_build, **kwargs)
 

pyartcd/pyartcd/pipelines/ocp4_scan_konflux.py

@@ -93,7 +93,8 @@ async def get_changes(self):
             f'--working-dir={self._doozer_working}',
             f'--data-path={self.data_path}',
             f'--group={group_param}',
-            f'--assembly={self.assembly}'
+            f'--assembly={self.assembly}',
+            '--build-system=konflux'
         ]
         if self.image_list:
             cmd.append(f'--images={self.image_list}')