diff --git a/pyartcd/pyartcd/pipelines/promote.py b/pyartcd/pyartcd/pipelines/promote.py index 52e95a92e..2c44a6693 100644 --- a/pyartcd/pyartcd/pipelines/promote.py +++ b/pyartcd/pyartcd/pipelines/promote.py @@ -30,14 +30,12 @@ from artcommonlib.util import isolate_major_minor_in_group from pyartcd.locks import Lock from pyartcd.signatory import AsyncSignatory, SigstoreSignatory -from pyartcd.util import nightlies_with_pullspecs from pyartcd import constants, locks, util, jenkins from artcommonlib import exectools from pyartcd.cli import cli, click_coroutine, pass_runtime from artcommonlib.exceptions import VerificationError from pyartcd.jira import JIRAClient from pyartcd.mail import MailService -from pyartcd.s3 import sync_dir_to_s3_mirror from pyartcd.oc import get_release_image_info, get_release_image_pullspec, extract_release_binary, \ extract_release_client_tools, get_release_image_info_from_pullspec, extract_baremetal_installer from pyartcd.runtime import Runtime, GroupRuntime @@ -1396,8 +1394,11 @@ async def build_release_image(self, release_name: str, arch: str, previous_list: env = os.environ.copy() env["GOTRACEBACK"] = "all" self._logger.info("Running %s", " ".join(cmd)) - await exectools.cmd_assert_async(cmd, env=env, stdout=sys.stderr) - pass + return await retry( + reraise=True, + stop=stop_after_attempt(10), # retry 10 times + wait=wait_fixed(30), # wait for 30 seconds between retries + )(exectools.cmd_gather_async)(cmd, env=env) @staticmethod async def get_image_stream(namespace: str, image_stream: str): diff --git a/pyartcd/tests/pipelines/test_promote.py b/pyartcd/tests/pipelines/test_promote.py index 3e5e13e94..0a37f7c89 100644 --- a/pyartcd/tests/pipelines/test_promote.py +++ b/pyartcd/tests/pipelines/test_promote.py @@ -621,8 +621,8 @@ async def test_promote_arch(self, get_image_stream: AsyncMock, get_release_image tag_release.assert_not_awaited() @patch("pyartcd.jira.JIRAClient.from_url", return_value=None) - @patch("pyartcd.pipelines.promote.exectools.cmd_assert_async", return_value=0) - async def test_build_release_image_from_reference_release(self, cmd_assert_async: AsyncMock, _): + @patch("pyartcd.pipelines.promote.exectools.cmd_gather_async", return_value=0) + async def test_build_release_image_from_reference_release(self, cmd_gather_async: AsyncMock, _): runtime = MagicMock( config={ "build_config": { @@ -645,29 +645,29 @@ async def test_build_release_image_from_reference_release(self, cmd_assert_async await pipeline.build_release_image("4.10.99", "x86_64", previous_list, [], metadata, dest_pullspec, reference_release, None, keep_manifest_list=False) expected_cmd = ["oc", "adm", "release", "new", "-n", "ocp", "--name=4.10.99", "--to-image=example.com/foo/release:4.10.99-x86_64", f"--from-release={reference_release}", "--previous=4.10.98,4.10.97,4.9.99", "--metadata", "{\"description\": \"whatever\", \"url\": \"https://access.redhat.com/errata/RHBA-2099:2222\"}"] - cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY) + cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY) # test aarch64 reference_release = "registry.ci.openshift.org/ocp-arm64/release-arm64:whatever-aarch64" dest_pullspec = "example.com/foo/release:4.10.99-aarch64" - cmd_assert_async.reset_mock() + cmd_gather_async.reset_mock() await pipeline.build_release_image("4.10.99", "aarch64", previous_list, [], metadata, dest_pullspec, reference_release, None, keep_manifest_list=False) expected_cmd = ["oc", "adm", "release", "new", "-n", "ocp-arm64", "--name=4.10.99", "--to-image=example.com/foo/release:4.10.99-aarch64", f"--from-release={reference_release}", "--previous=4.10.98,4.10.97,4.9.99", "--metadata", "{\"description\": \"whatever\", \"url\": \"https://access.redhat.com/errata/RHBA-2099:2222\"}"] - cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY) + cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY) # test multi-aarch64 reference_release = "registry.ci.openshift.org/ocp-arm64/release-arm64:whatever-multi-aarch64" dest_pullspec = "example.com/foo/release:4.10.99-multi-aarch64" - cmd_assert_async.reset_mock() + cmd_gather_async.reset_mock() await pipeline.build_release_image("4.10.99", "aarch64", previous_list, [], metadata, dest_pullspec, reference_release, None, keep_manifest_list=True) expected_cmd = ["oc", "adm", "release", "new", "-n", "ocp-arm64", "--name=4.10.99", "--to-image=example.com/foo/release:4.10.99-multi-aarch64", f"--from-release={reference_release}", "--keep-manifest-list", "--previous=4.10.98,4.10.97,4.9.99", "--metadata", "{\"description\": \"whatever\", \"url\": \"https://access.redhat.com/errata/RHBA-2099:2222\"}"] - cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY) + cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY) @patch("pyartcd.jira.JIRAClient.from_url", return_value=None) - @patch("pyartcd.pipelines.promote.exectools.cmd_assert_async", return_value=0) - async def test_build_release_image_from_image_stream(self, cmd_assert_async: AsyncMock, _): + @patch("pyartcd.pipelines.promote.exectools.cmd_gather_async", return_value=0) + async def test_build_release_image_from_image_stream(self, cmd_gather_async: AsyncMock, _): runtime = MagicMock(config={"build_config": {"ocp_build_data_url": "https://example.com/ocp-build-data.git"}, "jira": {"url": "https://issues.redhat.com/"}}, working_dir=Path("/path/to/working"), dry_run=False) @@ -682,17 +682,17 @@ async def test_build_release_image_from_image_stream(self, cmd_assert_async: Asy await pipeline.build_release_image("4.10.99", "x86_64", previous_list, [], metadata, dest_pullspec, reference_release, from_image_stream, keep_manifest_list=False) expected_cmd = ['oc', 'adm', 'release', 'new', '-n', 'ocp', '--name=4.10.99', '--to-image=example.com/foo/release:4.10.99-x86_64', '--reference-mode=source', '--from-image-stream=4.10-art-assembly-4.10.99', '--previous=4.10.98,4.10.97,4.9.99', '--metadata', '{"description": "whatever", "url": "https://access.redhat.com/errata/RHBA-2099:2222"}'] - cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY) + cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY) # test aarch64 reference_release = None dest_pullspec = "example.com/foo/release:4.10.99-aarch64" from_image_stream = "4.10-art-assembly-4.10.99-arm64" - cmd_assert_async.reset_mock() + cmd_gather_async.reset_mock() await pipeline.build_release_image("4.10.99", "aarch64", previous_list, [], metadata, dest_pullspec, reference_release, from_image_stream, keep_manifest_list=False) expected_cmd = ['oc', 'adm', 'release', 'new', '-n', 'ocp-arm64', '--name=4.10.99', '--to-image=example.com/foo/release:4.10.99-aarch64', '--reference-mode=source', '--from-image-stream=4.10-art-assembly-4.10.99-arm64', '--previous=4.10.98,4.10.97,4.9.99', '--metadata', '{"description": "whatever", "url": "https://access.redhat.com/errata/RHBA-2099:2222"}'] - cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY) + cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY) @patch("pyartcd.jira.JIRAClient.from_url", return_value=None) @patch("pyartcd.pipelines.promote.PromotePipeline.tag_release", return_value=None) @@ -1046,13 +1046,11 @@ def test_build_create_symlink(self, _): dry_run=False ) pipeline = PromotePipeline(runtime, group="openshift-4.10", assembly="4.10.99", signing_env="prod") - temp_dir = tempfile.mkdtemp() - os.chdir(temp_dir) - open("openshift-client-linux-4.3.0-0.nightly-2019-12-06-161135.tar.gz", "w").close() - open("openshift-client-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz", "w").close() - open("openshift-install-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz", "w").close() - pipeline.create_symlink(temp_dir, False, False) - self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-linux.tar.gz'))) - self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-mac.tar.gz'))) - self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-install-mac.tar.gz'))) - shutil.rmtree(temp_dir) + with tempfile.TemporaryDirectory() as temp_dir: + Path(temp_dir, "openshift-client-linux-4.3.0-0.nightly-2019-12-06-161135.tar.gz").open("w").close() + Path(temp_dir, "openshift-client-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz").open("w").close() + Path(temp_dir, "openshift-install-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz").open("w").close() + pipeline.create_symlink(temp_dir, False, False) + self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-linux.tar.gz'))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-mac.tar.gz'))) + self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-install-mac.tar.gz')))