Merge pull request #1274 from vfreex/retry-on-502

promote-assembly: retry on 502

pyartcd/pyartcd/pipelines/promote.py

@@ -30,14 +30,12 @@
 from artcommonlib.util import isolate_major_minor_in_group
 from pyartcd.locks import Lock
 from pyartcd.signatory import AsyncSignatory, SigstoreSignatory
-from pyartcd.util import nightlies_with_pullspecs
 from pyartcd import constants, locks, util, jenkins
 from artcommonlib import exectools
 from pyartcd.cli import cli, click_coroutine, pass_runtime
 from artcommonlib.exceptions import VerificationError
 from pyartcd.jira import JIRAClient
 from pyartcd.mail import MailService
-from pyartcd.s3 import sync_dir_to_s3_mirror
 from pyartcd.oc import get_release_image_info, get_release_image_pullspec, extract_release_binary, \
     extract_release_client_tools, get_release_image_info_from_pullspec, extract_baremetal_installer
 from pyartcd.runtime import Runtime, GroupRuntime
@@ -1396,8 +1394,11 @@ async def build_release_image(self, release_name: str, arch: str, previous_list:
         env = os.environ.copy()
         env["GOTRACEBACK"] = "all"
         self._logger.info("Running %s", " ".join(cmd))
-        await exectools.cmd_assert_async(cmd, env=env, stdout=sys.stderr)
-        pass
+        return await retry(
+            reraise=True,
+            stop=stop_after_attempt(10),  # retry 10 times
+            wait=wait_fixed(30),  # wait for 30 seconds between retries
+        )(exectools.cmd_gather_async)(cmd, env=env)
 
     @staticmethod
     async def get_image_stream(namespace: str, image_stream: str):

pyartcd/tests/pipelines/test_promote.py

@@ -621,8 +621,8 @@ async def test_promote_arch(self, get_image_stream: AsyncMock, get_release_image
         tag_release.assert_not_awaited()
 
     @patch("pyartcd.jira.JIRAClient.from_url", return_value=None)
-    @patch("pyartcd.pipelines.promote.exectools.cmd_assert_async", return_value=0)
-    async def test_build_release_image_from_reference_release(self, cmd_assert_async: AsyncMock, _):
+    @patch("pyartcd.pipelines.promote.exectools.cmd_gather_async", return_value=0)
+    async def test_build_release_image_from_reference_release(self, cmd_gather_async: AsyncMock, _):
         runtime = MagicMock(
             config={
                 "build_config": {
@@ -645,29 +645,29 @@ async def test_build_release_image_from_reference_release(self, cmd_assert_async
         await pipeline.build_release_image("4.10.99", "x86_64", previous_list, [],
                                            metadata, dest_pullspec, reference_release, None, keep_manifest_list=False)
         expected_cmd = ["oc", "adm", "release", "new", "-n", "ocp", "--name=4.10.99", "--to-image=example.com/foo/release:4.10.99-x86_64", f"--from-release={reference_release}", "--previous=4.10.98,4.10.97,4.9.99", "--metadata", "{\"description\": \"whatever\", \"url\": \"https://access.redhat.com/errata/RHBA-2099:2222\"}"]
-        cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY)
+        cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY)
 
         # test aarch64
         reference_release = "registry.ci.openshift.org/ocp-arm64/release-arm64:whatever-aarch64"
         dest_pullspec = "example.com/foo/release:4.10.99-aarch64"
-        cmd_assert_async.reset_mock()
+        cmd_gather_async.reset_mock()
         await pipeline.build_release_image("4.10.99", "aarch64", previous_list, [],
                                            metadata, dest_pullspec, reference_release, None, keep_manifest_list=False)
         expected_cmd = ["oc", "adm", "release", "new", "-n", "ocp-arm64", "--name=4.10.99", "--to-image=example.com/foo/release:4.10.99-aarch64", f"--from-release={reference_release}", "--previous=4.10.98,4.10.97,4.9.99", "--metadata", "{\"description\": \"whatever\", \"url\": \"https://access.redhat.com/errata/RHBA-2099:2222\"}"]
-        cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY)
+        cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY)
 
         # test multi-aarch64
         reference_release = "registry.ci.openshift.org/ocp-arm64/release-arm64:whatever-multi-aarch64"
         dest_pullspec = "example.com/foo/release:4.10.99-multi-aarch64"
-        cmd_assert_async.reset_mock()
+        cmd_gather_async.reset_mock()
         await pipeline.build_release_image("4.10.99", "aarch64", previous_list, [],
                                            metadata, dest_pullspec, reference_release, None, keep_manifest_list=True)
         expected_cmd = ["oc", "adm", "release", "new", "-n", "ocp-arm64", "--name=4.10.99", "--to-image=example.com/foo/release:4.10.99-multi-aarch64", f"--from-release={reference_release}", "--keep-manifest-list", "--previous=4.10.98,4.10.97,4.9.99", "--metadata", "{\"description\": \"whatever\", \"url\": \"https://access.redhat.com/errata/RHBA-2099:2222\"}"]
-        cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY)
+        cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY)
 
     @patch("pyartcd.jira.JIRAClient.from_url", return_value=None)
-    @patch("pyartcd.pipelines.promote.exectools.cmd_assert_async", return_value=0)
-    async def test_build_release_image_from_image_stream(self, cmd_assert_async: AsyncMock, _):
+    @patch("pyartcd.pipelines.promote.exectools.cmd_gather_async", return_value=0)
+    async def test_build_release_image_from_image_stream(self, cmd_gather_async: AsyncMock, _):
         runtime = MagicMock(config={"build_config": {"ocp_build_data_url": "https://example.com/ocp-build-data.git"},
                                     "jira": {"url": "https://issues.redhat.com/"}},
                             working_dir=Path("/path/to/working"), dry_run=False)
@@ -682,17 +682,17 @@ async def test_build_release_image_from_image_stream(self, cmd_assert_async: Asy
         await pipeline.build_release_image("4.10.99", "x86_64", previous_list, [],
                                            metadata, dest_pullspec, reference_release, from_image_stream, keep_manifest_list=False)
         expected_cmd = ['oc', 'adm', 'release', 'new', '-n', 'ocp', '--name=4.10.99', '--to-image=example.com/foo/release:4.10.99-x86_64', '--reference-mode=source', '--from-image-stream=4.10-art-assembly-4.10.99', '--previous=4.10.98,4.10.97,4.9.99', '--metadata', '{"description": "whatever", "url": "https://access.redhat.com/errata/RHBA-2099:2222"}']
-        cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY)
+        cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY)
 
         # test aarch64
         reference_release = None
         dest_pullspec = "example.com/foo/release:4.10.99-aarch64"
         from_image_stream = "4.10-art-assembly-4.10.99-arm64"
-        cmd_assert_async.reset_mock()
+        cmd_gather_async.reset_mock()
         await pipeline.build_release_image("4.10.99", "aarch64", previous_list, [],
                                            metadata, dest_pullspec, reference_release, from_image_stream, keep_manifest_list=False)
         expected_cmd = ['oc', 'adm', 'release', 'new', '-n', 'ocp-arm64', '--name=4.10.99', '--to-image=example.com/foo/release:4.10.99-aarch64', '--reference-mode=source', '--from-image-stream=4.10-art-assembly-4.10.99-arm64', '--previous=4.10.98,4.10.97,4.9.99', '--metadata', '{"description": "whatever", "url": "https://access.redhat.com/errata/RHBA-2099:2222"}']
-        cmd_assert_async.assert_awaited_once_with(expected_cmd, env=ANY, stdout=ANY)
+        cmd_gather_async.assert_awaited_once_with(expected_cmd, env=ANY)
 
     @patch("pyartcd.jira.JIRAClient.from_url", return_value=None)
     @patch("pyartcd.pipelines.promote.PromotePipeline.tag_release", return_value=None)
@@ -1046,13 +1046,11 @@ def test_build_create_symlink(self, _):
             dry_run=False
         )
         pipeline = PromotePipeline(runtime, group="openshift-4.10", assembly="4.10.99", signing_env="prod")
-        temp_dir = tempfile.mkdtemp()
-        os.chdir(temp_dir)
-        open("openshift-client-linux-4.3.0-0.nightly-2019-12-06-161135.tar.gz", "w").close()
-        open("openshift-client-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz", "w").close()
-        open("openshift-install-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz", "w").close()
-        pipeline.create_symlink(temp_dir, False, False)
-        self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-linux.tar.gz')))
-        self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-mac.tar.gz')))
-        self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-install-mac.tar.gz')))
-        shutil.rmtree(temp_dir)
+        with tempfile.TemporaryDirectory() as temp_dir:
+            Path(temp_dir, "openshift-client-linux-4.3.0-0.nightly-2019-12-06-161135.tar.gz").open("w").close()
+            Path(temp_dir, "openshift-client-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz").open("w").close()
+            Path(temp_dir, "openshift-install-mac-4.3.0-0.nightly-2019-12-06-161135.tar.gz").open("w").close()
+            pipeline.create_symlink(temp_dir, False, False)
+            self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-linux.tar.gz')))
+            self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-client-mac.tar.gz')))
+            self.assertTrue(os.path.exists(os.path.join(temp_dir, 'openshift-install-mac.tar.gz')))