Merge pull request #420 from openvinotoolkit/improve-handling-for-con…

…nection-errors Add retry mechanism to better handle `ConnectionError`
openvinotoolkit · May 24, 2024 · 8e77344 · 8e77344
2 parents d9505d6 + 7abcbc8
commit 8e77344
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 18 deletions.
diff --git a/geti_sdk/http_session/geti_session.py b/geti_sdk/http_session/geti_session.py
@@ -301,23 +301,25 @@ def get_rest_response(
         else:
             self.headers.pop("x-geti-csrf-protection", "")
 
-        try:
-            response = self.request(**request_params, proxies=self._proxies)
-        except requests.exceptions.SSLError as error:
-            raise requests.exceptions.SSLError(
-                f"Connection to Intel® Geti™ server at '{self.config.host}' failed, "
-                f"the server address can be resolved but the SSL certificate could not "
-                f"be verified. \n Full error description: {error.args[-1]}"
-            )
-        except ConnectionError as conn_error:
-            if conn_error.args[0] == "Connection aborted.":
-                # We fake a response and try to establish a
-                # new connection by re-authenticating
-                response = Response()
-                response.status_code = 401
-                response.raw = conn_error.args[-1]
-            else:
-                raise conn_error
+        # Make the request, retrying a maximum of 5 times in case of connection errors
+        retries = 5
+        last_conn_error: Optional[ConnectionError] = None
+        while retries:
+            try:
+                response = self.request(**request_params, proxies=self._proxies)
+                break
+            except requests.exceptions.SSLError as error:
+                raise requests.exceptions.SSLError(
+                    f"Connection to Intel® Geti™ server at '{self.config.host}' failed, "
+                    f"the server address can be resolved but the SSL certificate could not "
+                    f"be verified. \n Full error description: {error.args[-1]}"
+                )
+            except ConnectionError as conn_error:
+                last_conn_error = conn_error
+                retries -= 1
+
+        if last_conn_error is not None:
+            raise last_conn_error
 
         response_content_type = response.headers.get("Content-Type", [])
         if (

diff --git a/tests/pre-merge/integration/test_geti.py b/tests/pre-merge/integration/test_geti.py
@@ -590,6 +590,9 @@ def test_post_inference_hooks(
         image_np = cv2.cvtColor(image_bgr, cv2.COLOR_BGR2RGB)
         _ = deployment.infer(image_np)
 
+        # Small delay to ensure that the hooks have time to run
+        time.sleep(1)
+
         dataset_client = DatasetClient(
             session=fxt_geti.session,
             workspace_id=fxt_geti.workspace_id,

diff --git a/tests/pre-merge/unit/deployment/test_prediction_converter.py b/tests/pre-merge/unit/deployment/test_prediction_converter.py
@@ -53,7 +53,6 @@ def coords_to_xmin_xmax_width_height(
     return x1, y1, x2 - x1, y2 - y1
 
 
-@pytest.mark.JobsComponent
 class TestInferenceResultsToPredictionConverter:
     def test_classification_to_prediction_converter(self, fxt_label_schema_factory):
         # Arrange