[uss_qualifier/scenarios/netrid] specify some check severity in test …

…scenario documentation only (contrib #404) (#882)

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/heavy_traffic_concurrent.py

@@ -19,7 +19,6 @@
 from monitoring.monitorlib.mutate.rid import ChangedISA
 from monitoring.monitorlib.rid import RIDVersion
 from monitoring.prober.infrastructure import register_resource_type
-from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource
 from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource
 from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource
@@ -163,7 +162,6 @@ def _get_isas_by_id_concurrent_step(self):
                 if fetched_isa.status_code != 200:
                     main_check.record_failed(
                         f"ISA retrieval query failed for {isa_id}",
-                        severity=Severity.High,
                         details=f"ISA retrieval query for {isa_id} yielded code {fetched_isa.status_code}",
                     )
 
@@ -312,7 +310,6 @@ def _create_isas_concurrent_step(self):
                 if changed_isa.query.response.code != 200:
                     main_check.record_failed(
                         f"ISA creation failed for {isa_id}",
-                        severity=Severity.High,
                         details=f"ISA creation for {isa_id} returned {changed_isa.query.response.code}",
                     )
                 else:
@@ -347,7 +344,6 @@ def _search_area_step(self):
                     if isa_id not in isas.isas.keys():
                         sub_check.record_failed(
                             f"ISAs search did not return ISA {isa_id} that was just created",
-                            severity=Severity.High,
                             details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()} and is missing some of the created ISAs",
                             query_timestamps=[isas.dss_query.query.request.timestamp],
                         )
@@ -387,7 +383,6 @@ def _delete_isas(self):
                 if deleted_isa.query.response.code != 200:
                     main_check.record_failed(
                         f"ISA deletion failed for {isa_id}",
-                        severity=Severity.High,
                         details=f"ISA deletion for {isa_id} returned {deleted_isa.query.response.code}",
                     )
 
@@ -421,7 +416,6 @@ def _get_deleted_isas(self):
                 if fetched_isa.status_code != 404:
                     check.record_failed(
                         f"ISA retrieval succeeded for {isa_id}",
-                        severity=Severity.High,
                         details=f"ISA retrieval for {isa_id} returned {fetched_isa.status_code}",
                         query_timestamps=[fetched_isa.query.request.timestamp],
                     )
@@ -442,7 +436,6 @@ def _search_deleted_isas(self):
                 if isa_id in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search returned deleted ISA {isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()} that contained some of the ISAs we had previously deleted.",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )

monitoring/uss_qualifier/scenarios/astm/netrid/common/dss/isa_simple.py

@@ -7,7 +7,6 @@
 from monitoring.monitorlib.fetch import rid as fetch
 from monitoring.monitorlib.mutate import rid as mutate
 from monitoring.prober.infrastructure import register_resource_type
-from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstanceResource
 from monitoring.uss_qualifier.resources.interuss.id_generator import IDGeneratorResource
 from monitoring.uss_qualifier.resources.netrid.service_area import ServiceAreaResource
@@ -87,8 +86,7 @@ def _delete_isa_if_exists(self):
             if not fetched.success and fetched.status_code != 404:
                 check.record_failed(
                     "ISA information could not be retrieved",
-                    Severity.High,
-                    f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {self._isa_id}",
+                    details=f"{self._dss.participant_id} DSS instance returned {fetched.status_code} when queried for ISA {self._isa_id}",
                     query_timestamps=[fetched.query.request.timestamp],
                 )
 
@@ -109,8 +107,7 @@ def _delete_isa_if_exists(self):
                 if not deleted.dss_query.success:
                     check.record_failed(
                         "Could not delete pre-existing ISA",
-                        Severity.High,
-                        f"Attempting to delete ISA {self._isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}",
+                        details=f"Attempting to delete ISA {self._isa_id} from the {self._dss.participant_id} DSS returned error {deleted.dss_query.status_code}",
                         query_timestamps=[deleted.dss_query.query.request.timestamp],
                     )
             for subscriber_url, notification in deleted.notifications.items():
@@ -127,8 +124,7 @@ def _delete_isa_if_exists(self):
                         if not notification.success:
                             check.record_failed(
                                 "Could not notify ISA subscriber",
-                                Severity.Medium,
-                                f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}",
+                                details=f"Attempting to notify subscriber for ISA {self._isa_id} at {subscriber_url} resulted in {notification.status_code}",
                                 query_timestamps=[notification.query.request.timestamp],
                             )
 
@@ -149,8 +145,7 @@ def _get_isa_by_id_step(self):
             ):
                 check.record_failed(
                     "DSS returned ISA with incorrect version",
-                    Severity.High,
-                    f"DSS should have returned an ISA with the version {self._isa_version}, but instead the ISA returned had the version {fetched.isa.version}",
+                    details=f"DSS should have returned an ISA with the version {self._isa_version}, but instead the ISA returned had the version {fetched.isa.version}",
                     query_timestamps=[fetched.query.request.timestamp],
                 )
 
@@ -230,7 +225,6 @@ def _search_earliest_incl_step():
                 if self._isa_id not in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search did not return expected ISA {self._isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} from time {earliest} returned ISAs {isas.isas.keys()}",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )
@@ -258,7 +252,6 @@ def _search_earliest_excl_step():
                 if self._isa_id in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search returned unexpected ISA {self._isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} from time {earliest} returned ISAs {isas.isas.keys()}",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )
@@ -286,7 +279,6 @@ def _search_latest_incl_step():
                 if self._isa_id not in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search did not return expected ISA {self._isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} to time {latest} returned ISAs {isas.isas.keys()}",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )
@@ -314,7 +306,6 @@ def _search_latest_excl_step():
                 if self._isa_id in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search returned unexpected ISA {self._isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} to time {latest} returned ISAs {isas.isas.keys()}",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )
@@ -340,7 +331,6 @@ def _search_area_only_step():
                 if self._isa_id not in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search did not return expected ISA {self._isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()}",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )
@@ -483,7 +473,6 @@ def _search_isa_step():
                 if self._isa_id in isas.isas.keys():
                     check.record_failed(
                         f"ISAs search returned deleted ISA {self._isa_id}",
-                        severity=Severity.High,
                         details=f"Search in area {self._isa_area} returned ISAs {isas.isas.keys()}",
                         query_timestamps=[isas.dss_query.query.request.timestamp],
                     )

monitoring/uss_qualifier/scenarios/astm/netrid/common/nominal_behavior.py

@@ -5,7 +5,6 @@
 
 from monitoring.monitorlib.errors import stacktrace_string
 from monitoring.monitorlib.rid import RIDVersion
-from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.astm.f3411.dss import DSSInstancesResource
 from monitoring.uss_qualifier.resources.netrid import (
     FlightDataResource,
@@ -139,7 +138,6 @@ def cleanup(self):
                 stacktrace = stacktrace_string(e)
                 check.record_failed(
                     summary="Error while trying to delete test flight",
-                    severity=Severity.Medium,
                     details=f"While trying to delete a test flight from {sp.participant_id}, encountered error:\n{stacktrace}",
                 )
         self.end_cleanup()

monitoring/uss_qualifier/scenarios/astm/netrid/display_data_evaluator.py

@@ -323,7 +323,6 @@ def _evaluate_observation(
                 check.record_failed(
                     summary="Observation failed",
                     details=f"When queried for an observation in {_rect_str(rect)}, {observer.participant_id} returned code {query.status_code}",
-                    severity=Severity.Medium,
                     query_timestamps=[query.request.timestamp],
                 )
                 return
@@ -360,7 +359,6 @@ def _evaluate_normal_observation(
             if duplicates:
                 check.record_failed(
                     "Duplicate flight IDs in observation",
-                    Severity.Medium,
                     details="Duplicate flight IDs observed: " + ", ".join(duplicates),
                     query_timestamps=[query.request.timestamp],
                 )
@@ -444,7 +442,6 @@ def _evaluate_normal_observation(
                 ):
                     check.record_failed(
                         "Position Data is using extrapolation when Telemetry is available.",
-                        Severity.Medium,
                         details=(
                             f"{mapping.injected_flight.uss_participant_id}'s flight with injection ID "
                             f"{mapping.injected_flight.flight.injection_id} in test {mapping.injected_flight.test_id} had telemetry index {mapping.telemetry_index} at {injected_telemetry.timestamp} "
@@ -474,7 +471,6 @@ def _evaluate_normal_observation(
                     check.record_failed(
                         summary=f"Observation of details failed for {mapping.observed_flight.id}",
                         details=f"When queried for details of observation (ID {mapping.observed_flight.id}), {observer.participant_id} returned code {query.status_code}",
-                        severity=Severity.Medium,
                         query_timestamps=[query.request.timestamp],
                     )
                 else:
@@ -508,7 +504,6 @@ def _evaluate_area_too_large_observation(
                 check.record_failed(
                     summary="Did not receive expected error code for too-large area request",
                     details=f"{observer.participant_id} was queried for flights in {_rect_str(rect)} with a diagonal of {diagonal} which is larger than the maximum allowed diagonal of {self._rid_version.max_diagonal_km}.  The expected error code is 413, but instead code {query.status_code} was received.",
-                    severity=Severity.High,
                     query_timestamps=[query.request.timestamp],
                 )
 
@@ -760,7 +755,6 @@ def _evaluate_sp_observation(
             if not sp_observation.dss_isa_query.success:
                 check.record_failed(
                     summary="Could not query ISAs from DSS",
-                    severity=Severity.Medium,
                     details=f"Query to {self._dss.participant_id}'s DSS at {sp_observation.dss_isa_query.query.request.url} failed {sp_observation.dss_isa_query.query.status_code}",
                     query_timestamps=[
                         sp_observation.dss_isa_query.query.request.initiated_at.datetime
@@ -829,7 +823,6 @@ def _evaluate_normal_sp_observation(
                 if errors:
                     check.record_failed(
                         summary="/flights response failed schema validation",
-                        severity=Severity.Medium,
                         details="The response received from querying the /flights endpoint failed validation against the required OpenAPI schema:\n"
                         + "\n".join(
                             f"At {e.json_path} in the response: {e.message}"
@@ -1090,7 +1083,6 @@ def _evaluate_normal_sp_observation(
                 if not details_query.success:
                     check.record_failed(
                         summary="Flight details query not successful",
-                        severity=Severity.Medium,
                         details=f"Flight details query to {details_query.query.request.url} failed {details_query.status_code}",
                         query_timestamps=[details_query.query.request.timestamp],
                     )
@@ -1106,7 +1098,6 @@ def _evaluate_normal_sp_observation(
                 if errors:
                     check.record_failed(
                         summary="Flight details response failed schema validation",
-                        severity=Severity.Medium,
                         details="The response received from querying the flight details endpoint failed validation against the required OpenAPI schema:\n"
                         + "\n".join(
                             f"At {e.json_path} in the response: {e.message}"
@@ -1134,7 +1125,6 @@ def _evaluate_area_too_large_sp_observation(
                 check.record_failed(
                     summary="Flight discovered using too-large area request",
                     details=f"{mapping.injected_flight.uss_participant_id} was queried for flights in {_rect_str(rect)} with a diagonal of {diagonal} km which is larger than the maximum allowed diagonal of {self._rid_version.max_diagonal_km} km.  The expected error code is 413, but instead a valid response containing the expected flight was received.",
-                    severity=Severity.High,
                     query_timestamps=[
                         mapping.observed_flight.query.query.request.timestamp
                     ],
@@ -1154,7 +1144,6 @@ def _evaluate_sp_flight_recent_positions_times(
                     check.record_failed(
                         "A Position timestamp was older than the tolerance.",
                         details=f"Position timestamp: {p.time}, query time: {query_time}",
-                        severity=Severity.Medium,
                     )
 
     def _evaluate_sp_flight_recent_positions_crossing_area_boundary(
@@ -1169,7 +1158,6 @@ def fail_check():
                 check.record_failed(
                     "A position outside the area was neither preceded nor followed by a position inside the area.",
                     details=f"Positions: {f.recent_positions}, requested_area: {requested_area}",
-                    severity=Severity.Medium,
                 )
 
             positions = _chronological_positions(f)

monitoring/uss_qualifier/scenarios/astm/netrid/dss_wrapper.py

@@ -360,7 +360,6 @@ def put_isa(
             if mutated_isa.dss_query.query.status_code == 201:
                 sub_check.record_failed(
                     summary=f"PUT ISA returned technically-incorrect 201",
-                    severity=Severity.Low,
                     details="DSS should return 200 from PUT ISA, but instead returned the reasonable-but-technically-incorrect code 201",
                     query_timestamps=[t_dss],
                 )

monitoring/uss_qualifier/scenarios/astm/netrid/injection.py

@@ -12,7 +12,6 @@
     TestFlight,
     CreateTestParameters,
 )
-from monitoring.uss_qualifier.common_data_definitions import Severity
 from monitoring.uss_qualifier.resources.netrid import (
     FlightDataResource,
     NetRIDServiceProviders,
@@ -60,15 +59,13 @@ def inject_flights(
             if query.status_code != 200:
                 check.record_failed(
                     summary="Error while trying to inject test flight",
-                    severity=Severity.High,
                     details=f"Expected response code 200 from {target.participant_id} but received {query.status_code} while trying to inject a test flight",
                     query_timestamps=[query.request.timestamp],
                 )
 
             if "json" not in query.response or query.response.json is None:
                 check.record_failed(
                     summary="Response to test flight injection request did not contain a JSON body",
-                    severity=Severity.High,
                     details=f"Expected a JSON body in response to flight injection request",
                     query_timestamps=[query.request.timestamp],
                 )
@@ -115,7 +112,6 @@ def inject_flights(
         if errors:
             check.record_failed(
                 summary="Injected flights not suitable for test",
-                severity=Severity.High,
                 details="When checking the suitability of the flights (as injected) for the test, found:\n"
                 + "\n".join(errors),
                 query_timestamps=[f.query_timestamp for f in injected_flights],