Tickets/DM-46983: Use DefaultingValidator output to render the calibrations configuration in BaseCalsys, new defaults values and cleanups

doc/news/DM-46983.bugfix.rst

@@ -0,0 +1 @@
+In ``BaseCalsys.load_calibration_config_file``, fix schema validation to update configurations with default values applied by ``salobj.DefaultingValidator``.

doc/news/DM-46983.feature.1.rst

@@ -0,0 +1 @@
+Add new default values for ``ATCalsys`` configurations in ``atcalsys_schema.yaml``.

doc/news/DM-46983.feature.2.rst

@@ -0,0 +1 @@
+Cleanup of ``ATCalsys`` configuration file ``atcalsys.yaml`` by removing attributes that use default values.

doc/news/DM-46983.feature.3.rst

@@ -0,0 +1,2 @@
+Add new default values for ``MTCalsys`` configurations in ``mtcalsys_schema.yaml``.
+

doc/news/DM-46983.feature.4.rst

@@ -0,0 +1,2 @@
+Cleanup of ``MTCalsys`` configuration file ``mtcalsys.yaml`` by removing attributes that use default values.
+

python/lsst/ts/observatory/control/base_calsys.py

@@ -238,16 +238,27 @@ def load_calibration_config_file(self, filename: str | None = None) -> None:
             config_validator = salobj.DefaultingValidator(schema=yaml.safe_load(f))
 
         validation_errors = ""
+        log_defaults = ""
         for item in self.calibration_config:
+            config_original = dict(self.calibration_config[item])
             try:
-                config_validator.validate(self.calibration_config[item])
+                self.calibration_config[item] = config_validator.validate(
+                    self.calibration_config[item]
+                )
             except jsonschema.ValidationError as e:
                 validation_errors += f"\t{item} failed validation: {e.message}.\n"
                 self.log.exception(f"{item} failed validation.")
+            config_with_defaults = self.calibration_config[item]
+            defaulted_attributes = set(config_with_defaults) - set(config_original)
+            log_defaults += f"\n{item}:\n" + "\n".join(
+                f"    {attr}: {config_with_defaults[attr]}"
+                for attr in defaulted_attributes
+            )
         if validation_errors:
             raise RuntimeError(
                 f"Failed schema validation:\n{validation_errors}Check logs for more information."
             )
+        self.log.debug(f"\n=== Applied Default Values ===\n{log_defaults}\n")
 
     def get_calibration_configuration(self, name: str) -> dict[str, typing.Any]:
         """Returns the configuration attributes given a configuration

python/lsst/ts/observatory/control/data/atcalsys.yaml

@@ -3,56 +3,30 @@
 
 scan_g:
   calib_type: Mono
-  use_camera: true
   atspec_filter: SDSSg_65mm
-  atspec_grating: empty_1
   wavelength: 475.0
   wavelength_width: 250
-  wavelength_resolution: 5.0
   monochromator_grating: RED
   exit_slit: 5.0
   entrance_slit: 5.0
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
-  use_fiberspectrograph: true
-  use_electrometer: true
   exposure_times:
     - 15.0
 
 scan_r:
   calib_type: Mono
-  use_camera: true
-  atspec_filter: SDSSr_65mm
-  atspec_grating: empty_1
   wavelength: 625.0
   wavelength_width: 250
-  wavelength_resolution: 5.0
   monochromator_grating: RED
   exit_slit: 5.0
   entrance_slit: 5.0
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
-  use_fiberspectrograph: true
-  use_electrometer: true
   exposure_times:
     - 15.0
 
 at_whitelight_r:
-  calib_type: WhiteLight
-  use_camera: true
-  atspec_filter: SDSSr_65mm
   atspec_grating: empty_1  
-  wavelength: 500.0
   monochromator_grating: MIRROR 
   exit_slit: 5.0
   entrance_slit: 5.0
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
-  use_fiberspectrograph: true
-  use_electrometer: true
   exposure_times:
     - 6.0
     - 6.0
@@ -77,19 +51,7 @@ at_whitelight_r:
     - 6.0
 
 ptc_1:
-  calib_type: WhiteLight
-  use_camera: true
   use_fiberspectrograph: false 
-  use_electrometer: true
-  atspec_filter: SDSSr_65mm
-  atspec_grating: empty_1
-  wavelength: 500.0
-  monochromator_grating: null
-  exit_slit: 7.0
-  entrance_slit: 7.0
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 14.40
     - 14.40
@@ -161,19 +123,7 @@ ptc_1:
     - 0.93
 
 ptc_2:
-  calib_type: WhiteLight
-  use_camera: true
   use_fiberspectrograph: false 
-  use_electrometer: true
-  atspec_filter: SDSSr_65mm
-  atspec_grating: empty_1
-  wavelength: 500.0
-  monochromator_grating: null
-  exit_slit: 7.0
-  entrance_slit: 7.0
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.35
     - 15.35
@@ -244,19 +194,7 @@ ptc_2:
 
 
 ptc_3:
-  calib_type: WhiteLight
-  use_camera: true
   use_fiberspectrograph: false 
-  use_electrometer: true
-  atspec_filter: SDSSr_65mm
-  atspec_grating: empty_1
-  wavelength: 500.0
-  monochromator_grating: null
-  exit_slit: 7.0
-  entrance_slit: 7.0
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 0.52
     - 0.52

python/lsst/ts/observatory/control/data/atcalsys_schema.yaml

@@ -11,15 +11,19 @@ properties:
     enum:
       - Mono
       - WhiteLight
+    default: WhiteLight
   use_camera:
     description: Indicates whether the operation requires a camera.
     type: boolean
+    default: true
   atspec_filter:
     description: Name of the filter to select for the spectrograph.
     type: string
+    default: SDSSr_65mm
   atspec_grating:
     description: Name of the grating to select for the spectrograph.
     type: string
+    default: empty_1
   wavelength:
     description: >-
       Center wavelength value in nm, used for configuring the monochromator
@@ -32,13 +36,15 @@ properties:
       monochromator for the flat-field calibration sequence when using
       monochromatic light.
     type: number
+    default: 0.0
   wavelength_resolution:
     description: >-
       Optional. When using a monochromatic light source, it defines the
       resolution used to sample values within the scan range, which has a width
       defined by `wavelength_width` and is centered around the `wavelength`
       attribute.
     type: number
+    default: 5.0
   monochromator_grating:
     description: >-
       Select the grating of the monochromator that will be in the path of
@@ -75,6 +81,7 @@ properties:
       (default) integration = 1PLC; High Accuracy integration=10PLC. Here the
       integration is set in seconds.
     type: number
+    default: 0.1
   electrometer_mode:
     description: >-
       Set electrometer to use different modes. The units recorded will be Amps
@@ -86,6 +93,7 @@ properties:
       - CHARGE
       - VOLTAGE
       - RESISTANCE
+    default: CURRENT
   electrometer_range:
     description: >-
       Set measurement range, which effects the accuracy of measurements and the
@@ -99,14 +107,17 @@ properties:
       from 0 to 21e-3 Amps, Resistance from 0 to 100e18 Ohms, Charge from 0 to
       +2.1e-6 Coulombs.
     type: number
+    default: -1
   use_fiberspectrograph:
     description: >-
       Identifies if the fiberspectrograph will be used in the exposure.
     type: boolean
+    default: true
   use_electrometer:
     description: >-
       Identifies if the electrometer will be used in the exposure.
     type: boolean
+    default: true
   exposure_times:
     description: >-
       List of Camera exposure times, in secs

python/lsst/ts/observatory/control/data/mtcalsys.yaml

@@ -2,131 +2,70 @@
 # Created 2024-07-15
 
 whitelight_u:
-  calib_type: WhiteLight
-  use_camera: true
   mtcamera_filter: u
   led_name: 
     - M385L3
   wavelength: 385.0
   led_location: 210.21
   led_focus: 50.68
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0
 
 whitelight_g:
-  calib_type: WhiteLight
-  use_camera: true
   mtcamera_filter: g
   led_name:
     - M455L4
     - M505L4
   wavelength: 480.0
   led_location: 9.15
   led_focus: 17.029
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0
 
 whitelight_r:
-  calib_type: WhiteLight
-  use_camera: true
-  mtcamera_filter: r
   led_name:
     - M565L3
     - M660L4
   wavelength: 612.5
   led_location: 70.40
   led_focus: 16.279
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0
 
 whitelight_i:
-  calib_type: WhiteLight
-  use_camera: true
   mtcamera_filter: i
   led_name:
     - M730L5
     - M780LP1
   wavelength: 755.0
   led_location: 237.36
   led_focus: 15.829
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0
 
 whitelight_z:
-  calib_type: WhiteLight
-  use_camera: true
   mtcamera_filter: z
   led_name:
     - M850L3
     - M940L3
   wavelength: 895.0
   led_location: 299.034
   led_focus: 15.505
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0
 
 whitelight_y:
-  calib_type: WhiteLight
-  use_camera: true
   mtcamera_filter: y
   led_name:
     - M970L4
   wavelength: 970.0
-  led_location: 174.91
-  led_focus: 15.380
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0
 
 scan_r:
   calib_type: Mono
-  use_camera: true
-  mtcamera_filter: r
-  led_location: 174.91
-  led_focus: 15.380
   wavelength: 625.0
   wavelength_width: 250
   wavelength_resolution: 5.0
-  use_electrometer: true
-  use_fiberspectrograph_red: true
-  use_fiberspectrograph_blue: true
-  electrometer_integration_time: 0.1
-  electrometer_mode: CURRENT
-  electrometer_range: -1
   exposure_times:
     - 15.0