From 71a9df7265a7419463ad69688bd131fb6f82c50b Mon Sep 17 00:00:00 2001 From: Erik Jaegervall Date: Fri, 19 Jan 2024 16:05:47 +0100 Subject: [PATCH] Adding datatype checks If a unit specifies allowed datatypes we will check that only those types are used Signed-off-by: Erik Jaegervall --- tests/model/explicit_units.yaml | 2 + tests/model/test_vsstree.py | 92 +++++++++++++++++++++++++++++++++ vspec/model/constants.py | 32 ++++++++++-- vspec/model/vsstree.py | 39 +++++++++----- 4 files changed, 148 insertions(+), 17 deletions(-) diff --git a/tests/model/explicit_units.yaml b/tests/model/explicit_units.yaml index a2f1e50e..fdfd0a23 100644 --- a/tests/model/explicit_units.yaml +++ b/tests/model/explicit_units.yaml @@ -2,7 +2,9 @@ puncheon: definition: Volume measure in puncheons (1 puncheon = 318 liters) unit: Puncheon quantity: volume + allowed_datatypes: ['uint16', 'int16'] hogshead: definition: Volume measure in hogsheads (1 hogshead = 238 liters) unit: Hogshead quantity: volume + allowed_datatypes: ['numeric'] diff --git a/tests/model/test_vsstree.py b/tests/model/test_vsstree.py index 36653b88..c59cc574 100644 --- a/tests/model/test_vsstree.py +++ b/tests/model/test_vsstree.py @@ -8,6 +8,7 @@ import unittest import os +import pytest from vspec.model.constants import VSSType, VSSDataType, VSSUnitCollection, VSSTreeType from vspec.model.vsstree import VSSNode @@ -79,6 +80,7 @@ def test_merge_nodes(self): "datatype": "uint8", "unit": "hogshead", "min": 0, "max": 100, "$file_name$": "testfile"} unit_file = os.path.join(os.path.dirname(__file__), 'explicit_units.yaml') + VSSUnitCollection.reset_units() VSSUnitCollection.load_config_file(unit_file) node_target = VSSNode( @@ -107,6 +109,96 @@ def test_merge_nodes(self): self.assertEqual(0, node_target.min) self.assertEqual(100, node_target.max) + def test_unit_datatype(self): + + unit_file = os.path.join(os.path.dirname(__file__), 'explicit_units.yaml') + VSSUnitCollection.reset_units() + VSSUnitCollection.load_config_file(unit_file) + + # int16 explicitly listed + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "int16", "unit": "puncheon", "min": 0, "max": 100, "$file_name$": "testfile"} + + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + assert node_source.unit == VSSUnitCollection.get_unit("puncheon") + + # uint16 explicitly listed + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "uint16", "unit": "puncheon", "min": 0, "max": 100, "$file_name$": "testfile"} + + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + assert node_source.unit == VSSUnitCollection.get_unit("puncheon") + + # uint16[] is ok as uint16 is explicitly listed + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "uint16[]", "unit": "puncheon", "min": 0, "max": 100, "$file_name$": "testfile"} + + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + assert node_source.unit == VSSUnitCollection.get_unit("puncheon") + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "float", "unit": "puncheon", "min": 0, "max": 100, "$file_name$": "testfile"} + + # float not ok + + with pytest.raises(SystemExit): + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + + # Hogshead defined as "numeric", i.e. all numeric types shall be accepted + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "uint16", "unit": "hogshead", "min": 0, "max": 100, "$file_name$": "testfile"} + + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + assert node_source.unit == VSSUnitCollection.get_unit("hogshead") + + # Also array of numeric + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "uint16[]", "unit": "hogshead", "min": 0, "max": 100, "$file_name$": "testfile"} + + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + assert node_source.unit == VSSUnitCollection.get_unit("hogshead") + + # But not string + + source = {"description": "some desc", "type": "sensor", + "uuid": "2cc90035-e1c2-43bf-a394-1a439addc8ad", + "datatype": "string", "unit": "hogshead", "min": 0, "max": 100, "$file_name$": "testfile"} + + with pytest.raises(SystemExit): + node_source = VSSNode( + "MyNode2", + source, + VSSTreeType.SIGNAL_TREE.available_types()) + def test_tree_find(self): """ Test if tree can be found within VSS tree diff --git a/vspec/model/constants.py b/vspec/model/constants.py index 5ad6b7ed..a0bbdaed 100644 --- a/vspec/model/constants.py +++ b/vspec/model/constants.py @@ -18,7 +18,7 @@ import sys from enum import Enum, EnumMeta from typing import ( - Sequence, Type, TypeVar, Optional, Dict, TextIO + Sequence, Type, TypeVar, Optional, Dict, TextIO, List ) from collections import abc @@ -37,14 +37,16 @@ class VSSUnit(str): unit: Optional[str] = None # Typically full name like "Volt" definition: Optional[str] = None quantity: Optional[str] = None # Typically quantity, like "Voltage" + allowed_datatypes: Optional[List[str]] = None # Typically quantity, like "Voltage" def __new__(cls, id: str, unit: Optional[str] = None, definition: Optional[str] = None, - quantity: Optional[str] = None) -> VSSUnit: + quantity: Optional[str] = None, allowed_datatypes: Optional[List[str]] = None) -> VSSUnit: self = super().__new__(cls, id) self.id = id self.unit = unit self.definition = definition self.quantity = quantity + self.allowed_datatypes = allowed_datatypes return self @property @@ -148,6 +150,16 @@ class VSSDataType(Enum, metaclass=EnumMetaWithReverseLookup): DOUBLE_ARRAY = "double[]" STRING_ARRAY = "string[]" + @classmethod + def is_numeric(cls, datatype): + """ + Return true if this datatype accepts numerical values + """ + if datatype in [VSSDataType.STRING, VSSDataType.STRING_ARRAY, + VSSDataType.BOOLEAN, VSSDataType.BOOLEAN_ARRAY]: + return False + return True + class VSSUnitCollection(): units: Dict[str, VSSUnit] = dict() @@ -203,7 +215,21 @@ def load_config_file(cls, config_file: str) -> int: logging.info("Quantity %s used by unit %s has not been defined", quantity, k) VSSQuantityCollection.add_quantity(quantity) - unit_node = VSSUnit(k, unit, definition, quantity) + allowed_datatypes = None + + if "allowed_datatypes" in v: + allowed_datatypes = v["allowed_datatypes"] + for datatype in allowed_datatypes: + if datatype == "numeric": + # Symbolic type for all numeric types + continue + try: + VSSDataType.from_str(datatype) + except KeyError: + logging.error("Unknown datatype %s in unit definition", datatype) + sys.exit(-1) + + unit_node = VSSUnit(k, unit, definition, quantity, allowed_datatypes) if k in cls.units: logging.warning("Redefinition of unit %s", k) cls.units[k] = unit_node diff --git a/vspec/model/vsstree.py b/vspec/model/vsstree.py index a339e9ad..5e3a383e 100644 --- a/vspec/model/vsstree.py +++ b/vspec/model/vsstree.py @@ -129,6 +129,20 @@ def extractCoreAttribute(name: str): extractCoreAttribute(attribute) # Datatype and unit need special handling, so we do some further analysis + + # Units are applicable only for primitives. Not user defined types. + if self.has_unit(): + + if not (self.is_signal() or self.is_property()): + logging.error("Item %s cannot have unit, only allowed for signal and property!", self.name) + sys.exit(-1) + + unit = self.source_dict["unit"] + self.unit = VSSUnitCollection.get_unit(unit) + if self.unit is None: + logging.error(f"Unknown unit {unit} for signal {self.qualified_name()}. Terminating.") + sys.exit(-1) + # self.data_type shall only be set if base type is a primitive (VSSDataType) if self.has_datatype(): if self.is_signal() or self.is_property(): @@ -142,19 +156,9 @@ def extractCoreAttribute(name: str): (f"Incomplete element {self.name} from {self.source_dict['$file_name$']}: " f"Elements of type {self.type.value} need to have a datatype declared.")) - # Units are applicable only for primitives. Not user defined types. + # Datatype check for unit performed first when we have set the right datatype if self.has_unit(): - if not (self.is_signal() or self.is_property()): - logging.error("Item %s cannot have unit, only allowed for signal and property!", self.name) - sys.exit(-1) - - unit = self.source_dict["unit"] - self.unit = VSSUnitCollection.get_unit(unit) - if self.unit is None: - logging.error(f"Unknown unit {unit} for signal {self.qualified_name()}. Terminating.") - sys.exit(-1) - if not self.has_datatype(): logging.error("Unit specified for item not using standard datatype: %s", self.name) sys.exit(-1) @@ -362,8 +366,18 @@ def validate_and_set_datatype(self): require the entire tree to be rendered first. These are validated after the entire tree is rendered. """ is_array = ARRAY_SUBSCRIPT_OP in self.data_type_str + # get the base name without subscript decoration + undecorated_datatype_str = self.data_type_str.split( + DEFAULT_SEPARATOR)[-1].replace(ARRAY_SUBSCRIPT_OP, '') + try: self.datatype = VSSDataType.from_str(self.data_type_str) + + if self.unit and self.unit.allowed_datatypes: + if (not ((undecorated_datatype_str in self.unit.allowed_datatypes) or + (VSSDataType.is_numeric(self.datatype) and "numeric" in self.unit.allowed_datatypes))): + logging.error("Datatype %s not allowed for unit %s", self.data_type_str, self.unit.id) + sys.exit(-1) except KeyError as e: if self.type == VSSType.PROPERTY: # Fully Qualified name as data type name @@ -372,9 +386,6 @@ def validate_and_set_datatype(self): (f"Qualified datatype name {self.data_type_str} provided in node {self.qualified_name()}. ", "Semantic checks will be performed after the entire tree is rendered. SKIPPING NOW...")) else: - # get the base name without subscript decoration - undecorated_datatype_str = self.data_type_str.split( - DEFAULT_SEPARATOR)[-1].replace(ARRAY_SUBSCRIPT_OP, '') # Custom data types can contain names defined under the # same branch struct_fqn = self.get_struct_qualified_name(