From 526754f388303f401584d100230c784fe2e62b4c Mon Sep 17 00:00:00 2001 From: Sebastian Schleemilch Date: Mon, 25 Nov 2024 09:17:49 +0100 Subject: [PATCH 1/2] Improved jsonschema exporter Signed-off-by: Sebastian Schleemilch --- src/vss_tools/exporters/jsonschema.py | 243 ++++++++---------- src/vss_tools/model.py | 4 + tests/vspec/test_comment/expected.jsonschema | 2 +- .../vspec/test_datatypes/expected.jsonschema | 2 +- .../vspec/test_instances/expected.jsonschema | 2 +- tests/vspec/test_min_max/expected.jsonschema | 2 +- 6 files changed, 113 insertions(+), 142 deletions(-) diff --git a/src/vss_tools/exporters/jsonschema.py b/src/vss_tools/exporters/jsonschema.py index d1974527..55fe7e5f 100644 --- a/src/vss_tools/exporters/jsonschema.py +++ b/src/vss_tools/exporters/jsonschema.py @@ -11,120 +11,37 @@ import json from pathlib import Path -from typing import Any, Dict +from typing import Any import rich_click as click import vss_tools.cli_options as clo from vss_tools import log +from vss_tools.datatypes import Datatypes, is_array, resolve_datatype from vss_tools.main import get_trees -from vss_tools.model import VSSDataBranch, VSSDataDatatype, VSSDataStruct +from vss_tools.model import VSSDataBranch, VSSDataDatatype from vss_tools.tree import VSSNode -type_map = { - "int8": "integer", - "uint8": "integer", - "int16": "integer", - "uint16": "integer", - "int32": "integer", - "uint32": "integer", - "int64": "integer", - "uint64": "integer", - "boolean": "boolean", - "float": "number", - "double": "number", - "string": "string", - "int8[]": "array", - "uint8[]": "array", - "int16[]": "array", - "uint16[]": "array", - "int32[]": "array", - "uint32[]": "array", - "int64[]": "array", - "uint64[]": "array", - "boolean[]": "array", - "float[]": "array", - "double[]": "array", - "string[]": "array", -} +class JsonSchemaExporterException(Exception): + pass -def export_node( - json_dict, - node: VSSNode, - all_extended_attributes: bool, - no_additional_properties: bool, - require_all_properties: bool, -): - """Preparing nodes for JSON schema output.""" - # keyword with X- sign are left for extensions and they are not part of official JSON schema - data = node.get_vss_data() - json_dict[node.name] = { - "description": data.description, - } - - if isinstance(data, VSSDataDatatype): - json_dict[node.name]["type"] = type_map[data.datatype] - - min = getattr(data, "min", None) - if min is not None: - json_dict[node.name]["minimum"] = min - - max = getattr(data, "max", None) - if max is not None: - json_dict[node.name]["maximum"] = max - - allowed = getattr(data, "allowed", None) - if allowed: - json_dict[node.name]["enum"] = allowed - - default = getattr(data, "default", None) - if default: - json_dict[node.name]["default"] = default - - if isinstance(data, VSSDataStruct): - json_dict[node.name]["type"] = "object" - - if all_extended_attributes: - json_dict[node.name]["x-VSStype"] = data.type.value - datatype = getattr(data, "datatype", None) - if datatype: - json_dict[node.name]["x-datatype"] = datatype - if data.deprecation: - json_dict[node.name]["x-deprecation"] = data.deprecation - - # in case of unit or aggregate, the attribute will be missing - unit = getattr(data, "unit", None) - if unit: - json_dict[node.name]["x-unit"] = unit - - aggregate = getattr(data, "aggregate", None) - if aggregate: - json_dict[node.name]["x-aggregate"] = aggregate - if aggregate: - json_dict[node.name]["type"] = "object" - - if data.comment: - json_dict[node.name]["x-comment"] = data.comment - - for field in data.get_extra_attributes(): - json_dict[node.name][field] = getattr(data, field) - - # Generate child nodes - if isinstance(data, VSSDataBranch) or isinstance(node.data, VSSDataStruct): - if no_additional_properties: - json_dict[node.name]["additionalProperties"] = False - json_dict[node.name]["properties"] = {} - if require_all_properties: - json_dict[node.name]["required"] = [child.name for child in node.children] - for child in node.children: - export_node( - json_dict[node.name]["properties"], - child, - all_extended_attributes, - no_additional_properties, - require_all_properties, - ) + +type_map = { + Datatypes.INT8[0]: ("integer", -128, 127), + Datatypes.UINT8[0]: ("integer", 0, 255), + Datatypes.INT16[0]: ("integer", -32768, 32767), + Datatypes.UINT16[0]: ("integer", 0, 65535), + Datatypes.INT32[0]: ("integer", -2147483648, 2147483647), + Datatypes.UINT32[0]: ("integer", 0, 4294967295), + Datatypes.INT64[0]: ("integer",), + Datatypes.UINT64[0]: ("integer",), + Datatypes.FLOAT[0]: ("number",), + Datatypes.DOUBLE[0]: ("number",), + Datatypes.NUMERIC[0]: ("number",), + Datatypes.BOOLEAN[0]: ("boolean",), + Datatypes.STRING[0]: ("string",), +} @click.command() @@ -189,39 +106,89 @@ def cli( log.info("Serializing pretty JSON schema...") indent = 2 - signals_json_schema: Dict[str, Any] = {} - export_node( - signals_json_schema, - tree, - extend_all_attributes, - no_additional_properties, - require_all_properties, - ) + schema = {"$schema": "https://json-schema.org/draft/2020-12/schema", "title": tree.name} + + add_node(schema, tree, datatype_tree, no_additional_properties, require_all_properties, extend_all_attributes) - # Add data types to the schema - if datatype_tree is not None: - data_types_json_schema: Dict[str, Any] = {} - export_node( - data_types_json_schema, - datatype_tree, - extend_all_attributes, - no_additional_properties, - require_all_properties, - ) - if extend_all_attributes: - signals_json_schema["x-ComplexDataTypes"] = data_types_json_schema - - # VSS models only have one root, so there should only be one - # key in the dict - assert len(signals_json_schema.keys()) == 1 - top_node_name = list(signals_json_schema.keys())[0] - signals_json_schema = signals_json_schema.pop(top_node_name) - - json_schema = { - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": top_node_name, - "type": "object", - **signals_json_schema, - } with open(output, "w", encoding="utf-8") as output_file: - json.dump(json_schema, output_file, indent=indent, sort_keys=False) + json.dump(schema, output_file, indent=indent, sort_keys=False) + + +def find_type_node(datatype_tree: VSSNode | None, fqn: str) -> VSSNode | None: + if not datatype_tree: + return None + return datatype_tree.get_node_with_fqn(fqn) + + +def add_x_attributes(schema: dict[str, Any], node: VSSNode) -> None: + data = node.get_vss_data() + schema["x-VSStype"] = data.type.value + if isinstance(node.data, VSSDataDatatype): + schema["x-datatype"] = node.data.datatype + if node.data.unit: + schema["x-unit"] = node.data.unit + if data.deprecation: + schema["x-deprecation"] = data.deprecation + if isinstance(node.data, VSSDataBranch): + schema["x-aggregate"] = node.data.aggregate + if data.comment: + schema["x-comment"] = data.comment + + +def add_node( + schema: dict[str, Any], + node: VSSNode, + dtree: VSSNode | None, + no_additional_props: bool, + require_all_properties: bool, + extend_all_attributes: bool, +) -> None: + schema["type"] = "object" + schema["description"] = node.get_vss_data().description + if extend_all_attributes: + add_x_attributes(schema, node) + if isinstance(node.data, VSSDataDatatype): + ref = schema + if is_array(node.data.datatype): + schema["type"] = "array" + schema["items"] = {} + ref = schema["items"] + datatype = node.data.datatype.rstrip("[]") + if datatype in type_map: + target_type = type_map[datatype] + ref["type"] = target_type[0] + if len(target_type) > 1: + ref["minimum"] = target_type[1] + if len(target_type) > 2: + ref["maximum"] = target_type[2] + if node.data.min is not None: + ref["minimum"] = node.data.min + if node.data.max is not None: + ref["maximum"] = node.data.max + if node.data.allowed: + ref["enum"] = node.data.allowed + else: + fqn = resolve_datatype(node.data.datatype, node.get_fqn()).rstrip("[]") + type_node = find_type_node(dtree, fqn) + if not type_node: + raise JsonSchemaExporterException() + add_node(ref, type_node, dtree, no_additional_props, require_all_properties, extend_all_attributes) + else: + schema["properties"] = {} + if no_additional_props: + schema["additionalProperties"] = False + for child in node.children: + if require_all_properties: + if "required" in schema: + schema["required"].append(child.name) + else: + schema["required"] = [child.name] + schema["properties"][child.name] = {} + add_node( + schema["properties"][child.name], + child, + dtree, + no_additional_props, + require_all_properties, + extend_all_attributes, + ) diff --git a/src/vss_tools/model.py b/src/vss_tools/model.py index 27c681e4..eec3d023 100644 --- a/src/vss_tools/model.py +++ b/src/vss_tools/model.py @@ -207,6 +207,10 @@ def check_min_max_valid_datatype(self) -> Self: raise ValueError(f"Cannot define min/max for datatype '{self.datatype}'") if is_array(self.datatype): raise ValueError("Cannot define min/max for array datatypes") + if self.min: + assert Datatypes.is_datatype(self.min, self.datatype), f"min '{self.min}' is not an '{self.datatype}'" + if self.max: + assert Datatypes.is_datatype(self.max, self.datatype), f"max '{self.max}' is not an '{self.datatype}'" return self def check_default_min_max(self) -> Self: diff --git a/tests/vspec/test_comment/expected.jsonschema b/tests/vspec/test_comment/expected.jsonschema index 80415673..152f859e 100644 --- a/tests/vspec/test_comment/expected.jsonschema +++ b/tests/vspec/test_comment/expected.jsonschema @@ -1 +1 @@ -{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"SingleLineNotQuoted": {"description": "A sensor.", "type": "number"}, "SingleLineInternalQuotes": {"description": "A sensor.", "type": "number"}, "SingleLineQuoted": {"description": "A sensor.", "type": "number"}, "SingleLineQuotedInternalQuotes": {"description": "A sensor.", "type": "number"}, "SingleLineComma": {"description": "A sensor.", "type": "number"}, "SingleLineCommaQuoted": {"description": "A sensor.", "type": "number"}, "MultiLineCommaNotQuoted": {"description": "A sensor.", "type": "number"}, "MultiLineCommaQuoted": {"description": "A sensor.", "type": "number"}, "MultiLineStyleInitialBreak": {"description": "A sensor.", "type": "number"}, "MultiLineLiteralStyleQuote": {"description": "A sensor.", "type": "number"}}} \ No newline at end of file +{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"SingleLineNotQuoted": {"type": "number", "description": "A sensor."}, "SingleLineInternalQuotes": {"type": "number", "description": "A sensor."}, "SingleLineQuoted": {"type": "number", "description": "A sensor."}, "SingleLineQuotedInternalQuotes": {"type": "number", "description": "A sensor."}, "SingleLineComma": {"type": "number", "description": "A sensor."}, "SingleLineCommaQuoted": {"type": "number", "description": "A sensor."}, "MultiLineCommaNotQuoted": {"type": "number", "description": "A sensor."}, "MultiLineCommaQuoted": {"type": "number", "description": "A sensor."}, "MultiLineStyleInitialBreak": {"type": "number", "description": "A sensor."}, "MultiLineLiteralStyleQuote": {"type": "number", "description": "A sensor."}}} \ No newline at end of file diff --git a/tests/vspec/test_datatypes/expected.jsonschema b/tests/vspec/test_datatypes/expected.jsonschema index 58f40335..88e9767d 100644 --- a/tests/vspec/test_datatypes/expected.jsonschema +++ b/tests/vspec/test_datatypes/expected.jsonschema @@ -1 +1 @@ -{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"UInt8": {"description": "A uint8.", "type": "integer"}, "Int8": {"description": "An int8.", "type": "integer"}, "UInt16": {"description": "A uint16.", "type": "integer"}, "Int16": {"description": "An int16.", "type": "integer"}, "UInt32": {"description": "A uint32.", "type": "integer"}, "Int32": {"description": "An int32", "type": "integer"}, "UInt64": {"description": "A uint64.", "type": "integer"}, "Int64": {"description": "An int64", "type": "integer"}, "IsBoolean": {"description": "A boolean", "type": "boolean"}, "Float": {"description": "A float.", "type": "number"}, "Double": {"description": "A double.", "type": "number"}}} \ No newline at end of file +{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"UInt8": {"type": "integer", "description": "A uint8.", "minimum": 0, "maximum": 255}, "Int8": {"type": "integer", "description": "An int8.", "minimum": -128, "maximum": 127}, "UInt16": {"type": "integer", "description": "A uint16.", "minimum": 0, "maximum": 65535}, "Int16": {"type": "integer", "description": "An int16.", "minimum": -32768, "maximum": 32767}, "UInt32": {"type": "integer", "description": "A uint32.", "minimum": 0, "maximum": 65535}, "Int32": {"type": "integer", "description": "An int32", "minimum": -2147483648, "maximum": 2147483647}, "UInt64": {"type": "integer", "description": "A uint64."}, "Int64": {"type": "integer", "description": "An int64"}, "IsBoolean": {"type": "boolean", "description": "A boolean"}, "Float": {"type": "number", "description": "A float."}, "Double": {"type": "number", "description": "A double."}}} \ No newline at end of file diff --git a/tests/vspec/test_instances/expected.jsonschema b/tests/vspec/test_instances/expected.jsonschema index 1792aec9..e54b3a3a 100644 --- a/tests/vspec/test_instances/expected.jsonschema +++ b/tests/vspec/test_instances/expected.jsonschema @@ -1 +1 @@ -{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"B": {"description": "This description will be duplicated.", "properties": {"Row1": {"description": "This description will be duplicated.", "properties": {"Left": {"description": "This description will be duplicated.", "properties": {"C": {"description": "This description will also exist multiple times.", "type": "integer"}}}, "Right": {"description": "This description will be duplicated.", "properties": {"C": {"description": "This description will also exist multiple times.", "type": "integer"}}}}}, "Row2": {"description": "This description will be duplicated.", "properties": {"Left": {"description": "This description will be duplicated.", "properties": {"C": {"description": "This description will also exist multiple times.", "type": "integer"}}}, "Right": {"description": "This description will be duplicated.", "properties": {"C": {"description": "This description will also exist multiple times.", "type": "integer"}}}}}}}}} \ No newline at end of file +{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"B": {"type": "object", "description": "This description will be duplicated.", "properties": {"Row1": {"type": "object", "description": "This description will be duplicated.", "properties": {"Left": {"type": "object", "description": "This description will be duplicated.", "properties": {"C": {"type": "integer", "description": "This description will also exist multiple times.", "minimum": -128, "maximum": 127}}}, "Right": {"type": "object", "description": "This description will be duplicated.", "properties": {"C": {"type": "integer", "description": "This description will also exist multiple times.", "minimum": -128, "maximum": 127}}}}}, "Row2": {"type": "object", "description": "This description will be duplicated.", "properties": {"Left": {"type": "object", "description": "This description will be duplicated.", "properties": {"C": {"type": "integer", "description": "This description will also exist multiple times.", "minimum": -128, "maximum": 127}}}, "Right": {"type": "object", "description": "This description will be duplicated.", "properties": {"C": {"type": "integer", "description": "This description will also exist multiple times.", "minimum": -128, "maximum": 127}}}}}}}}} \ No newline at end of file diff --git a/tests/vspec/test_min_max/expected.jsonschema b/tests/vspec/test_min_max/expected.jsonschema index 7b56510c..76af18bf 100644 --- a/tests/vspec/test_min_max/expected.jsonschema +++ b/tests/vspec/test_min_max/expected.jsonschema @@ -1 +1 @@ -{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"IntNoMinMax": {"description": "No Min Max.", "type": "integer"}, "IntOnlyMax": {"description": "Only Max.", "type": "integer", "maximum": 32}, "IntOnlyMin": {"description": "Only Min.", "type": "integer", "minimum": 3}, "IntMinMax": {"description": "Min & Max.", "type": "integer", "minimum": 3, "maximum": 6}, "IntMaxZero": {"description": "Max Zero.", "type": "integer", "maximum": 0}, "IntMinZero": {"description": "Min Zero.", "type": "integer", "minimum": 0}, "FloatNoMinMax": {"description": "No Min Max.", "type": "number"}, "FloatOnlyMax": {"description": "Only Max.", "type": "number", "maximum": 32.3}, "FloatOnlyMin": {"description": "Only Min.", "type": "number", "minimum": -2.5}, "FloatMinMax": {"description": "Min & Max.", "type": "number", "minimum": -165.56323, "maximum": 236723.4}, "FloatMaxZero": {"description": "Max Zero.", "type": "number", "maximum": 0.0}, "FloatMinZero": {"description": "Min Zero.", "type": "number", "minimum": 0.0}, "FloatMaxZeroInt": {"description": "Max Zero.", "type": "number", "maximum": 0}, "FloatMinZeroInt": {"description": "Min Zero.", "type": "number", "minimum": 0}}} \ No newline at end of file +{"$schema": "https://json-schema.org/draft/2020-12/schema", "title": "A", "type": "object", "description": "Branch A.", "properties": {"IntNoMinMax": {"type": "integer", "description": "No Min Max.", "minimum": -128, "maximum": 127}, "IntOnlyMax": {"type": "integer", "description": "Only Max.", "minimum": -128, "maximum": 32}, "IntOnlyMin": {"type": "integer", "description": "Only Min.", "minimum": 3, "maximum": 127}, "IntMinMax": {"type": "integer", "description": "Min & Max.", "minimum": 3, "maximum": 6}, "IntMaxZero": {"type": "integer", "description": "Max Zero.", "minimum": -128, "maximum": 0}, "IntMinZero": {"type": "integer", "description": "Min Zero.", "minimum": 0, "maximum": 127}, "FloatNoMinMax": {"type": "number", "description": "No Min Max."}, "FloatOnlyMax": {"type": "number", "description": "Only Max.", "maximum": 32.3}, "FloatOnlyMin": {"type": "number", "description": "Only Min.", "minimum": -2.5}, "FloatMinMax": {"type": "number", "description": "Min & Max.", "minimum": -165.56323, "maximum": 236723.4}, "FloatMaxZero": {"type": "number", "description": "Max Zero.", "maximum": 0.0}, "FloatMinZero": {"type": "number", "description": "Min Zero.", "minimum": 0.0}, "FloatMaxZeroInt": {"type": "number", "description": "Max Zero.", "maximum": 0}, "FloatMinZeroInt": {"type": "number", "description": "Min Zero.", "minimum": 0}}} \ No newline at end of file From 12fc25d9e9570f64eb40de4ef537fab7856a1da0 Mon Sep 17 00:00:00 2001 From: "Kostadin Ivanov (BD/TBC-BG)" Date: Wed, 4 Dec 2024 11:55:21 +0200 Subject: [PATCH 2/2] Add entry for samm exporter in the vspec docs - help example. Signed-off-by: Kostadin Ivanov (BD/TBC-BG) --- docs/vspec.md | 35 ++++++++++++------------ src/vss_tools/exporters/samm/__init__.py | 2 +- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/docs/vspec.md b/docs/vspec.md index de88138c..20591f0e 100644 --- a/docs/vspec.md +++ b/docs/vspec.md @@ -8,23 +8,24 @@ For the most up do date usage information, please call the tool help via: vspec export --help Usage: vspec export [OPTIONS] COMMAND [ARGS]... -╭─ Options ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│ --help Show this message and exit. │ -╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│ apigear Export to ApiGear. │ -│ binary Export to Binary. │ -│ csv Export as CSV. │ -│ ddsidl Export as DDSIDL. │ -│ franca Export as Franca. │ -│ graphql Export as GraphQL. │ -│ id Export as IDs. │ -│ json Export as JSON. │ -│ jsonschema Export as a jsonschema. │ -│ protobuf Export as protobuf. │ -│ yaml Export as YAML. │ -│ tree Export as Tree. │ -╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ --help Show this message and exit. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│ apigear Export to ApiGear. │ +│ binary Export to Binary. │ +│ csv Export as CSV. │ +│ ddsidl Export as DDSIDL. │ +│ franca Export as Franca. │ +│ graphql Export as GraphQL. │ +│ id Export as IDs. │ +│ json Export as JSON. │ +│ jsonschema Export as a jsonschema. │ +│ protobuf Export as protobuf. │ +| samm Export as Eclipse Semantic Modeling Framework (ESMF) - Semantic Aspect Meta Model (SAMM) - .ttl files. | +│ yaml Export as YAML. │ +│ tree Export as Tree. │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ``` diff --git a/src/vss_tools/exporters/samm/__init__.py b/src/vss_tools/exporters/samm/__init__.py index bdd0c4f4..0dce88be 100644 --- a/src/vss_tools/exporters/samm/__init__.py +++ b/src/vss_tools/exporters/samm/__init__.py @@ -176,7 +176,7 @@ def cli( split_depth, ) -> None: """ - Export COVESA VSS to Eclipse Semantic Modeling Framework (ESMF) - Semantic Aspect Meta Model (SAMM) - .ttl files. + Export as Eclipse Semantic Modeling Framework (ESMF) - Semantic Aspect Meta Model (SAMM) - .ttl files. """ log.info("Loading VSS Tree...\n")