add jq filter to transform output

PDOK · Dec 3, 2022 · 8880122 · 8880122
1 parent c6a6915
commit 8880122
Show file tree

Hide file tree

Showing 3 changed files with 23 additions and 5 deletions.
diff --git a/ngr_spider/cli.py b/ngr_spider/cli.py
@@ -50,6 +50,7 @@ def main_services(args):
     az_container = args.azure_storage_container
     yaml_output = args.yaml
     no_updated = args.no_updated
+    jq_filter = args.jq_filter
 
     protocol_list = PROTOCOLS
     if protocols:
@@ -100,7 +101,7 @@ def main_services(args):
             }
 
         report_services_summary(services, protocol_list)
-        content = get_output(pretty, yaml_output, config, no_updated)
+        content = get_output(pretty, yaml_output, config, no_updated, jq_filter)
         write_output(output_file, az_conn_string, az_container, yaml_output, content)
 
         logging.info(f"output written to {output_file}")
@@ -121,6 +122,7 @@ def main_layers(args):
     az_conn_string = args.azure_storage_connection_string
     az_container = args.azure_storage_container
     no_updated = args.no_updated
+    jq_filter = args.jq_filter
 
     protocol_list = PROTOCOLS
     if protocols:
@@ -209,7 +211,7 @@ def main_layers(args):
             if not snake_case:
                 config = [replace_keys(x, convert_snake_to_camelcase) for x in layers]
 
-        content = get_output(pretty, yaml_output, config, no_updated)
+        content = get_output(pretty, yaml_output, config, no_updated, jq_filter)
         write_output(output_file, az_conn_string, az_container, yaml_output, content)
         lookup = {
             WMTS_PROTOCOL: "layers",
@@ -277,6 +279,13 @@ def main():
         default=os.environ.get("AZURE_STORAGE_CONTAINER"),
     )
 
+    parent_parser.add_argument(
+        "--jq-filter",
+        action="store",
+        type=str,
+        help=f"Apply JQ filter to output",
+    )
+
     parent_parser.add_argument(
         "--service-owner",
         action="store",
@@ -294,7 +303,6 @@ def main():
 
     parent_parser.add_argument(
         "--no-updated",
-        dest="snake_case",
         action="store_true",
         help="do not add updated field to output file",
     )

diff --git a/ngr_spider/constants.py b/ngr_spider/constants.py
@@ -1,5 +1,5 @@
 CSW_URL = "https://nationaalgeoregister.nl/geonetwork/srv/dut/csw"
-LOG_LEVEL = "INFO"
+LOG_LEVEL = "ERROR"
 WFS_PROTOCOL = "OGC:WFS"
 WMS_PROTOCOL = "OGC:WMS"
 WCS_PROTOCOL = "OGC:WCS"

diff --git a/ngr_spider/util.py b/ngr_spider/util.py
@@ -11,6 +11,7 @@
 from typing import Optional, Union
 from urllib import parse
 
+import jq
 import requests
 import yaml
 from azure.storage.blob import BlobClient, ContentSettings
@@ -54,13 +55,22 @@
 
 
 def get_output(
-    pretty, yaml_output, config: dict[str, Union[str, list[dict]]], no_timestamp
+    pretty,
+    yaml_output,
+    config: dict[str, Union[str, list[dict]]],
+    no_timestamp,
+    jq_filter,
 ):
     if not no_timestamp:
         timestamp = (
             datetime.datetime.now().astimezone().replace(microsecond=0).isoformat()
         )
         config["updated"] = timestamp
+
+    if jq_filter:
+        transformed_config_text = jq.compile(jq_filter).input(config).text()
+        config = json.loads(transformed_config_text)
+
     if yaml_output:
         content = yaml.dump(config, default_flow_style=False)
     else: