diff --git a/ngr_spider/cli.py b/ngr_spider/cli.py index cb8c7ef..3d7a586 100755 --- a/ngr_spider/cli.py +++ b/ngr_spider/cli.py @@ -50,6 +50,7 @@ def main_services(args): az_container = args.azure_storage_container yaml_output = args.yaml no_updated = args.no_updated + jq_filter = args.jq_filter protocol_list = PROTOCOLS if protocols: @@ -100,7 +101,7 @@ def main_services(args): } report_services_summary(services, protocol_list) - content = get_output(pretty, yaml_output, config, no_updated) + content = get_output(pretty, yaml_output, config, no_updated, jq_filter) write_output(output_file, az_conn_string, az_container, yaml_output, content) logging.info(f"output written to {output_file}") @@ -121,6 +122,7 @@ def main_layers(args): az_conn_string = args.azure_storage_connection_string az_container = args.azure_storage_container no_updated = args.no_updated + jq_filter = args.jq_filter protocol_list = PROTOCOLS if protocols: @@ -209,7 +211,7 @@ def main_layers(args): if not snake_case: config = [replace_keys(x, convert_snake_to_camelcase) for x in layers] - content = get_output(pretty, yaml_output, config, no_updated) + content = get_output(pretty, yaml_output, config, no_updated, jq_filter) write_output(output_file, az_conn_string, az_container, yaml_output, content) lookup = { WMTS_PROTOCOL: "layers", @@ -277,6 +279,13 @@ def main(): default=os.environ.get("AZURE_STORAGE_CONTAINER"), ) + parent_parser.add_argument( + "--jq-filter", + action="store", + type=str, + help=f"Apply JQ filter to output", + ) + parent_parser.add_argument( "--service-owner", action="store", @@ -294,7 +303,6 @@ def main(): parent_parser.add_argument( "--no-updated", - dest="snake_case", action="store_true", help="do not add updated field to output file", ) diff --git a/ngr_spider/constants.py b/ngr_spider/constants.py index bba5be1..474b12b 100644 --- a/ngr_spider/constants.py +++ b/ngr_spider/constants.py @@ -1,5 +1,5 @@ CSW_URL = "https://nationaalgeoregister.nl/geonetwork/srv/dut/csw" -LOG_LEVEL = "INFO" +LOG_LEVEL = "ERROR" WFS_PROTOCOL = "OGC:WFS" WMS_PROTOCOL = "OGC:WMS" WCS_PROTOCOL = "OGC:WCS" diff --git a/ngr_spider/util.py b/ngr_spider/util.py index 1bb6b4a..4b03730 100644 --- a/ngr_spider/util.py +++ b/ngr_spider/util.py @@ -11,6 +11,7 @@ from typing import Optional, Union from urllib import parse +import jq import requests import yaml from azure.storage.blob import BlobClient, ContentSettings @@ -54,13 +55,22 @@ def get_output( - pretty, yaml_output, config: dict[str, Union[str, list[dict]]], no_timestamp + pretty, + yaml_output, + config: dict[str, Union[str, list[dict]]], + no_timestamp, + jq_filter, ): if not no_timestamp: timestamp = ( datetime.datetime.now().astimezone().replace(microsecond=0).isoformat() ) config["updated"] = timestamp + + if jq_filter: + transformed_config_text = jq.compile(jq_filter).input(config).text() + config = json.loads(transformed_config_text) + if yaml_output: content = yaml.dump(config, default_flow_style=False) else: