From d057bcc8a372aeaa990683e849c8eabe9798b713 Mon Sep 17 00:00:00 2001 From: Preston Hartzell Date: Sat, 1 Jul 2023 14:55:07 -0400 Subject: [PATCH] Update sentinel-3 to use stactools package (#220) * update to use stactools package * apply isort and black * fix: patch stactools issues - https://github.com/stactools-packages/sentinel3/issues/24 - https://github.com/stactools-packages/sentinel3/issues/25 - https://github.com/stactools-packages/sentinel3/issues/28 * update requirement pin * finish up --- datasets/sentinel-3/README.md | 18 +- datasets/sentinel-3/dataset.yaml | 161 +++++----- datasets/sentinel-3/requirements.txt | 2 +- datasets/sentinel-3/sentinel_3.py | 282 ++++++++++++++++++ datasets/sentinel-3/sentinel_3/__init__.py | 0 .../sentinel-3/sentinel_3/sentinel_3_base.py | 257 ---------------- .../sentinel_3_olci_lfr_l2_netcdf.py | 55 ---- .../sentinel_3_olci_wfr_l2_netcdf.py | 56 ---- .../sentinel_3_slstr_frp_l2_netcdf.py | 34 --- .../sentinel_3_slstr_lst_l2_netcdf.py | 53 ---- .../sentinel_3_slstr_wst_l2_netcdf.py | 47 --- .../sentinel_3_sral_lan_l2_netcdf.py | 47 --- .../sentinel_3_sral_wat_l2_netcdf.py | 48 --- .../sentinel_3_synergy_aod_l2_netcdf.py | 36 --- .../sentinel_3_synergy_syn_l2_netcdf.py | 51 ---- .../sentinel_3_synergy_v10_l2_netcdf.py | 40 --- .../sentinel_3_synergy_vg1_l2_netcdf.py | 40 --- .../sentinel_3_synergy_vgp_l2_netcdf.py | 59 ---- .../sentinel_3/sentinel_3_winding.py | 97 ------ datasets/sentinel-3/update.yaml | 184 ------------ 20 files changed, 380 insertions(+), 1187 deletions(-) create mode 100644 datasets/sentinel-3/sentinel_3.py delete mode 100644 datasets/sentinel-3/sentinel_3/__init__.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_base.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_olci_lfr_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_olci_wfr_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_slstr_frp_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_slstr_lst_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_slstr_wst_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_sral_lan_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_sral_wat_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_synergy_aod_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_synergy_syn_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_synergy_v10_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vg1_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vgp_l2_netcdf.py delete mode 100644 datasets/sentinel-3/sentinel_3/sentinel_3_winding.py delete mode 100644 datasets/sentinel-3/update.yaml diff --git a/datasets/sentinel-3/README.md b/datasets/sentinel-3/README.md index fbdb83d0..5fcc6709 100644 --- a/datasets/sentinel-3/README.md +++ b/datasets/sentinel-3/README.md @@ -19,12 +19,6 @@ collection/sentinel-3-synergy-vg1-l2-netcdf collection/sentinel-3-synergy-vgp-l2-netcdf ``` -The package `sentinel_3` contains the python code for making the STAC items. -Each collection has its own module -(`sentinel_3/sentinel_3_olci_lfr_l2_netcdf.py`). The name of the module should -match the name of the collection. The module should have a class named -`Collection` that is the pctasks `dataset.Collection` subclass. - ## Tests The `datasets/sentinel-3/tests` directory contains some tests. Run those with @@ -34,7 +28,7 @@ The `datasets/sentinel-3/tests` directory contains some tests. Run those with $ PYTHONPATH=datasets/sentinel-3 python -m pytest datasets/sentinel-3/tests/ ``` -### Dynamic updates +## Dynamic updates ```console $ ls datasets/sentinel-3/collection/ | xargs -I {} \ @@ -44,4 +38,14 @@ $ ls datasets/sentinel-3/collection/ | xargs -I {} \ --workflow-id={}-update \ --is-update-workflow \ --upsert +``` + +**Notes:** + +- Takes about 30 minutes to chunk and create items for all collections using the test batch account, a `year-prefix` argument, and a `--since` argument limiting the chunks to a few days. + +## Docker container + +```shell +az acr build -r {the registry} --subscription {the subscription} -t pctasks-sentinel-3:latest -t pctasks-sentinel-3:{date}.{count} -f datasets/sentinel-3/Dockerfile . ``` \ No newline at end of file diff --git a/datasets/sentinel-3/dataset.yaml b/datasets/sentinel-3/dataset.yaml index 346608d9..94dbe2f1 100644 --- a/datasets/sentinel-3/dataset.yaml +++ b/datasets/sentinel-3/dataset.yaml @@ -1,11 +1,12 @@ id: sentinel-3 -image: ${{ args.registry }}/pctasks-sentinel-3:2023.5.1.0 +image: ${{ args.registry }}/pctasks-sentinel-3:20230630.1 args: - registry + - year-prefix code: - src: ${{ local.path(./sentinel_3) }} + src: ${{ local.path(./sentinel_3.py) }} environment: AZURE_TENANT_ID: ${{ secrets.task-tenant-id }} @@ -15,169 +16,179 @@ environment: collections: - id: sentinel-3-olci-lfr-l2-netcdf template: ${{ local.path(./collection/sentinel-3-olci-lfr-l2-netcdf/) }} - class: sentinel_3.sentinel_3_olci_lfr_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - # create-chunks took ~8 minutes. Consider splitting by year. + # The blob storage pattern is + # | sentinel-3 + # | OLCI + # | OL_2_LFR___ + # | 2023 + # | 06 + # | 21 + # | S3A_OL_2_LFR____20230621T003934_20230621T004051_20230621T030311_0077_100_145_1080_PS1_O_NR_002.SEN3 + # | xfdumanifest.xml + # | ... + # | ... + - uri: blob://sentinel3euwest/sentinel-3/OLCI/OL_2_LFR___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 # split by month, about 15 minutes for all xfdumanifest.xml files options: - name_starts_with: OLCI/OL_2_LFR___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 # takes about 15 minutes to create items for ~2 days of data chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-olci-lfr-l2-netcdf - id: sentinel-3-olci-wfr-l2-netcdf template: ${{ local.path(./collection/sentinel-3-olci-wfr-l2-netcdf/) }} - class: sentinel_3.sentinel_3_olci_wfr_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/OLCI/OL_2_WFR___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: OLCI/OL_2_WFR___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-olci-wfr-l2-netcdf - id: sentinel-3-synergy-aod-l2-netcdf template: ${{ local.path(./collection/sentinel-3-synergy-aod-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_aod_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SYNERGY/SY_2_AOD___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SYNERGY/SY_2_AOD___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-aod-l2-netcdf - id: sentinel-3-synergy-syn-l2-netcdf template: ${{ local.path(./collection/sentinel-3-synergy-syn-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_syn_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SYNERGY/SY_2_SYN___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SYNERGY/SY_2_SYN___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-syn-l2-netcdf - id: sentinel-3-synergy-v10-l2-netcdf template: ${{ local.path(./collection/sentinel-3-synergy-v10-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_v10_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SYNERGY/SY_2_V10___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SYNERGY/SY_2_V10___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-v10-l2-netcdf - id: sentinel-3-synergy-vg1-l2-netcdf template: ${{ local.path(./collection/sentinel-3-synergy-vg1-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_vg1_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SYNERGY/SY_2_VG1___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SYNERGY/SY_2_VG1___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-vg1-l2-netcdf - id: sentinel-3-synergy-vgp-l2-netcdf template: ${{ local.path(./collection/sentinel-3-synergy-vgp-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_vgp_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SYNERGY/SY_2_VGP___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SYNERGY/SY_2_VGP___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-vgp-l2-netcdf - id: sentinel-3-sral-lan-l2-netcdf template: ${{ local.path(./collection/sentinel-3-sral-lan-l2-netcdf) }} - class: sentinel_3.sentinel_3_sral_lan_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SRAL/SR_2_LAN___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SRAL/SR_2_LAN___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-sral-lan-l2-netcdf - id: sentinel-3-sral-wat-l2-netcdf template: ${{ local.path(./collection/sentinel-3-sral-wat-l2-netcdf) }} - class: sentinel_3.sentinel_3_sral_wat_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SRAL/SR_2_WAT___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SRAL/SR_2_WAT___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-sral-wat-l2-netcdf - id: sentinel-3-slstr-frp-l2-netcdf template: ${{ local.path(./collection/sentinel-3-slstr-frp-l2-netcdf) }} - class: sentinel_3.sentinel_3_slstr_frp_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SLSTR/SL_2_FRP___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SLSTR/SL_2_FRP___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-frp-l2-netcdf - id: sentinel-3-slstr-lst-l2-netcdf template: ${{ local.path(./collection/sentinel-3-slstr-lst-l2-netcdf) }} - class: sentinel_3.sentinel_3_slstr_lst_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SLSTR/SL_2_LST___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SLSTR/SL_2_LST___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-lst-l2-netcdf - id: sentinel-3-slstr-wst-l2-netcdf template: ${{ local.path(./collection/sentinel-3-slstr-wst-l2-netcdf) }} - class: sentinel_3.sentinel_3_slstr_wst_l2_netcdf:Collection + class: sentinel_3:Sentinel3Collections asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} + - uri: blob://sentinel3euwest/sentinel-3/SLSTR/SL_2_WST___/${{ args.year-prefix }}/ chunks: + splits: + - depth: 1 options: - name_starts_with: SLSTR/SL_2_WST___/ - extensions: [.json] - chunk_length: 5000 + ends_with: xfdumanifest.xml + chunk_length: 200 chunk_storage: uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-wst-l2-netcdf \ No newline at end of file diff --git a/datasets/sentinel-3/requirements.txt b/datasets/sentinel-3/requirements.txt index 10eba107..3aa350eb 100644 --- a/datasets/sentinel-3/requirements.txt +++ b/datasets/sentinel-3/requirements.txt @@ -1 +1 @@ -antimeridian==0.2.3 \ No newline at end of file +git+https://github.com/stactools-packages/sentinel3.git@36375cc63c053087380664ff931ceed5ad3b5f83 diff --git a/datasets/sentinel-3/sentinel_3.py b/datasets/sentinel-3/sentinel_3.py new file mode 100644 index 00000000..6d648552 --- /dev/null +++ b/datasets/sentinel-3/sentinel_3.py @@ -0,0 +1,282 @@ +import logging +import os +from hashlib import md5 +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import List, Union + +import pystac +import requests +import urllib3 +from stactools.sentinel3.stac import create_item + +import pctasks.dataset.collection +from pctasks.core.models.task import WaitTaskResult +from pctasks.core.storage import Storage, StorageFactory +from pctasks.core.utils.backoff import is_common_throttle_exception, with_backoff + +handler = logging.StreamHandler() +handler.setFormatter(logging.Formatter("[%(levelname)s]:%(asctime)s: %(message)s")) +handler.setLevel(logging.INFO) +logger = logging.getLogger(__name__) +logger.addHandler(handler) +logger.setLevel(logging.INFO) + + +def browse_jpg_asset(jpg_path: str) -> pystac.Asset: + # https://github.com/stactools-packages/sentinel3/issues/28 + # https://github.com/stactools-packages/sentinel3/issues/24 + return pystac.Asset( + href=jpg_path, + media_type=pystac.MediaType.JPEG, + description="temp", + roles=["thumbnail"], + extra_fields={ + "file:size": os.path.getsize(jpg_path), + "file:checksum": md5(open(jpg_path, "rb").read()).hexdigest(), + }, + ) + + +def eop_metadata_asset(eop_metadata_path: str) -> pystac.Asset: + # https://github.com/stactools-packages/sentinel3/issues/28 + # https://github.com/stactools-packages/sentinel3/issues/24 + manifest_text = open(eop_metadata_path, encoding="utf-8").read() + manifest_text_encoded = manifest_text.encode(encoding="UTF-8") + return pystac.Asset( + href=eop_metadata_path, + media_type=pystac.MediaType.XML, + description="temp", + roles=["metadata"], + extra_fields={ + "file:size": os.path.getsize(eop_metadata_path), + "file:checksum": md5(manifest_text_encoded).hexdigest(), + }, + ) + + +def olci_lfr_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + asset_descriptions = { + "safe-manifest": "SAFE product manifest", + "gifapar": "Green instantaneous Fraction of Absorbed Photosynthetically Active Radiation (FAPAR)", # noqa + "ogvi": "OLCI Global Vegetation Index (OGVI)", + "otci": "OLCI Terrestrial Chlorophyll Index (OTCI)", + "iwv": "Integrated water vapour column", + "rc-ogvi": "Rectified reflectance", + "rc-gifapar": "Rectified reflectance", + "lqsf": "Land quality and science flags", + "time-coordinates": "Time coordinate annotations", + "geo-coordinates": "Geo coordinate annotations", + "tie-geo-coordinates": "Tie-point geo coordinate annotations", + "tie-geometries": "Tie-point geometry annotations", + "tie-meteo": "Tie-point meteo annotations", + "instrument-data": "Instrument annotations", + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + return item + + +def olci_wfr_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + browse_jpg_path = os.path.join(temp_sen3_dir, "browse.jpg") + if os.path.isfile(browse_jpg_path): + item.assets["browse-jpg"] = browse_jpg_asset(browse_jpg_path) + + eop_metadata_path = os.path.join(temp_sen3_dir, "EOPMetadata.xml") + if os.path.isfile(eop_metadata_path): + item.assets["eop-metadata"] = eop_metadata_asset(eop_metadata_path) + + asset_descriptions = { + "chl-nn": "Neural net chlorophyll concentration", + "chl-oc4me": "OC4Me algorithm chlorophyll concentration", + "iop-nn": "Inherent optical properties of water", + "iwv": "Integrated water vapour column", + "par": "Photosynthetically active radiation", + "w-aer": "Aerosol over water", + "geo-coordinates": "Geo coordinate annotations", + "instrument-data": "Instrument annotations", + "tie-geo-coordinates": "Tie-point geo coordinate annotations", + "tie-geometries": "Tie-point geometry annotations", + "tie-meteo": "Tie-point meteo annotations", + "time-coordinates": "Time coordinate annotations", + "wqsf": "Water quality and science flags", + "eop-metadata": "Metadata produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 + "browse-jpg": "Preview image produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + + return item + + +def slstr_lst_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + asset_descriptions = { + "safe-manifest": "SAFE product manifest", + "lst-in": "Land Surface Temperature (LST) values", + "st-ancillary-ds": "LST ancillary measurement dataset", + "slstr-flags-in": "Global flags for the 1km TIR grid, nadir view", + "slstr-indices-in": "Scan, pixel and detector indices annotations for the 1km TIR grid, nadir view", # noqa: E501 + "slstr-time-in": "Time annotations for the 1km grid", + "slstr-geodetic-in": "Full resolution geodetic coordinates for the 1km TIR grid, nadir view", + "slstr-cartesian-in": "Full resolution cartesian coordinates for the 1km TIR grid, nadir view", + "slstr-geometry-tn": "16km solar and satellite geometry annotations, nadir view", + "slstr-geodetic-tx": "16km geodetic coordinates", + "slstr-cartesian-tx": "16km cartesian coordinates", + "slstr-met-tx": "Meteorological parameters regridded onto the 16km tie points", + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + return item + + +def slstr_wst_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + browse_jpg_path = os.path.join(temp_sen3_dir, "browse.jpg") + if os.path.isfile(browse_jpg_path): + item.assets["browse-jpg"] = browse_jpg_asset(browse_jpg_path) + + eop_metadata_path = os.path.join(temp_sen3_dir, "EOPMetadata.xml") + if os.path.isfile(eop_metadata_path): + item.assets["eop-metadata"] = eop_metadata_asset(eop_metadata_path) + + asset_descriptions = { + "safe-manifest": "SAFE product manifest", + "l2p": "Skin Sea Surface Temperature (SST) values", + "eop-metadata": "Metadata produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 + "browse-jpg": "Preview image produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + return item + + +def sral_lan_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + asset_descriptions = { + "safe-manifest": "SAFE product manifest", + "standard-measurement": "Standard measurement data file", + "enhanced-measurement": "Enhanced measurement data file", + "reduced-measurement": "Reduced measurement data file", + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + + # https://github.com/stactools-packages/sentinel3/issues/25 + asset.extra_fields.pop("shape", None) + asset.extra_fields.pop("s3:shape", None) + + return item + + +def sral_wat_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + eop_metadata_path = os.path.join(temp_sen3_dir, "EOPMetadata.xml") + if os.path.isfile(eop_metadata_path): + item.assets["eop-metadata"] = eop_metadata_asset(eop_metadata_path) + + asset_descriptions = { + "safe-manifest": "SAFE product manifest", + "standard-measurement": "Standard measurement data file", + "enhanced-measurement": "Enhanced measurement data file", + "reduced-measurement": "Reduced measurement data file", + "eop-metadata": "Product metadata file produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + + # https://github.com/stactools-packages/sentinel3/issues/25 + asset.extra_fields.pop("shape", None) + asset.extra_fields.pop("s3:shape", None) + + return item + + +def synergy_vgp_fixups(item: pystac.Item, temp_sen3_dir: str) -> pystac.Item: + asset_descriptions = { + "safe-manifest": "SAFE product manifest", + "b0": "Top of atmosphere reflectance data set associated with the VGT-B0 channel", + "b2": "Top of atmosphere reflectance data set associated with the VGT-B2 channel", + "b3": "Top of atmosphere reflectance data set associated with the VGT-B3 channel", + "mir": "Top of atmosphere Reflectance data set associated with the VGT-MIR channel", + "vaa": "View azimuth angle data", + "vza": "View zenith angle data", + "saa": "Solar azimuth angle data", + "sza": "Solar zenith angle data", + "ag": "Aerosol optical thickness data", + "og": "Total ozone column data", + "wvg": "Total column water vapour data", + "sm": "Status map data", + } + for asset_key, asset in item.assets.items(): + if asset_key in asset_descriptions: + asset.description = asset_descriptions[asset_key] + return item + + +FIXUP_FUNCS = { + "olci-lfr": olci_lfr_fixups, + "olci-wfr": olci_wfr_fixups, + "slstr-lst": slstr_lst_fixups, + "slstr-wst": slstr_wst_fixups, + "sral-lan": sral_lan_fixups, + "sral-wat": sral_wat_fixups, + "synergy-vgp": synergy_vgp_fixups, +} + + +def backoff_throttle_check(e: Exception) -> bool: + return ( + is_common_throttle_exception(e) + or isinstance(e, urllib3.exceptions.ReadTimeoutError) + or isinstance(e, requests.exceptions.ConnectionError) + ) + + +class Sentinel3Collections(pctasks.dataset.collection.Collection): + @classmethod + def create_item( + cls, asset_uri: str, storage_factory: StorageFactory + ) -> Union[List[pystac.Item], WaitTaskResult]: + + # Only create Items for NT (Not Time critical) products + sen3_archive = os.path.dirname(asset_uri) + assert sen3_archive.endswith(".SEN3") + timeliness = sen3_archive[-11:-9] + assert timeliness in ["NR", "ST", "NT"] + if sen3_archive[-11:-9] != "NT": + return [] + + sen3_storage = storage_factory.get_storage(sen3_archive) + + with TemporaryDirectory() as temp_dir: + temp_sen3_dir = Path(temp_dir, Path(sen3_archive).name) + temp_sen3_dir.mkdir() + for path in sen3_storage.list_files(): + with_backoff( + lambda: sen3_storage.download_file( + path, str(Path(temp_sen3_dir, path)) + ), + is_throttle=backoff_throttle_check, + ) + + try: + item: pystac.Item = create_item(str(temp_sen3_dir)) + except FileNotFoundError: + # occasionally there is an empty file, e.g., a 0-byte netcdf + logger.exception( + f"Missing file when attempting to create item for {asset_uri}" + ) + return [] + + fixup_function = FIXUP_FUNCS.get(item.properties["s3:product_name"], None) + if fixup_function is not None: + item = fixup_function(item, str(temp_sen3_dir)) + + for asset in item.assets.values(): + path = Path(asset.href).name + asset.href = sen3_storage.get_url(path) + + return [item] diff --git a/datasets/sentinel-3/sentinel_3/__init__.py b/datasets/sentinel-3/sentinel_3/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_base.py b/datasets/sentinel-3/sentinel_3/sentinel_3_base.py deleted file mode 100644 index 46acb887..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_base.py +++ /dev/null @@ -1,257 +0,0 @@ -import logging -from decimal import Decimal -from pathlib import Path -from typing import Any, Dict, List, Optional - -import antimeridian -import pystac -import shapely.geometry -from sentinel_3.sentinel_3_winding import get_winding - -import pctasks.dataset.collection - -handler = logging.StreamHandler() -handler.setFormatter(logging.Formatter("[%(levelname)s]:%(asctime)s: %(message)s")) -handler.setLevel(logging.INFO) -logger = logging.getLogger(__name__) -logger.addHandler(handler) -logger.setLevel(logging.INFO) - -PRODUCT_NAMES = { - "OL_2_LFR___": "olci-lfr", - "OL_2_WFR___": "olci-wfr", - "SL_2_FRP___": "slstr-frp", - "SL_2_LST___": "slstr-lst", - "SL_2_WST___": "slstr-wst", - "SR_2_LAN___": "sral-lan", - "SR_2_WAT___": "sral-wat", - "SY_2_AOD___": "synergy-aod", - "SY_2_SYN___": "synergy-syn", - "SY_2_V10___": "synergy-v10", - "SY_2_VG1___": "synergy-vg1", - "SY_2_VGP___": "synergy-vgp", -} - - -def recursive_round(coordinates: List[Any], precision: int) -> List[Any]: - """Rounds a list of numbers. The list can contain additional nested lists - or tuples of numbers. - - Any tuples encountered will be converted to lists. - - Args: - coordinates (List[Any]): A list of numbers, possibly containing nested - lists or tuples of numbers. - precision (int): Number of decimal places to use for rounding. - - Returns: - List[Any]: The list of numbers rounded to the given precision. - """ - rounded: List[Any] = [] - for value in coordinates: - if isinstance(value, (int, float)): - rounded.append(round(value, precision)) - else: - rounded.append(recursive_round(list(value), precision)) - return rounded - - -def nano2micro(value: float) -> float: - """Converts nanometers to micrometers while handling floating - point arithmetic errors.""" - return float(Decimal(str(value)) / Decimal("1000")) - - -def hz2ghz(value: float) -> float: - """Converts hertz to gigahertz while handling floating point - arithmetic errors.""" - return float(Decimal(str(value)) / Decimal("1000000000")) - - -class BaseSentinelCollection(pctasks.dataset.collection.Collection): # type: ignore - def base_updates(item_dict: Dict[str, Any]) -> Optional[Dict[str, Any]]: - """Apply item updates. - - Args: - item_dict (Dict[str, Any]): The Item dictionary as read from Azure. - max_delta_lon (float): The maximum longitude difference used by - the winding detection algorithm. - - Returns: - Optional[Dict[str, Any]]: The updated Item dictionary, or None if - the Item is not a NT (Not Time critical) product. - """ - - # Skip item if it is not a NT (Not Time critical) product. - asset_directory = Path(item_dict["assets"]["safe-manifest"]["href"]).parent.name - assert asset_directory.endswith(".SEN3") - timeliness = asset_directory[-11:-9] - assert timeliness in ["NR", "ST", "NT"] - if asset_directory[-11:-9] != "NT": - return None - - # Strip any trailing underscores from the ID - original_id = item_dict["id"] - item_dict["id"] = original_id.rstrip("_") - - # ---- PROPERTIES ---- - properties = item_dict.pop("properties") - - # Update placeholder platform ID to the final version - sat_id = "sat:platform_international_designator" - if properties[sat_id] == "0000-000A": - properties[sat_id] = "2016-011A" - elif properties[sat_id] == "0000-000B": - properties[sat_id] = "2018-039A" - - if properties["instruments"] == ["SYNERGY"]: - # "SYNERGY" is not a instrument - properties["instruments"] = ["OLCI", "SLSTR"] - - # Add the processing timelessness to the properties - properties["s3:processing_timeliness"] = timeliness - - # Add a user-friendly name - properties["s3:product_name"] = PRODUCT_NAMES[properties["s3:productType"]] - - # Providers should be supplied in the Collection, not the Item - properties.pop("providers", None) - - # start_datetime and end_datetime are incorrectly formatted - properties["start_datetime"] = pystac.utils.datetime_to_str( - pystac.utils.str_to_datetime(properties["start_datetime"]) - ) - properties["end_datetime"] = pystac.utils.datetime_to_str( - pystac.utils.str_to_datetime(properties["end_datetime"]) - ) - properties["datetime"] = pystac.utils.datetime_to_str( - pystac.utils.str_to_datetime(properties["datetime"]) - ) - - # Remove s3:mode, which is always set to EO (Earth # Observation). It - # offers no additional information. - properties.pop("s3:mode", None) - - # Use underscores instead of camel case for consistency - new_properties = {} - for key in properties: - if key.startswith("s3:"): - new_key = "".join( - "_" + char.lower() if char.isupper() else char for char in key - ) - # strip "_pixels_percentages" to match eo:cloud_cover pattern - if new_key.endswith("_pixels_percentage"): - new_key = new_key.replace("_pixels_percentage", "") - elif new_key.endswith("_pixelss_percentage"): - new_key = new_key.replace("_pixelss_percentage", "") - elif new_key.endswith("_percentage"): - new_key = new_key.replace("_percentage", "") - new_properties[new_key] = properties[key] - else: - new_properties[key] = properties[key] - - item_dict["properties"] = new_properties - - # ---- LINKS ---- - links = item_dict.pop("links") - - # license is the same for all Items; include on the Collection, not the Item - for link in links: - if link["rel"] == "license": - links.remove(link) - - item_dict["links"] = links - - # ---- ASSETS ---- - assets = item_dict.pop("assets") - - new_assets = {} - for asset_key, asset in assets.items(): - # remove local paths - asset.pop("file:local_path", None) - - # Add a description to the safe_manifest asset - if asset_key == "safe-manifest": - asset["description"] = "SAFE product manifest" - - # correct eo:bands - if "eo:bands" in asset: - for band in asset["eo:bands"]: - band["center_wavelength"] = nano2micro(band["center_wavelength"]) - band["full_width_half_max"] = nano2micro(band["band_width"]) - band.pop("band_width") - - # Tune up the radar altimetry bands. Radar altimetry is different - # enough than radar imagery that the existing SAR extension doesn't - # quite work (plus, the SAR extension doesn't have a band object). - # We'll use a band construct similar to eo:bands, but follow the - # naming and unit conventions in the SAR extension. - if "sral:bands" in asset: - asset["s3:altimetry_bands"] = asset.pop("sral:bands") - for band in asset["s3:altimetry_bands"]: - band["frequency_band"] = band.pop("name") - band["center_frequency"] = hz2ghz(band.pop("central_frequency")) - band["band_width"] = hz2ghz(band.pop("band_width_in_Hz")) - - # Some titles are just the filenames - if asset_key == "eopmetadata" or asset_key == "browse_jpg": - asset.pop("title", None) - - # Make asset keys kebab-case - new_asset_key = "" - for first, second in zip(asset_key, asset_key[1:]): - new_asset_key += first - if first.islower() and second.isupper(): - new_asset_key += "-" - new_asset_key += asset_key[-1] - new_asset_key = new_asset_key.replace("_", "-") - new_asset_key = new_asset_key.lower() - if asset_key == "eopmetadata": - new_asset_key = "eop-metadata" - new_assets[new_asset_key] = asset - - item_dict["assets"] = new_assets - - # ---- GEOMETRY ---- - geometry_dict = item_dict["geometry"] - assert geometry_dict["type"] == "Polygon" - - if item_dict["properties"]["s3:product_name"] in [ - "synergy-v10", - "synergy-vg1", - ]: - max_delta_lon = 300 - else: - max_delta_lon = 120 - - coords = geometry_dict["coordinates"][0] - winding = get_winding(coords, max_delta_lon) - if winding == "CW": - geometry_dict["coordinates"][0] = coords[::-1] - elif winding is None: - logger.warning( - f"Could not determine winding order of polygon in " - f"Item: '{item_dict['id']}'" - ) - - geometry = shapely.geometry.shape(geometry_dict) - - # slstr-lst strip geometries are incorrect, so we apply a hack - if item_dict["properties"][ - "s3:product_name" - ] == "slstr-lst" and original_id.endswith("_____"): - geometry = antimeridian.fix_polygon(geometry, force_north_pole=True) - else: - geometry = antimeridian.fix_polygon(geometry) - - if not geometry.is_valid: - geometry = geometry.buffer(0) - - item_dict["bbox"] = list(geometry.bounds) - item_dict["geometry"] = shapely.geometry.mapping(geometry) - item_dict["geometry"]["coordinates"] = recursive_round( - list(item_dict["geometry"]["coordinates"]), precision=4 - ) - item_dict["bbox"] = recursive_round(list(item_dict["bbox"]), precision=4) - - return item_dict diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_olci_lfr_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_olci_lfr_l2_netcdf.py deleted file mode 100644 index ac0c2179..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_olci_lfr_l2_netcdf.py +++ /dev/null @@ -1,55 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "safe-manifest": "SAFE product manifest", - "gifapar": "Green instantaneous Fraction of Absorbed Photosynthetically Active Radiation (FAPAR)", # noqa - "ogvi": "OLCI Global Vegetation Index (OGVI)", - "otci": "OLCI Terrestrial Chlorophyll Index (OTCI)", - "iwv": "Integrated water vapour column", - "rc-ogvi": "Rectified reflectance", - "rc-gifapar": "Rectified reflectance", - "lqsf": "Land quality and science flags", - "time-coordinates": "Time coordinate annotations", - "geo-coordinates": "Geo coordinate annotations", - "tie-geo-coordinates": "Tie-point geo coordinate annotations", - "tie-geometries": "Tie-point geometry annotations", - "tie-meteo": "Tie-point meteo annotations", - "instrument-data": "Instrument annotations", -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - # Skip any NT scenes - return [] - - # Grab the shape; we'll move it to assets to be consistent with the - # other collections - shape = item_dict["properties"].pop("s3:shape") - - for asset_key, asset in item_dict["assets"].items(): - if "resolution" in asset: - # flip to row, column order - asset["s3:spatial_resolution"] = asset.pop("resolution")[::-1] - # add shape, flip to row, column order - asset["s3:shape"] = shape[::-1] - - # clean up descriptions - if asset_key in ASSET_DESCRIPTIONS: - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_olci_wfr_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_olci_wfr_l2_netcdf.py deleted file mode 100644 index 5abbe6df..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_olci_wfr_l2_netcdf.py +++ /dev/null @@ -1,56 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "chl-nn": "Neural net chlorophyll concentration", - "chl-oc4me": "OC4Me algorithm chlorophyll concentration", - "iop-nn": "Inherent optical properties of water", - "iwv": "Integrated water vapour column", - "par": "Photosynthetically active radiation", - "w-aer": "Aerosol over water", - "geo-coordinates": "Geo coordinate annotations", - "instrument-data": "Instrument annotations", - "tie-geo-coordinates": "Tie-point geo coordinate annotations", - "tie-geometries": "Tie-point geometry annotations", - "tie-meteo": "Tie-point meteo annotations", - "time-coordinates": "Time coordinate annotations", - "wqsf": "Water quality and science flags", - "eop-metadata": "Metadata produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 - "browse-jpg": "Preview image produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - # Skip any NT scenes - return [] - - # Grab the shape; we'll move it to assets to be consistent with the - # other collections - shape = item_dict["properties"].pop("s3:shape") - - for asset_key, asset in item_dict["assets"].items(): - if "resolution" in asset: - # flip to row, column order - asset["s3:spatial_resolution"] = asset.pop("resolution")[::-1] - # add shape, flip to row, column order - asset["s3:shape"] = shape[::-1] - - # clean up descriptions - if asset_key in ASSET_DESCRIPTIONS: - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_frp_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_frp_l2_netcdf.py deleted file mode 100644 index 73e98048..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_frp_l2_netcdf.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - # Grab the shape; we'll move it to assets to be consistent with the - # other collections - shape = item_dict["properties"].pop("s3:shape") - - for asset in item_dict["assets"].values(): - if "resolution" in asset: - # flip to row, column order - asset["s3:spatial_resolution"] = asset.pop("resolution")[::-1] - # add shape, flip to row, column order - asset["s3:shape"] = shape[::-1] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_lst_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_lst_l2_netcdf.py deleted file mode 100644 index fd8226ac..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_lst_l2_netcdf.py +++ /dev/null @@ -1,53 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "safe-manifest": "SAFE product manifest", - "lst-in": "Land Surface Temperature (LST) values", - "st-ancillary-ds": "LST ancillary measurement dataset", - "slstr-flags-in": "Global flags for the 1km TIR grid, nadir view", - "slstr-indices-in": "Scan, pixel and detector indices annotations for the 1km TIR grid, nadir view", # noqa: E501 - "slstr-time-in": "Time annotations for the 1km grid", - "slstr-geodetic-in": "Full resolution geodetic coordinates for the 1km TIR grid, nadir view", - "slstr-cartesian-in": "Full resolution cartesian coordinates for the 1km TIR grid, nadir view", - "slstr-geometry-tn": "16km solar and satellite geometry annotations, nadir view", - "slstr-geodetic-tx": "16km geodetic coordinates", - "slstr-cartesian-tx": "16km cartesian coordinates", - "slstr-met-tx": "Meteorological parameters regridded onto the 16km tie points", -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - # Grab the shape; we'll move it to assets to be consistent with the - # other collections - shape = item_dict["properties"].pop("s3:shape") - - for asset_key, asset in item_dict["assets"].items(): - if "resolution" in asset: - # flip to row, column order - asset["s3:spatial_resolution"] = asset.pop("resolution")[::-1] - # add shape, flip to row, column order - asset["s3:shape"] = shape[::-1] - - # clean up descriptions - if asset_key in ASSET_DESCRIPTIONS: - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_wst_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_wst_l2_netcdf.py deleted file mode 100644 index 5d0324b8..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_slstr_wst_l2_netcdf.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "safe-manifest": "SAFE product manifest", - "l2p": "Skin Sea Surface Temperature (SST) values", - "eop-metadata": "Metadata produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 - "browse-jpg": "Preview image produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa: E501 -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - # Grab the shape; we'll move it to assets to be consistent with the - # other collections - shape = item_dict["properties"].pop("s3:shape") - - for asset_key, asset in item_dict["assets"].items(): - if "resolution" in asset: - # resolution is a text string, not a list of numbers - assert asset["resolution"] == "1 km at nadir" - asset.pop("resolution") - asset["s3:spatial_resolution"] = [1000, 1000] - # add shape, flip to row, column order - asset["s3:shape"] = shape[::-1] - - # clean up descriptions - if asset_key in ASSET_DESCRIPTIONS: - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_sral_lan_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_sral_lan_l2_netcdf.py deleted file mode 100644 index 21758a9a..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_sral_lan_l2_netcdf.py +++ /dev/null @@ -1,47 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "safe-manifest": "SAFE product manifest", - "standard-measurement": "Standard measurement data file", - "enhanced-measurement": "Enhanced measurement data file", - "reduced-measurement": "Reduced measurement data file", -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - for asset_key, asset in item_dict["assets"].items(): - if "shape" in asset: - # Shape is a list of single entry dicts, each dict containing - # the length of a series of 1D variables. Removing since we - # can't format to match the other collections, which are a - # single 2D list for the primary 2D variable. - asset.pop("shape") - - # clean up descriptions - if asset_key in ASSET_DESCRIPTIONS: - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - # eo extension is not used - item_dict["stac_extensions"] = [ - ext for ext in item_dict["stac_extensions"] if "/eo/" not in ext - ] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_sral_wat_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_sral_wat_l2_netcdf.py deleted file mode 100644 index 11e8b092..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_sral_wat_l2_netcdf.py +++ /dev/null @@ -1,48 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "safe-manifest": "SAFE product manifest", - "standard-measurement": "Standard measurement data file", - "enhanced-measurement": "Enhanced measurement data file", - "reduced-measurement": "Reduced measurement data file", - "eop-metadata": "Product metadata file produced by the European Organisation for the Exploitation of Meteorological Satellites (EUMETSAT)", # noqa -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - for asset_key, asset in item_dict["assets"].items(): - if "shape" in asset: - # Shape is a list of single entry dicts containing the length - # of a series of 1D variables. Removing since we can't format - # to match the other collections, which are a single 2D list for - # the primary 2D variable. - asset.pop("shape") - - # clean up descriptions - if asset_key in ASSET_DESCRIPTIONS: - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - # eo extension is not used - item_dict["stac_extensions"] = [ - ext for ext in item_dict["stac_extensions"] if "/eo/" not in ext - ] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_aod_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_aod_l2_netcdf.py deleted file mode 100644 index c47007e4..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_aod_l2_netcdf.py +++ /dev/null @@ -1,36 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - # Grab the custom shape field and place on the assets for consistency - # with the other sentinel-3 collections - s3_shape = item_dict["properties"].pop("s3:shape") - - for asset_key, asset in item_dict["assets"].items(): - if asset_key == "ntc-aod": - # Place the custom shape field on the asset. Reverse the order - # to be in [row, column] order. - asset["s3:shape"] = s3_shape[::-1] - - # Reverse the provided resolution order to match the shape order - asset["s3:spatial_resolution"] = asset.pop("resolution")[::-1] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_syn_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_syn_l2_netcdf.py deleted file mode 100644 index 703ab6ff..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_syn_l2_netcdf.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - for asset in item_dict["assets"].values(): - # standardize the shape property - if "syn:shape" in asset: - assert "resolution" in asset - shape = asset.pop("syn:shape") - resolution = asset.pop("resolution") - - if len(shape) == 1: - asset["s3:shape"] = [list(shape[0].values())[0]] - else: - columns = next( - (d["columns"] for d in shape if "columns" in d), False - ) - rows = next((d["rows"] for d in shape if "rows" in d), False) - # "removed pixels" are a columnar dimension. - removed_pixels = next( - (d["removed_pixels"] for d in shape if "removed_pixels" in d), - False, - ) - assert not (columns and removed_pixels) - # Use [row, column] order to align with the ordering - # provided in the netcdf descriptions and the order used by - # xarray, numpy, and rasterio. - asset["s3:shape"] = [rows, columns or removed_pixels] - - # Reverse the provided resolution order to match the shape order - asset["s3:spatial_resolution"] = resolution[::-1] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_v10_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_v10_l2_netcdf.py deleted file mode 100644 index ad7cc0dd..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_v10_l2_netcdf.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - for asset in item_dict["assets"].values(): - # standardize the shape property - if "v10:shape" in asset: - assert "resolution" in asset - shape = asset.pop("v10:shape") - resolution = asset.pop("resolution") - - latitude = next((d["latitude"] for d in shape if "latitude" in d)) - longitude = next((d["longitude"] for d in shape if "longitude" in d)) - # Use [row, column] order to align with the ordering - # provided in the netcdf descriptions and the order used by - # xarray, numpy, and rasterio. - asset["s3:shape"] = [latitude, longitude] - - # Reverse the provided resolution order to match the shape order - asset["s3:spatial_resolution"] = resolution[::-1] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vg1_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vg1_l2_netcdf.py deleted file mode 100644 index f142a15d..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vg1_l2_netcdf.py +++ /dev/null @@ -1,40 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - for asset in item_dict["assets"].values(): - # standardize the shape property - if "vg1:shape" in asset: - assert "resolution" in asset - shape = asset.pop("vg1:shape") - resolution = asset.pop("resolution") - - latitude = next((d["latitude"] for d in shape if "latitude" in d)) - longitude = next((d["longitude"] for d in shape if "longitude" in d)) - # Use [row, column] order to align with the ordering - # provided in the netcdf descriptions and the order used by - # xarray, numpy, and rasterio. - asset["s3:shape"] = [latitude, longitude] - - # Reverse the provided resolution order to match the shape order - asset["s3:spatial_resolution"] = resolution[::-1] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vgp_l2_netcdf.py b/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vgp_l2_netcdf.py deleted file mode 100644 index d99a2f7e..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_synergy_vgp_l2_netcdf.py +++ /dev/null @@ -1,59 +0,0 @@ -from typing import List, Union - -import pystac -from sentinel_3.sentinel_3_base import BaseSentinelCollection - -from pctasks.core.models.task import WaitTaskResult -from pctasks.core.storage import StorageFactory - -ASSET_DESCRIPTIONS = { - "safe-manifest": "SAFE product manifest", - "b0": "Top of atmosphere reflectance data set associated with the VGT-B0 channel", - "b2": "Top of atmosphere reflectance data set associated with the VGT-B2 channel", - "b3": "Top of atmosphere reflectance data set associated with the VGT-B3 channel", - "mir": "Top of atmosphere Reflectance data set associated with the VGT-MIR channel", - "vaa": "View azimuth angle data", - "vza": "View zenith angle data", - "saa": "Solar azimuth angle data", - "sza": "Solar zenith angle data", - "ag": "Aerosol optical thickness data", - "og": "Total ozone column data", - "wvg": "Total column water vapour data", - "sm": "Status map data", -} - - -class Collection(BaseSentinelCollection): - @classmethod - def create_item( - cls, asset_uri: str, storage_factory: StorageFactory - ) -> Union[List[pystac.Item], WaitTaskResult]: - - storage, json_path = storage_factory.get_storage_for_file(asset_uri) - item_dict = storage.read_json(json_path) - - item_dict = cls.base_updates(item_dict) - if item_dict is None: - return [] - - for asset_key, asset in item_dict["assets"].items(): - # standardize the shape property - if "vgp:shape" in asset: - assert "resolution" in asset - shape = asset.pop("vgp:shape") - resolution = asset.pop("resolution") - - latitude = next((d["latitude"] for d in shape if "latitude" in d)) - longitude = next((d["longitude"] for d in shape if "longitude" in d)) - # Use [row, column] order to align with the ordering - # provided in the netcdf descriptions and the order used by - # xarray, numpy, and rasterio. - asset["s3:shape"] = [latitude, longitude] - - # Reverse the provided resolution order to match the shape order - asset["s3:spatial_resolution"] = resolution[::-1] - - # Add missing descriptions - asset["description"] = ASSET_DESCRIPTIONS[asset_key] - - return [pystac.Item.from_dict(item_dict)] diff --git a/datasets/sentinel-3/sentinel_3/sentinel_3_winding.py b/datasets/sentinel-3/sentinel_3/sentinel_3_winding.py deleted file mode 100644 index b304dcf0..00000000 --- a/datasets/sentinel-3/sentinel_3/sentinel_3_winding.py +++ /dev/null @@ -1,97 +0,0 @@ -from itertools import groupby -from typing import List, Optional - - -def crossing_longitude( - coord1: List[float], coord2: List[float], center_lat: float, max_delta_lon: float -) -> float: - # check if we are interpolating across the antimeridian - delta_lon_1 = abs(coord2[0] - coord1[0]) - if delta_lon_1 > max_delta_lon: - delta_lon_2 = 360 - delta_lon_1 - if coord1[0] < 0: - coord2[0] = coord1[0] - delta_lon_2 - else: - coord2[0] = coord1[0] + delta_lon_2 - - # interpolate - crossing_longitude = ((center_lat - coord1[1]) / (coord2[1] - coord1[1])) * ( - coord2[0] - coord1[0] - ) + coord1[0] - - # force interpolated longitude to be in the range [-180, 180] - crossing_longitude = ((crossing_longitude + 180) % 360) - 180 - - return crossing_longitude - - -def ccw_or_cw(lon_crossings: List[List[float]], max_delta_lon: float) -> Optional[str]: - for cross1, cross2 in zip(lon_crossings, lon_crossings[1:]): - delta_lon = cross2[0] - cross1[0] - if delta_lon < max_delta_lon: - if cross1[1] != -cross2[1]: - raise ValueError("Crossings should be in opposite directions") - if cross1[1] == -1: - return "CCW" - else: - return "CW" - return None - - -def get_winding(coords: List[List[float]], max_delta_lon: float) -> Optional[str]: - """Heuristic method for determining the winding for complex Sentinel-3 - 'strip' polygons that self-intersect and overlap and for simple Sentinel-3 - 'chip' polygons that may cross the antimeridian. - - Args: - coords (List[List[float]]): List of coordinates in the polygon. - max_delta_lon (float): This argument serves two purposes: - 1. Longitude crossings (of the center latitude of the polygon) that - are within this distance of each other are considered to be on either - side of polygon interior. - 2. When interpolating a longitude crossing, if the difference in - longitudes between the two points is greater than this value, then - we assume that we are interpolating across the antimeridian. - - --> Recommended values are 120 degrees for strips and chips, and - 300 degrees for the rectangular synergy-v10 and synergy-vg1 products. - """ - # force all longitudes to be in the range [-180, 180] - for i, point in enumerate(coords): - coords[i] = [((point[0] + 180) % 360) - 180, point[1]] - - # duplicate points will cause a divide by zero problem - coords = [c for c, _ in groupby(coords)] - - # get center latitude against which we will check for crossings - lats = [coord[1] for coord in coords] - center_lat = (max(lats) + min(lats)) / 2 - - # find all longitude crossings of the center latitude - lon_crossings = [] - for coord1, coord2 in zip(coords, coords[1:]): - if coord1[1] >= center_lat and coord2[1] < center_lat: - lon_crossings.append( - [crossing_longitude(coord1, coord2, center_lat, max_delta_lon), -1] - ) - elif coord1[1] <= center_lat and coord2[1] > center_lat: - lon_crossings.append( - [crossing_longitude(coord1, coord2, center_lat, max_delta_lon), 1] - ) - if len(lon_crossings) == 0: - raise ValueError("No crossings found") - if len(lon_crossings) % 2 != 0: - raise ValueError("Number of crossings should always be a multiple of 2") - lon_crossings = sorted(lon_crossings, key=lambda x: x[0]) - - # get winding - winding = ccw_or_cw(lon_crossings, max_delta_lon) - if winding is None: - # we could have an antimeridian crossing - for crossing in lon_crossings: - if crossing[0] < 0: - crossing[0] += 360 - lon_crossings = sorted(lon_crossings, key=lambda x: x[0]) - winding = ccw_or_cw(lon_crossings, max_delta_lon) - - return winding diff --git a/datasets/sentinel-3/update.yaml b/datasets/sentinel-3/update.yaml deleted file mode 100644 index a68216aa..00000000 --- a/datasets/sentinel-3/update.yaml +++ /dev/null @@ -1,184 +0,0 @@ -id: sentinel-3 -image: ${{ args.registry }}/pctasks-sentinel-3:2023.5.1.0 - -args: - - registry - - since - -code: - src: ${{ local.path(./sentinel_3) }} - -environment: - AZURE_TENANT_ID: ${{ secrets.task-tenant-id }} - AZURE_CLIENT_ID: ${{ secrets.task-client-id }} - AZURE_CLIENT_SECRET: ${{ secrets.task-client-secret }} - -collections: - - id: sentinel-3-olci-lfr-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-olci-lfr-l2-netcdf/) }} - class: sentinel_3.sentinel_3_olci_lfr_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - # create-chunks took ~8 minutes. Consider splitting by year. - chunks: - options: - name_starts_with: OLCI/OL_2_LFR___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-olci-lfr-l2-netcdf - - - id: sentinel-3-olci-wfr-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-olci-wfr-l2-netcdf/) }} - class: sentinel_3.sentinel_3_olci_wfr_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: OLCI/OL_2_WFR___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-olci-wfr-l2-netcdf - - - id: sentinel-3-synergy-aod-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-synergy-aod-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_aod_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SYNERGY/SY_2_AOD___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-aod-l2-netcdf - - - id: sentinel-3-synergy-syn-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-synergy-syn-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_syn_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SYNERGY/SY_2_SYN___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-syn-l2-netcdf - - - id: sentinel-3-synergy-v10-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-synergy-v10-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_v10_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SYNERGY/SY_2_V10___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-v10-l2-netcdf - - - id: sentinel-3-synergy-vg1-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-synergy-vg1-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_vg1_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SYNERGY/SY_2_VG1___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-vg1-l2-netcdf - - - id: sentinel-3-synergy-vgp-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-synergy-vgp-l2-netcdf) }} - class: sentinel_3.sentinel_3_synergy_vgp_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SYNERGY/SY_2_VGP___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-synergy-vgp-l2-netcdf - - - id: sentinel-3-sral-lan-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-sral-lan-l2-netcdf) }} - class: sentinel_3.sentinel_3_sral_lan_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SRAL/SR_2_LAN___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-sral-lan-l2-netcdf - - - id: sentinel-3-sral-wat-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-sral-wat-l2-netcdf) }} - class: sentinel_3.sentinel_3_sral_wat_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SRAL/SR_2_WAT___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-sral-wat-l2-netcdf - - - id: sentinel-3-slstr-frp-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-slstr-frp-l2-netcdf) }} - class: sentinel_3.sentinel_3_slstr_frp_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SLSTR/SL_2_FRP___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-frp-l2-netcdf - - - id: sentinel-3-slstr-lst-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-slstr-lst-l2-netcdf) }} - class: sentinel_3.sentinel_3_slstr_lst_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SLSTR/SL_2_LST___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-lst-l2-netcdf - - - id: sentinel-3-slstr-wst-l2-netcdf - template: ${{ local.path(./collection/sentinel-3-slstr-wst-l2-netcdf) }} - class: sentinel_3.sentinel_3_slstr_wst_l2_netcdf:Collection - asset_storage: - - uri: blob://sentinel3euwest/sentinel-3-stac/ - token: ${{ pc.get_token(sentinel3euwest, sentinel-3-stac) }} - chunks: - options: - name_starts_with: SLSTR/SL_2_WST___/ - extensions: [.json] - chunk_length: 5000 - chunk_storage: - uri: blob://sentinel3euwest/sentinel-3-etl-data/pctasks-chunks/sentinel-3-slstr-wst-l2-netcdf \ No newline at end of file