Merge pull request #90 from umr-lops/replace-datatree-xarray

first attempt to replace xarray-datatree by xarray native DataTrees
umr-lops · Jan 7, 2025 · 56708ea · 56708ea
2 parents e1d9469 + a3b7dc9
commit 56708ea
Show file tree

Hide file tree

Showing 39 changed files with 1,658 additions and 1,340 deletions.
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -0,0 +1,84 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  detect-skip-ci-trigger:
+    name: "Detect CI Trigger: [skip-ci]"
+    if: |
+      github.repository == 'umr-lops/utils_xsarslc_l1b'
+      && (
+          github.event_name == 'push' || github.event_name == 'pull_request'
+      )
+    runs-on: ubuntu-latest
+    outputs:
+      triggered: ${{ steps.detect-trigger.outputs.trigger-found }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 2
+      - uses: xarray-contrib/ci-trigger@v1
+        id: detect-trigger
+        with:
+          keyword: "[skip-ci]"
+
+  ci:
+    name: ${{ matrix.os }} py${{ matrix.python-version }}
+    runs-on: ${{ matrix.os }}
+    needs: detect-skip-ci-trigger
+
+    if: needs.detect-skip-ci-trigger.outputs.triggered == 'false'
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+        os: ["ubuntu-latest", "macos-latest", "windows-latest"]
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+        with:
+          # need to fetch all tags to get a correct version
+          fetch-depth: 0 # fetch all branches and tags
+
+      - name: Setup environment variables
+        run: |
+          echo "TODAY=$(date +'%Y-%m-%d')" >> $GITHUB_ENV
+
+          echo "CONDA_ENV_FILE=ci/requirements/environment.yaml" >> $GITHUB_ENV
+
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: ${{ env.CONDA_ENV_FILE }}
+          environment-name: slcl1butils-tests
+          cache-environment: true
+          cache-environment-key: "${{runner.os}}-${{runner.arch}}-py${{matrix.python-version}}-${{env.TODAY}}-${{hashFiles(env.CONDA_ENV_FILE)}}"
+          create-args: >-
+            python=${{matrix.python-version}}
+
+      - name: Install slcl1butils
+        run: |
+          python -m pip install --no-deps -e .
+
+      - name: Import slcl1butils
+        run: |
+          python -c "import slcl1butils"
+
+      - name: Run tests
+        run: |
+          python -m pytest --cov=slcl1butils
diff --git a/ci/requirements/environment.yaml b/ci/requirements/environment.yaml
@@ -2,7 +2,7 @@ name: slcl1butils-tests
 channels:
   - conda-forge
 dependencies:
-  - python=3.10
+  - python
   # development
   - ipython
   - pre-commit
@@ -14,6 +14,7 @@ dependencies:
   # testing
   - pytest
   - pytest-reportlog
+  - pytest-cov
   - hypothesis
   - coverage
   # I/O
@@ -23,7 +24,6 @@ dependencies:
   - fsspec
   # data
   - xarray
-  - xarray-datatree
   - dask
   - numpy
   - pandas

diff --git a/docs/conf.py b/docs/conf.py
@@ -60,7 +60,6 @@
     "numpy": ("https://numpy.org/doc/stable", None),
     "xarray": ("https://docs.xarray.dev/en/latest/", None),
     "rasterio": ("https://rasterio.readthedocs.io/en/latest/", None),
-    "datatree": ("https://xarray-datatree.readthedocs.io/en/latest/", None),
     'geoviews': ('https://geoviews.org/index.html', None)
 }
 

diff --git a/docs/examples/display_a_IW_L1B_xspectra.ipynb b/docs/examples/display_a_IW_L1B_xspectra.ipynb
@@ -27,7 +27,7 @@
    "source": [
     "import glob\n",
     "import os\n",
-    "import datatree\n",
+    "import xarray as xr\n",
     "from slcl1butils.utils import get_test_file\n",
     "#l1bncfile_pattern = os.path.abspath('../../assests/*iw*nc')\n",
     "one_safe_l1b = get_test_file('S1B_IW_XSP__1SDV_20210328T055258_20210328T055325_026211_0320D4_DC31_A13.SAFE')\n",
@@ -47,7 +47,7 @@
    },
    "outputs": [],
    "source": [
-    "dt = datatree.open_datatree(l1bncfile)\n",
+    "dt = xr.open_datatree(l1bncfile)\n",
     "dt"
    ]
   },
@@ -187,7 +187,7 @@
     "tile_line_i = 0\n",
     "tile_sample_i = 3\n",
     "fig = plt.figure()\n",
-    "slcl1butils.plotting.display_one_spectra.plot_a_single_xspec_cart_L1B_IW(ds,'VV',tile_line_i,tile_sample_i,'test display',fig,cat_xspec=cat_xspec,part='Re')"
+    "slcl1butils.plotting.display_one_spectra.plot_a_single_xspec_cart_l1b_iw(ds,'VV',tile_line_i,tile_sample_i,'test display',fig,cat_xspec=cat_xspec,part='Re')"
    ]
   }
  ],
@@ -207,7 +207,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.0"
+   "version": "3.9.15"
   }
  },
  "nbformat": 4,

diff --git a/docs/examples/do_L1C_SAFE_from_L1B_SAFE_example.ipynb b/docs/examples/do_L1C_SAFE_from_L1B_SAFE_example.ipynb
@@ -108,7 +108,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.0"
+   "version": "3.9.15"
   }
  },
  "nbformat": 4,

diff --git a/docs/examples/plotting_L1B_geometry_with_holoview_example.ipynb b/docs/examples/plotting_L1B_geometry_with_holoview_example.ipynb
@@ -164,7 +164,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.0"
+   "version": "3.9.15"
   }
  },
  "nbformat": 4,

diff --git a/docs/index.rst b/docs/index.rst
@@ -6,7 +6,7 @@ slcl1butils: functions to play with IFREMER L1B Sentinel-1 SLC SAR products
 
 Acquisition modes available in L1B IFREMER product family are IW and WV.
 
-The products are *netCDF* files containing `datatree`_ object.
+The products are *netCDF* files containing `xarray.datatree` object.
 
 
 
@@ -104,5 +104,4 @@ Last documentation build: |today|
    basic_api
 
 .. _on github: https://github.com/umr-lops/utils_xsarslc_l1b
-.. _datatree: https://github.com/xarray-contrib/datatree
 .. _xarray: http://xarray.pydata.org
diff --git a/docs/installing.rst b/docs/installing.rst
@@ -4,8 +4,8 @@
 Installation
 ************
 
-L1B SAR SLC IFREMER products are netCDF (.nc) files containing groups.
-To read netCDF files with groups, a possible python library is `xarray-datatree`.
+Level-1B SAR SLC IFREMER products are netCDF (.nc) files containing groups.
+To read netCDF files with groups, a possible python library is `xarray`.
 Installation in a conda_ environment is recommended.
 
 

diff --git a/environment.yml b/environment.yml
@@ -6,7 +6,6 @@ dependencies:
   - geoviews
   - fsspec
   - numpy
-  - xarray-datatree
   - xarray
   - rasterio
   - rioxarray

diff --git a/get_polygons_from_l1b.py b/get_polygons_from_l1b.py
@@ -1,15 +1,10 @@
 # +
-import datatree
 import numpy as np
-from glob import glob
-import os
-import matplotlib.pyplot as plt
 from shapely import geometry
 from shapely import wkt
-import time
+
 import xarray as xr
 
-# from xsarslc.tools import xndindex
 
 # +
 def get_swath_tiles_polygons_from_l1bgroup(l1b_ds, swath_only=False, variable_names = [None],ik=0,burst_type=None):
@@ -176,66 +171,3 @@ def get_swath_tiles_polygons_from_l1bfiles(l1b_files, variable_names = [None], i
         return polygons,coordinates,variables
     else:
         return polygons,coordinates
-
-
-# def get_polygons_from_l1b(files, varnames = None):
-
-#     pts_tiles_intra = []; pts_tiles_inter = []; pts_swath = []
-#     t0 = time.time()
-
-#     pts_tiles = {}
-#     burst_type = ['intraburst','interburst']
-#     variables = {}
-#     for brst in burst_type:
-#         variables[brst] = {}
-#         pts_tiles[brst] = []
-#         if (varnames is not None):
-#             for varname in varnames:
-#                 variables[brst][varname] = []
-
-#     for cpt,_file in enumerate(files):
-
-#         dt = datatree.open_datatree(_file)
-
-#         polyswath = wkt.loads(dt['intraburst'].ds.attrs['footprint'])
-#         lon,lat = polyswath.exterior.xy
-#         _pts_swath = [(x,y) for x,y in zip(lon,lat)]
-#         pts_swath.append(_pts_swath)
-
-
-#         for brst in burst_type:
-
-#             lon_corners = dt[brst]['corner_longitude'].squeeze()
-#             lat_corners = dt[brst]['corner_latitude'].squeeze()
-#             Nt = dt[brst].ds.sizes['tile_sample']
-#             bursts = dt[brst]['burst'].values        
-#             for ib in bursts:
-#                 for it in np.arange(Nt):
-
-#                     # Get corner list
-#                     _lon1 = lon_corners.sel(burst=ib).isel(tile_sample=it,c_line=0)
-#                     _lon2 = lon_corners.sel(burst=ib).isel(tile_sample=it,c_line=1)
-#                     _lat1 = lat_corners.sel(burst=ib).isel(tile_sample=it,c_line=0)
-#                     _lat2 = lat_corners.sel(burst=ib).isel(tile_sample=it,c_line=1)
-#                     lon = list(_lon1.values) + list(_lon2.values[::-1])
-#                     lat = list(_lat1.values) + list(_lat2.values[::-1])
-#                     _pts_tiles = [ (x,y) for x,y in zip(lon,lat)]
-#                     pts_tiles[brst].append(_pts_tiles)
-
-#                     # Get variables
-#                     if (varnames is not None):
-#                         for varname in varnames:
-#                                 variables[brst][varname].append(dt[brst][varname].sel(burst=ib).isel(tile_sample=it).values[0])
-
-
-
-
-#     t1 = time.time()
-#     print(t1-t0)
-#     if (varnames is not None):
-#         return pts_swath, pts_tiles, variables
-#     else:
-#         return pts_swath, pts_tiles
-# -
-
-
diff --git a/high-level-tests/do_L1C_SAFE_from_L1B_SAFE.py b/high-level-tests/do_L1C_SAFE_from_L1B_SAFE.py
@@ -0,0 +1,34 @@
+import logging
+import os
+from importlib import reload
+
+from slcl1butils.get_config import get_conf
+from slcl1butils.scripts.do_IW_L1C_SAFE_from_L1B_SAFE import do_L1C_SAFE_from_L1B_SAFE
+from slcl1butils.utils import get_test_file
+
+reload(logging)
+logging.basicConfig(level=logging.INFO)
+conf = get_conf()
+one_safe_l1b = get_test_file(
+    "S1B_IW_XSP__1SDV_20210328T055258_20210328T055325_026211_0320D4_DC31_A13.SAFE"
+)
+ancillary_datasets = conf["auxilliary_dataset"]
+ancillary_datasets.pop("ww3hindcast_spectra", None)
+ancillary_datasets.pop("ww3_global_yearly_3h", None)
+full_safe_files = [one_safe_l1b]
+version = "0.1"
+for ffi, full_safe_file in enumerate(full_safe_files):
+    print("%i/%i" % (ffi, len(full_safe_files)))
+    print("===")
+    print(os.path.basename(full_safe_file))
+    print("===")
+    ret = do_L1C_SAFE_from_L1B_SAFE(
+        full_safe_file,
+        version=version,
+        outputdir=conf["iw_outputdir"],
+        ancillary_list=ancillary_datasets,
+        dev=True,
+        overwrite=True,
+    )
+    logging.info("new file: %s", ret)
+logging.info("high level check: OK (successful)")
diff --git a/pyproject.toml b/pyproject.toml
@@ -18,7 +18,6 @@ dependencies = [
     "numpy",
     "netCDF4",
     "shapely",
-    "xarray-datatree",
     'tqdm',
     "zarr",
     'fsspec',
@@ -41,14 +40,47 @@ packages = ["slcl1butils"]
 [tool.setuptools_scm]
 fallback_version = "999"
 
+[project.scripts]
+do_IW_L1C_SAFE_from_L1B_SAFE = "slcl1butils.scripts.do_IW_L1C_SAFE_from_L1B_SAFE:main"
+do_WV_L1C_SAFE_from_L1B_SAFE = "slcl1butils.scripts.do_WV_L1C_SAFE_from_L1B_SAFE:main"
+stack_WV_L1C_monthly = 'slcl1butils.compute.stack_wv_l1c_monthly:main'
+
 [tool.isort]
 profile = "black"
 skip_gitignore = true
 float_to_top = true
 default_section = "THIRDPARTY"
 known_first_party = "slcl1butils"
 
-[project.scripts]
-do_IW_L1C_SAFE_from_L1B_SAFE = "slcl1butils.scripts.do_IW_L1C_SAFE_from_L1B_SAFE:main"
-do_WV_L1C_SAFE_from_L1B_SAFE = "slcl1butils.scripts.do_WV_L1C_SAFE_from_L1B_SAFE:main"
-stack_WV_L1C_monthly = 'slcl1butils.compute.stack_wv_l1c_monthly:main'
+[tool.coverage.report]
+show_missing = true
+exclude_lines = ["pragma: no cover", "if TYPE_CHECKING"]
+
+[tool.ruff.lint]
+ignore = [
+  "E402",  # module level import not at top of file
+  "E501",  # line too long - let black worry about that
+  "E731",  # do not assign a lambda expression, use a def
+  "UP038", # type union instead of tuple for isinstance etc
+]
+select = [
+  "F",   # Pyflakes
+  "E",   # Pycodestyle
+  "I",   # isort
+  "UP",  # Pyupgrade
+  "TID", # flake8-tidy-imports
+  "W",
+]
+extend-safe-fixes = [
+  "TID252", # absolute imports
+  "UP031",  # percent string interpolation
+]
+fixable = ["I", "TID252", "UP"]
+
+[tool.ruff.lint.isort]
+known-first-party = ["safe_s1"]
+known-third-party = ["xarray", "toolz", "construct"]
+
+[tool.ruff.lint.flake8-tidy-imports]
+# Disallow all relative imports.
+ban-relative-imports = "all"
diff --git a/slcl1butils/__init__.py b/slcl1butils/__init__.py
@@ -1,7 +1,6 @@
-
-__all__ = ['utils','compute','plotting','scripts','coloc','legacy_ocean']
+__all__ = ["utils", "compute", "plotting", "scripts", "coloc", "legacy_ocean"]
 try:
     from importlib import metadata
-except ImportError: # for Python<3.8
+except ImportError:  # for Python<3.8
     import importlib_metadata as metadata
-__version__ = metadata.version('slcl1butils')
+__version__ = metadata.version("slcl1butils")