Merge pull request #164 from ACCESS-Community-Hub/prerelease

Release of v1.0
ACCESS-Community-Hub · Aug 23, 2024 · 4486426 · 4486426
2 parents c90ae11 + d22f933
commit 4486426
Show file tree

Hide file tree

Showing 129 changed files with 2,541 additions and 1,307 deletions.
diff --git a/.github/workflows/mopper-conda-release.yaml b/.github/workflows/mopper-conda-release.yaml
@@ -0,0 +1,36 @@
+name: Build of mopper conda package for new release
+
+# Controls when the action will run.
+on:
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  conda_deployment_with_new_tag:
+    name: Test conda deployment of package with Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4.1.7
+        with:
+          fetch-depth: 0
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Conda environment creation and activation
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: mopper_env 
+          environment-file: conda/environment.yaml    # Path to the build conda environment
+          show-channel-urls: true #
+      - name: Build but do not upload the conda packages
+        uses: uibcdf/action-build-and-upload-conda-packages@v1.3.0
+        with:
+          meta_yaml_dir: conda
+          python-version: ${{ matrix.python-version }} # Values previously defined in `matrix`
+          user: coecms
+          label: auto
+          upload: true
+          token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret
diff --git a/.github/workflows/mopper-pytest.yaml b/.github/workflows/mopper-pytest.yaml
@@ -0,0 +1,61 @@
+name: mopper-all-tests
+
+#on: [push]
+on: 
+  push:
+    branches:
+      - prerelease
+      - main
+  pull_request:
+    branches:
+      - main
+      - prerelease
+
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - name: Set up Python 3.10/3.11
+      uses: actions/setup-python@v5.1.1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Miniconda
+      uses: conda-incubator/setup-miniconda@v3.0.4
+      with:
+        activate-environment: testenv
+        environment-file: conda/testenv.yaml
+        python-version: ${{ matrix.python-version }}
+        channels: conda-forge
+    - name: Lint with ruff
+      shell: bash -el {0}
+      run: |
+        ruff check --output-format=github .
+      continue-on-error: true
+
+    # making sure we are testing installed package
+    - name: Install package
+      shell: bash -el {0}
+      run: |
+        conda activate testenv
+        pip install  -e .
+    - name: Test with pytest
+      shell: bash -el {0}
+      run: |
+        conda run python -m pytest
+        #conda run coverage run --source src -m py.test
+   # - name: Upload to codecov 
+   #   shell: bash -el {0}
+   #   if: steps.build.outcome == 'success'
+   #   run: |
+   #     curl -Os https://uploader.codecov.io/latest/linux/codecov
+   #     chmod +x codecov
+   #     ./codecov
+
diff --git a/.github/workflows/mopper-test-calcs.yaml b/.github/workflows/mopper-test-calcs.yaml
@@ -0,0 +1,49 @@
+# this workflow can be used as a template for a worklfow
+# that runs automatically only specific tests when pushing 
+# to a selected branch. 
+name: mopper-specific-tests
+
+on: 
+  push:
+    branches:
+      - class
+
+
+jobs:
+  build-linux:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    strategy:
+      max-parallel: 5
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+
+    steps:
+    - uses: actions/checkout@v4.1.7
+    - name: Set up Python 3.10/3.11
+      uses: actions/setup-python@v5.1.1
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install Miniconda
+      uses: conda-incubator/setup-miniconda@v3.0.4
+      with:
+        activate-environment: testenv
+        environment-file: conda/testenv.yaml
+        python-version: ${{ matrix.python-version }}
+        channels: conda-forge
+    - name: Lint with ruff
+      shell: bash -el {0}
+      run: |
+        ruff check --output-format=github .
+      continue-on-error: true
+
+    # making sure we are testing installed package
+    - name: Install package
+      shell: bash -el {0}
+      run: |
+        conda activate testenv
+        pip install  -e .
+    - name: Test with pytest
+      shell: bash -el {0}
+      run: |
+        conda run python -m pytest -q tests/test_calculations.py
diff --git a/.github/workflows/mopper-test-conda.yaml b/.github/workflows/mopper-test-conda.yaml
@@ -0,0 +1,42 @@
+name: Test Build of mopper conda package
+
+# Controls when the action will run.
+on:
+  # Triggers the workflow on push or pull request events but only for the prerelase branch
+  push:
+    branches: [ prerelease ]
+  pull_request:
+    branches: [ prerelease ]
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+jobs:
+  conda_deployment_with_new_tag:
+    name: Test conda deployment of package with Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.10", "3.11", "3.12"]
+    steps:
+      - uses: actions/checkout@v4.1.7
+        with:
+          fetch-depth: 0
+      - name: Set env
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+      - name: Conda environment creation and activation
+        uses: conda-incubator/setup-miniconda@v3.0.4
+        with:
+          python-version: ${{ matrix.python-version }}
+          activate-environment: mopper_env 
+          environment-file: conda/environment.yaml    # Path to the build conda environment
+          show-channel-urls: true #
+      - name: Build but do not upload the conda packages
+        uses: uibcdf/action-build-and-upload-conda-packages@v1.3.0
+        with:
+          meta_yaml_dir: conda
+          python-version: ${{ matrix.python-version }} # Values previously defined in `matrix`
+          user: coecms
+          label: auto
+          upload: false
+          token: ${{ secrets.ANACONDA_TOKEN }} # Replace with the right name of your secret
diff --git a/.gitignore b/.gitignore
@@ -1,3 +1,10 @@
-custom_app4_*.sh
 __pycache__/
 build/
+mopper_venv/
+*.csv
+*.yaml
+*.json
+localdata/
+src/mopper.egg-info/
+extras/
+*.txt
diff --git a/ACDD_conf.yaml b/ACDD_conf.yaml
@@ -83,16 +83,25 @@ cmor:
     grids: "ACDD_grids.json"
   # Additional NCI information:
     # NCI project to charge compute; $PROJECT = your default project
-    # NCI queue to use; hugemem is recommended
     project: v45 
     # additional NCI projects to be included in the storage flags
     addprojs: []
-    # queue and memory (GB) per CPU (depends on queue) 
+    # queue and memory (GB) per CPU (depends on queue),
+    # hugemem is reccomended for high reoslution data and/or derived variables 
+    # hugemem requires a minimum of 6 cpus this is handled by the code
     queue: hugemem
     mem_per_cpu: 32 
     # walltime in "hh:mm:ss"
     walltime: '8:00:00'
     mode: custom
+    # conda_env to use by default hh5 analysis3-unstable
+    # as this has the code and all dependecies installed
+    # you can override that by supplying the env to pass to "source"
+    # Ex 
+    # conda_env: <custom-env-path>/bin/activate
+    # or you can set "test: true" and modify mopper_job.sh manually
+    conda_env: default
+
 #
 # Global attributes: these will be added to each files comment unwanted ones
 # Using ACDD CV vocab to check validity of global attributes
@@ -165,4 +174,4 @@ attrs:
     parent: !!bool false 
     # CMOR will add a tracking_id if you want to define a prefix add here
     tracking_id_prefix: 
-    comment: "post-processed using ACCESS-MOPPeR v0.6.0 https://doi.org/10.5281/zenodo.10346216"
+    comment: "post-processed using ACCESS-MOPPeR v1.0.0 https://doi.org/10.5281/zenodo.10346216"
diff --git a/CMIP6_conf.yaml b/CMIP6_conf.yaml
@@ -84,11 +84,15 @@ cmor:
     # additional NCI projects to be included in the storage flags
     addprojs: []
     # queue and memory (GB) per CPU (depends on queue) 
+    # hugemem is reccomended for high reoslution data and/or derived variables
+    # hugemem requires a minimum of 6 cpus this is handled by the code
     queue: hugemem
     mem_per_cpu: 30
     # walltime in "hh:mm:ss"
     walltime: '8:00:00'
     mode: cmip6
+    # conda_env: <custom-env-path>/bin/activate
+    conda_env: default
 
 # Global attributes: these will be added to each files comment unwanted ones
 # the labels CMIP6/ACDD indicates which ones are necessary to comply with respective standards
@@ -159,4 +163,4 @@ attrs:
     #CMOR will add a tracking_id if you want to define a prefix add here
     tracking_id_prefix:
     Conventions: "CF-1.7 CMIP-6.2" 
-    comment: "post-processed using ACCESS-MOPPeR v0.6.0 https://doi.org/10.5281/zenodo.10346216"
+    comment: "post-processed using ACCESS-MOPPeR v1.0.0 https://doi.org/10.5281/zenodo.10346216"
diff --git a/README.md b/README.md
@@ -1,6 +1,6 @@
 # [ACCESS Model Output Post-Processor (MOPPeR)](https://access-mopper.readthedocs.io/en/latest)
 [![Read the docs](https://readthedocs.org/projects/access-mopper/badge/?version=latest)](https://access-mopper.readthedocs.io/en/latest/)
-[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.10976467.svg)](https://doi.org/10.5281/zenodo.10976467)
+[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.12747219.svg)](https://doi.org/10.5281/zenodo.12747219)
 
 This code is derived from the [APP4](https://doi.org/10.5281/zenodo.7703469), initially created by Peter Uhe for CMIP5, and further developed for CMIP6-era by Chloe Mackallah from CSIRO, O&A Aspendale.
 
@@ -15,12 +15,13 @@ Designed for use on ACCESS model output that has been archived using the [ACCESS
 
 Although we retained a differentiation between `custom` and `cmip` mode the main workflow is the same and `mode` is now only another field in the main  configuration file.
 
+See [MOPPeR ReadtheDocs](https://access-mopper.readthedocs.io/en/stable/) for the full documentation.
 
 ### Install
 
 You can install the latest version of `mopper` directly from conda (accessnri channel)::
 
-   conda install -c accessnri mopper
+   conda install -c coecms mopper
 
 If you want to install an unstable version or a different branch:
 
@@ -35,6 +36,6 @@ If you want to install an unstable version or a different branch:
 MOPPeR is pre-installed into a Conda environment at NCI. Load it with::
 
     module use /g/data3/hh5/public/modules
-    module load conda/analysis3-unstable
+    module load conda/analysis3
 
   NB. You need to be a member of the hh5 project to load the modules.
diff --git a/conda/environment.yaml b/conda/environment.yaml
@@ -0,0 +1,10 @@
+name: mopper_env
+channels:
+  - conda-forge
+  - coecms
+  - default
+
+dependencies:
+  - anaconda-client
+  - conda-build
+  - conda-verify
diff --git a/conda/meta.yaml b/conda/meta.yaml
@@ -1,19 +1,19 @@
-{% set version = "0.6.1" %}
 package:
     name: mopper 
-    version: {{ version }}
+    version: "{{ environ.get('GIT_DESCRIBE_TAG', '1.0') }}"
 
 #source:
 #    path: ./
 
 source:
   #url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR/archive/refs/tags/{{version}}.tar.gz
   git_url: https://github.com/ACCESS-Hive/ACCESS-MOPPeR.git
-  git_rev: {{ version }} 
-  git_depth: 1 # (Defaults to -1/not shallow)
+  #git_tag: prerelease
+  git_rev: "{{ version }}" 
+  #git_depth: 1 # (Defaults to -1/not shallow)
 
 build:
-    number: 0
+    number: 1
     noarch: python
     script: "{{ PYTHON }} -m pip install . --no-deps --ignore-installed"
     entry_points:
@@ -24,7 +24,6 @@ requirements:
     host:
         - python
         - pip
-        - pbr
     run:
         - python
         - click
@@ -35,12 +34,29 @@ requirements:
         - pyyaml
         - cftime
         - python-dateutil
+
 test:
+  #imports:
+  #  - mopdb
+  #  - mopper
   source_files:
+    - tests
     - tests/testdata
+  requires:
+    - cmor
+    - xarray
+    - numpy
+    - dask
+    - pyyaml
+    - cftime
+    - python-dateutil
+    - pytest
+    - pyfakefs
+    - coverage
+    - codecov
 
 about:
     home: https://github.com/ACCESS-Hive/ACCESS-MOPPeR
     license: Apache 2.0
     #license_file: LICENSE.txt
-    summary: 'ACCESS-MOPPeR post-process ACCESS raw model output to ESGF data standards' 
+    summary: 'ACCESS-MOPPeR post-process ACCESS raw model output using CMOR and pre-defined data standards' 
diff --git a/conda/run_test.sh b/conda/run_test.sh
@@ -1,2 +1,3 @@
 #!/bin/bash
-py.test
+echo 'calling run_test'
+python -m pytest
diff --git a/conda/run_test_coverage.sh b/conda/run_test_coverage.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
 pip install coverage pytest-cov
-py.test --cov=mopper --cov-report xml:/tmp/artefacts/tests/pytest/coverage.xml --junit-xml /tmp/artefacts/tests/pytest/results.xml
-py.test --cov=mopdb --cov-report xml:/tmp/artefacts/tests/pytest/coverage.xml --junit-xml /tmp/artefacts/tests/pytest/results.xml
+python -m pytest --cov=mopper --cov-report xml:/tmp/artefacts/tests/pytest/coverage.xml --junit-xml /tmp/artefacts/tests/pytest/results.xml
+python -m pytest --cov=mopdb --cov-report xml:/tmp/artefacts/tests/pytest/coverage.xml --junit-xml /tmp/artefacts/tests/pytest/results.xml
 
diff --git a/conda/testenv.yaml b/conda/testenv.yaml
@@ -0,0 +1,18 @@
+name: testenv
+channels:
+  - conda-forge
+
+dependencies:
+  - cmor
+  - click
+  - xarray
+  - numpy
+  - dask
+  - pyyaml
+  - cftime
+  - python-dateutil
+  - pytest 
+  - coverage 
+  - codecov
+  - pyfakefs
+  - ruff