HARMONY-1874: As a harmony service provider, I want to execute a regr…

…ession test based on receiving a deployment event in the harmony-regression-tests repo (#98)

Co-authored-by: Owen Littlejohns <owen.m.littlejohns@nasa.gov>

.github/workflows/notebook-test-suite.yml

@@ -0,0 +1,177 @@
+# This workflow runs Jupyter notebook based regression test suites.
+# It can be run manually run on github Actions or invoked through
+# the github Action workflow api.
+# When invoked through the github Action workflow api, a event type needs
+# to be provided. The event type should be either the name of one of the
+# Harmony services or 'all'.
+# The notebook test suites associated with Harmony service matching the event
+# will be executed, or all the notebook test suites will be exectued when 'all'
+# is provided which is triggered when a Harmonny server deployment occurs.
+#
+# Required settings in the repository:
+# Environments (exactly the following strings):
+# - prod
+# - uat
+#
+# Secrets in each environment:
+# - EDL_USER
+# - EDL_PASSWORD
+#
+# Note: A workflow_call event can only have inputs of type boolean, number or
+# string.
+name: Run Jupyter notebook based test suite
+run-name: ${{ github.event.action || inputs.service-name }} service regression tests
+
+on:
+  workflow_call:
+    inputs:
+      harmony-environment:
+        required: true
+        type: string
+      docker-image-tag:
+        required: false
+        type: string
+        default: 'latest'
+
+  workflow_dispatch:
+    inputs:
+      harmony-environment:
+        description: "Select the Harmony environment to run the test in"
+        required: true
+        type: choice
+        options:
+          - uat
+          - production
+      service-name:
+        description: "Select the service to run tests against"
+        required: true
+        type: choice
+        options:
+          - geoloco
+          - harmony
+          - harmony-gdal-adapter
+          # - harmony-netcdf-to-zarr  # Commented out because needs AWS credentials
+          - harmony-regression
+          - harmony-regridder
+          - harmony-service-example
+          - hoss
+          - hybig
+          - query-cmr
+          - subset-band-name
+          - swath-projector
+          - trajectory-subsetter
+      docker-image-tag:
+        description: "Optional Docker image tag to use for the tests"
+        required: false
+        type: string
+        default: 'latest'
+
+  repository_dispatch:
+    types: [all,geoloco,harmony-gdal-adapter,harmony-netcdf-to-zarr,harmony-service-example,hoss,hybig,net2cog,swath-projector,trajectory-subsetter]
+    inputs:
+      harmony-environment:
+        required: true
+        type: choice
+        options:
+          - uat
+          - prod
+      docker-image-tag:
+        required: false
+        type: string
+        default: 'latest'
+
+jobs:
+  base:
+    environment: ${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.create-matrix.outputs.matrix }}
+      harmony_host_url: ${{ env.HARMONY_HOST_URL }}
+      run_url: ${{ env.RUN_URL }}
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Get the URL of the current workflow run
+        run: echo "RUN_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" >> $GITHUB_ENV
+
+      - name: Set HARMONY_HOST_URL
+        run: |
+          if [ "${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}" == "uat" ]; then
+            echo "HARMONY_HOST_URL=https://harmony.uat.earthdata.nasa.gov"
+            echo "HARMONY_HOST_URL=https://harmony.uat.earthdata.nasa.gov" >> $GITHUB_ENV
+          elif [ "${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}" == "prod" ]; then
+            echo "HARMONY_HOST_URL=https://harmony.earthdata.nasa.gov"
+            echo "HARMONY_HOST_URL=https://harmony.earthdata.nasa.gov" >> $GITHUB_ENV
+          else
+            echo "HARMONY_HOST_URL=https://harmony.sit.earthdata.nasa.gov"
+            echo "HARMONY_HOST_URL=https://harmony.sit.earthdata.nasa.gov" >> $GITHUB_ENV
+          fi
+
+      - name: Create matrix configuration from Environment Variable
+        id: create-matrix
+        run: |
+          # Use the event type to select the appropriate service list
+          SERVICES_TESTS=$(cat ./config/services_tests_config_${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}.json | jq -r '."${{ github.event.action || inputs.service-name }}"')
+          # Split the selected service list into an array format and set it as the matrix
+          MATRIX=$(echo ",$SERVICES_TESTS" | tr ',' '\n' | awk '{$1=$1;print}' | jq -c -R '[inputs] | {service: .}')
+          echo "matrix=$MATRIX" >> $GITHUB_OUTPUT
+
+  test:
+    needs: base
+    environment: ${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}
+    runs-on: ubuntu-latest
+    outputs:
+      run_url: ${{ needs.base.outputs.run_url }}
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.base.outputs.matrix) }}
+
+    steps:
+      - name: Checkout the repository
+        uses: actions/checkout@v4
+
+      - name: Pull Docker image
+        run: docker pull ghcr.io/nasa/regression-tests-${{ matrix.service }}:${{ github.event.client_payload.docker-image-tag || 'latest' }}
+
+      - name: Execute notebook
+        id: test-step
+        working-directory: ./test
+        env:
+          EDL_USER: ${{ secrets.EDL_USER }}
+          EDL_PASSWORD: ${{ secrets.EDL_PASSWORD }}
+          HARMONY_HOST_URL: ${{ needs.base.outputs.harmony_host_url }}
+        run: ./run_notebooks.sh ${{ matrix.service }}
+
+      - name: Save notebook as an artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: notebook-output-${{ matrix.service }}
+          path: test/output/${{ matrix.service }}/Results.ipynb
+
+  email:
+    needs: test
+    if: always()
+    environment: ${{ github.event.client_payload.harmony-environment || inputs.harmony-environment }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get TO_EMAIL
+        run: |
+          SERVICE_EMAILS=$(echo '${{ vars.SERVICES_EMAILS_CONFIG }}' | jq -r '."${{ github.event.action || inputs.service-name }}"')
+          if [ "$SERVICE_EMAILS" == "null" ]; then
+            SERVICE_EMAILS=${{ vars.TO_EMAIL_DEFAULT }}
+          fi
+          echo "TO_EMAIL=$SERVICE_EMAILS" >> $GITHUB_ENV
+
+      - name: Send mail
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{ secrets.SMTP_USER }}
+          password: ${{ secrets.SMTP_PASSWORD }}
+          subject: ${{ github.event.action || inputs.service-name }} regression test result is ${{ needs.test.result }}
+          to: ${{ env.TO_EMAIL }}
+          from: ${{ secrets.FROM_EMAIL }}
+          body: ${{ github.event.action || inputs.service-name }} regression test result is ${{ needs.test.result }}! View the details at ${{ needs.test.outputs.run_url }}.

README.md

@@ -25,6 +25,21 @@ installed on your system, as it's used to manage some large files stored in
 GitHub.
 
 
+## Running the Tests in GitHub:
+
+Each test suite can be individually invoked via a GitHub workflow. Navigate to
+the [GitHub Actions tab](https://github.com/nasa/harmony-regression-tests/actions)
+for this repository. Then select the "Run test suite" workflow from the lefthand
+menu. On the right hand side, click the "Run workflow" dropdown, and select the
+correct Docker image and Harmony environment. That should manually trigger the
+workflow.
+
+The regression test GitHub actions can also be invoked through different event types
+after a Harmony service is successfully deployed in Harmony or after a new version of
+the Harmony server is deployed.
+Note: Only the `latest` tag of the regression docker image will be used to run the
+Jupyter notebook tests.
+
 ## Running the Tests Locally
 
 Each test suite is run in a separate Docker container using a temporary Docker image
@@ -122,6 +137,9 @@ environment before installing from the environment.yml.
    all_images=(<pre existing test suites> <new-suite-name>)
    ```
 1. Update `script/test-in-bamboo.sh` to list the new suite name in `all_tests`.
+1. Update `config/services_tests_config_<env>.json` to associate the new suite name
+   with a Harmony service and add it to the `all` list so that it will be run when
+   the associated Harmony service or Harmony server is deployed.
 
 With this in place, the new test suite should be able to be built and run:
 

config/services_tests_config_prod.json

@@ -0,0 +1,21 @@
+{
+    "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter",
+    "batchee": "TBD",
+    "geoloco": "geoloco",
+    "giovanni-adapter": "TBD",
+    "harmony-gdal-adapter": "hga",
+    "harmony-netcdf-to-zarr": "n2z",
+    "harmony-regridder": "regridder",
+    "harmony-service-example": "harmony-regression",
+    "hoss": "hoss",
+    "hybig": "hybig",
+    "net2cog": "net2cog",
+    "podaac-concise": "TBD",
+    "podaac-l2-subsetter": "TBD",
+    "query-cmr": "geoloco",
+    "sds-maskfill": "TBD",
+    "stitchee": "TBD",
+    "subset-band-name": "subset-band-name",
+    "swath-projector": "swath-projector",
+    "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter"
+}

config/services_tests_config_uat.json

@@ -0,0 +1,21 @@
+{
+    "all": "harmony,harmony-regression,geoloco,hga,n2z,regridder,hoss,variable-subsetter,hybig,net2cog,subset-band-name,swath-projector,nsidc-icesat2,trajectory-subsetter",
+    "batchee": "TBD",
+    "geoloco": "geoloco",
+    "giovanni-adapter": "TBD",
+    "harmony-gdal-adapter": "hga",
+    "harmony-netcdf-to-zarr": "n2z",
+    "harmony-regridder": "regridder",
+    "harmony-service-example": "harmony-regression",
+    "hoss": "hoss, variable-subsetter",
+    "hybig": "hybig",
+    "net2cog": "net2cog",
+    "podaac-concise": "TBD",
+    "podaac-l2-subsetter": "TBD",
+    "query-cmr": "variable-subsetter",
+    "sds-maskfill": "TBD",
+    "stitchee": "TBD",
+    "subset-band-name": "subset-band-name",
+    "swath-projector": "swath-projector",
+    "trajectory-subsetter": "nsidc-icesat2,trajectory-subsetter"
+}