Disable AI tests in CI, only test them on Release (#77)

* SET to cheaper model * disable llm test in ci * Disable AI and test AI only on release * Fix Path * Disable AI tests Unless approved * Fix Path * Fix Path * Fix Path * Fix Path * Fix shell * Reusable Workflow * Fix README * Try reusable CI * Fallback to the more descriptive option * Remove print * Clean leftover
Certora · Jan 9, 2025 · 3384c4d · 3384c4d
1 parent 0721bb9
commit 3384c4d
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 3 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -22,7 +22,8 @@ jobs:
     env:
       ETHSCAN_API_KEY: ${{ secrets.ETHSCAN_API_KEY }}
       QUORUM_PATH: "Quorum/tests"
-      ANTHROPIC_API_KEY: ${{ github.event.review.state == 'approved' && secrets.ANTHROPIC_API_KEY || 'SKIP_TEST_KEY' }}
+      ANTHROPIC_API_KEY: SKIP_TEST_KEY
+
     runs-on: ubuntu-latest
     container:
       image: public.ecr.aws/certora/cvt-image:2024.10.23-4499-c4f8ad4

diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml
@@ -7,7 +7,68 @@ on:
   # workflow_dispatch:
 
 jobs:
+ Regression:
+    env:
+      ETHSCAN_API_KEY: ${{ secrets.ETHSCAN_API_KEY }}
+      QUORUM_PATH: "Quorum/tests"
+      ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
+
+    runs-on: ubuntu-latest
+    container:
+      image: public.ecr.aws/certora/cvt-image:2024.10.23-4499-c4f8ad4
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Poetry
+        run: pip3 install poetry==1.8.5
+
+      - name: Install Quorum Repo
+        run: poetry install
+
+      - name: Lint with Ruff
+        run: |
+          poetry run ruff check src
+          poetry run ruff format --check src
+
+      - name: Run unit tests
+        run: poetry run pytest src/quorum/tests --maxfail=1 --disable-warnings --tb=short
+
+      - name: Run Quorum setup
+        run: poetry run quorum setup --working-dir workdir
+
+      - name: Init working directory
+        run: |
+          mv src/quorum/tests/regression.json workdir/regression.json
+          mv src/quorum/tests/ground_truth.json workdir/ground_truth.json
+          echo "ETHSCAN_API_KEY=${ETHSCAN_API_KEY}" >> workdir/.env
+          echo "ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY}" >> workdir/.env
+
+      - name: Single Address Test
+        working-directory: workdir
+        run: poetry run quorum validate-address --protocol-name Aave --chain Ethereum --payload-address 0xAD6c03BF78A3Ee799b86De5aCE32Bb116eD24637
+
+      - name: Batch Test
+        working-directory: workdir
+        run: poetry run quorum validate-batch --config regression.json
+
+      - name: Proposal ID Test
+        working-directory: workdir
+        run: poetry run quorum validate-by-id --proposal-id 137 --protocol-name Aave
+
+      - name: IPFS Test
+        working-directory: workdir
+        run: poetry run quorum validate-ipfs --proposal-id 20 --chain Scroll --payload-address 0x2B25cb729D90630395Cd3140f3460a73A41Fe5f0
+
+      - name: Generate Report
+        working-directory: workdir
+        run: poetry run quorum generate-report --proposal_id 137
+
   publish:
+    needs: Regression
     runs-on: ubuntu-latest
     permissions:
       contents: write

diff --git a/README.md b/README.md
@@ -177,12 +177,12 @@ quorum validate-by-id --customer "Aave" --proposal_id 137
 quorum validate-ipfs --proposal_id 132 --chain "Ethereum" --proposal_address "0xAD6..."
 ```
 
-### 5. **create-report**
+### 5. **generate-report**
 
 **Purpose:** Generates a human-readable report of the proposal details, leveraging Jinja2 templates.
 
 ```bash
-quorum create-report --proposal_id 137 \
+quorum generate-report --proposal_id 137 \
                      --template "src/quorum/auto_report/AaveReportTemplate.md.j2" \
                      --generate_report_path "reports/v3-137.md"
 ```