Skip to content

Commit

Permalink
Delete test_models job using SHARK-TestSuite/iree_tests. (iree-org#19614
Browse files Browse the repository at this point in the history
)

Fixes iree-org#17344.

After nod-ai/SHARK-TestSuite#418, there are only
two tests running in that test suite, both of which are XFAIL'd due to
programs needing to be regenerated.
  • Loading branch information
ScottTodd authored Jan 6, 2025
1 parent 1445cef commit b245e6b
Show file tree
Hide file tree
Showing 6 changed files with 0 additions and 215 deletions.
98 changes: 0 additions & 98 deletions .github/workflows/pkgci_regression_test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,104 +18,6 @@ on:
default: ""

jobs:
test_models:
name: "test_models :: ${{ matrix.name }}"
runs-on: ${{ matrix.runs-on }}
strategy:
fail-fast: false

# Note: these jobs should use persistent runners with local caches.
# Downloading test files (50GB+) without a cache can take 20+ minutes.
matrix:
include:
# CPU
- name: cpu_llvm_task
models-config-file: models_cpu_llvm_task.json
runs-on:
- self-hosted # must come first
- persistent-cache
- Linux
- X64

# AMD GPU
- name: amdgpu_rocm_mi250_gfx90a
models-config-file: models_gpu_rocm_gfx90a.json
runs-on: nodai-amdgpu-mi250-x86-64
- name: amdgpu_rocm_mi300_gfx942
models-config-file: models_gpu_rocm_gfx942.json
runs-on: nodai-amdgpu-mi300-x86-64
- name: amdgpu_vulkan
models-config-file: models_gpu_vulkan.json
runs-on: nodai-amdgpu-w7900-x86-64

# NVIDIA GPU
# None at the moment. Could maybe use the persistent a100 runners:
# - self-hosted # must come first
# - runner-group=${{ needs.setup.outputs.runner-group }}
# - environment=${{ needs.setup.outputs.runner-env }}
# - a100
# - os-family=Linux
# (note: would need to plumb the presubmit/postsubmit runner-group through to here too)
env:
PACKAGE_DOWNLOAD_DIR: ${{ github.workspace }}/.packages
IREE_TEST_PATH_EXTENSION: ${{ github.workspace }}/build_tools/pkgci/external_test_suite
MODELS_CONFIG_FILE_PATH: build_tools/pkgci/external_test_suite/${{ matrix.models-config-file }}
VENV_DIR: ${{ github.workspace }}/venv
steps:
- name: Checking out IREE repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
submodules: false
- uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
# Must match the subset of versions built in pkgci_build_packages.
python-version: "3.11"
- uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
with:
name: linux_x86_64_release_packages
path: ${{ env.PACKAGE_DOWNLOAD_DIR }}
- name: Setup venv
run: |
./build_tools/pkgci/setup_venv.py ${VENV_DIR} \
--artifact-path=${PACKAGE_DOWNLOAD_DIR} \
--fetch-gh-workflow=${{ inputs.artifact_run_id }}
# Out of tree tests
- name: Check out external TestSuite repository
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
repository: nod-ai/SHARK-TestSuite
ref: 601db0e472600a94ddb69b37d05cd7d4a17f89b2
path: SHARK-TestSuite
submodules: false
lfs: true
- name: Install external TestSuite Python requirements
run: |
source ${VENV_DIR}/bin/activate
python3 -m pip install -r SHARK-TestSuite/iree_tests/requirements.txt
pip install --no-compile --pre --upgrade -e SHARK-TestSuite/common_tools
- name: Download remote files for real weight model tests
run: |
source ${VENV_DIR}/bin/activate
python SHARK-TestSuite/iree_tests/download_remote_files.py --root-dir iree_tests/pytorch/models
python SHARK-TestSuite/iree_tests/download_remote_files.py --root-dir iree_tests/sharktank
- name: Run external tests - models with real weights
if: "matrix.models-config-file != '' && !cancelled()"
run: |
source ${VENV_DIR}/bin/activate
pytest \
SHARK-TestSuite/iree_tests/pytorch/models \
SHARK-TestSuite/iree_tests/sharktank \
-rA \
-k real_weights \
--no-skip-tests-missing-files \
--capture=no \
--log-cli-level=info \
--timeout=600 \
--durations=0 \
--config-files=${MODELS_CONFIG_FILE_PATH}
test_regression_suite:
name: "test_regression_suite :: ${{ matrix.name }}"
runs-on: ${{ matrix.runs-on }}
Expand Down
15 changes: 0 additions & 15 deletions build_tools/pkgci/external_test_suite/README.md

This file was deleted.

26 changes: 0 additions & 26 deletions build_tools/pkgci/external_test_suite/models_cpu_llvm_task.json

This file was deleted.

24 changes: 0 additions & 24 deletions build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx90a.json

This file was deleted.

25 changes: 0 additions & 25 deletions build_tools/pkgci/external_test_suite/models_gpu_rocm_gfx942.json

This file was deleted.

27 changes: 0 additions & 27 deletions build_tools/pkgci/external_test_suite/models_gpu_vulkan.json

This file was deleted.

0 comments on commit b245e6b

Please sign in to comment.