pytest.yaml

name: Run tests

on:
  push:
    paths:
      - ".github/workflows/pytest.yaml"
      - "datapipe/**"
      - "tests/**"
      - "pyproject.toml"


jobs:
  lint:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          # - "3.8"
          - "3.9"
          - "3.10"

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      
      - name: Install dependencies
        run: |
          pip install ".[sqlite,excel,milvus,gcsfs,s3fs,redis,qdrant,gcp]" "flake8" "mypy"
          pip install pandas-stubs pytest-cases types-setuptools types-redis types-tqdm types-pillow types-PyYAML

      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics

      - name: Lint with mypy
        run: |
          mypy -p datapipe --ignore-missing-imports --follow-imports=silent --namespace-packages

  test:
    # needs: lint

    runs-on: ubuntu-latest
    strategy:
      matrix:
        include:
          # - python-version: "3.8"
          #   test-db-env: "postgres"
          #   pip-extra: "sqlalchemy <2"
          # - python-version: "3.8"
          #   test-db-env: "postgres"
          #   pip-extra: "'sqlalchemy>2'"
          # - python-version: "3.8"
          #   test-db-env: "sqlite"

          - python-version: "3.9"
            test-db-env: "postgres"
            pip-extra: '"sqlalchemy>2"'
          # - python-version: "3.9"
          #   test-db-env: "sqlite"

          # - python-version: "3.10"
          #   test-db-env: "postgres"
          # - python-version: "3.10"
          #   test-db-env: "sqlite"

          - python-version: "3.11"
            test-db-env: "postgres"
            pip-extra: '"sqlalchemy<2" "pandas<2.2"'
          - python-version: "3.11"
            test-db-env: "postgres"
            pip-extra: '"sqlalchemy>2"'
          - python-version: "3.11"
            test-db-env: "sqlite"
            pip-extra: '"sqlalchemy>2"'

          - python-version: "3.12"
            test-db-env: "postgres"
          - python-version: "3.12"
            test-db-env: "sqlite"

    services:
      # Label used to access the service container
      postgres:
        # Docker Hub image
        image: postgres:12
        # Provide the password for postgres
        env:
          POSTGRES_PASSWORD: password
        # Set health checks to wait until postgres has started
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 5432:5432
      redis:
        image: redis
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          - 6379:6379
      quadrant:
        image: qdrant/qdrant:latest
        ports:
          - 6333:6333

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      
      - name: Install dependencies
        run: |
          pip install ${{ matrix.pip-extra }} ".[sqlite,excel,milvus,gcsfs,s3fs,redis,qdrant,gcp]" "pytest<8" "pytest_cases"

      - name: Test with pytest
        run: |
          pytest
        env:
          TEST_DB_ENV: ${{ matrix.test-db-env }}
          # The hostname used to communicate with the PostgreSQL service container
          POSTGRES_HOST: localhost
          # The default PostgreSQL port
          POSTGRES_PORT: 5432
          REDIS_HOST: localhost
          REDIS_PORT: 6379