From ed5c0b56531528b33df11eb16695d3f07095cae9 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Thu, 18 Jul 2024 23:44:57 +0800 Subject: [PATCH] ci: use self-start test database containers Signed-off-by: Chenxiong Qi --- .github/workflows/unittests.yaml | 69 ++++++++---------------------- Makefile | 73 ++++++++++++++++++++++++-------- renovate.json | 12 ++++++ 3 files changed, 85 insertions(+), 69 deletions(-) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index aacb80b4..dfb4c38c 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -14,46 +14,6 @@ jobs: name: "py${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.db }}" runs-on: ubuntu-24.04 - services: - mariadb: - image: mariadb:10.11.8@sha256:75f6e61397758489d1dccf95db33b6b49ebfc7ec1253d40060fdf8ceb7f938a3 - ports: - - 33061:3306 - env: - MYSQL_DATABASE: nitrate - MYSQL_ROOT_PASSWORD: pass - options: >- - --name=testdb-mariadb - --health-cmd="mysqladmin ping -uroot -ppass" - --health-interval=5s - --health-timeout=2s - --health-retries=3 - postgres: - image: postgres:16.3@sha256:0aafd2ae7e6c391f39fb6b7621632d79f54068faebc726caf469e87bd1d301c0 - ports: - - 54321:5432 - env: - POSTGRES_PASSWORD: pass - options: >- - --name=testdb-pgsql - --health-cmd="echo pass | psql -U postgres -W --command 'SELECT 1'" - --health-interval=5s - --health-timeout=2s - --health-retries=3 - mysql: - image: mysql:8.0.22@sha256:0fd2898dc1c946b34dceaccc3b80d38b1049285c1dab70df7480de62265d6213 - ports: - - 33062:3306 - env: - MYSQL_DATABASE: nitrate - MYSQL_ROOT_PASSWORD: pass - options: >- - --name=testdb-mysql - --health-cmd="mysqladmin ping -uroot -ppass" - --health-interval=5s - --health-timeout=2s - --health-retries=3 - strategy: max-parallel: 7 matrix: @@ -75,7 +35,7 @@ jobs: - name: Install packages used to setup testenv run: | sudo apt update - sudo apt-get install gcc python3-dev libkrb5-dev libmariadb-dev libcrypt-dev + sudo apt-get install -y jq make gcc python3-dev libkrb5-dev libmariadb-dev libcrypt-dev - name: Checkout code uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 @@ -85,19 +45,26 @@ jobs: with: python-version: ${{ matrix.python-version }} - - name: Install dependencies + - name: Run the tests + env: + DB: ${{ matrix.db }} + PYTHON_VERSION: ${{ matrix.python-version }} run: | - python -m pip install --upgrade pip - python -m pip install tox coveralls + set -eu + set -o pipefail - - run: | - echo "tox_pyver=${pyver}" | sed "s/\.//" >> $GITHUB_ENV - env: - pyver: ${{ matrix.python-version }} + python3 -m pip install --upgrade pip + python3 -m pip install tox coveralls - - name: Tox - run: | - tox -e py$tox_pyver-${{ matrix.django-version }}-${{ matrix.db }} + python3 -m pip --version + tox -l + + if [ "x$DB" != "xsqlite" ]; then + make start-db-${DB} detach=yes + trap 'make stop-db-${DB}' EXIT ERR + make check-testdb-health DB=${DB/postgres/pgsql} + fi + tox -e py${PYTHON_VERSION/./}-${PYTHON_VERSION}-${DB} - name: Parallel coverage report uses: coverallsapp/github-action@3dfc5567390f6fa9267c0ee9c251e4c8c3f18949 # v2 diff --git a/Makefile b/Makefile index 8b1aa2ea..91e1699c 100644 --- a/Makefile +++ b/Makefile @@ -38,17 +38,29 @@ db_envs: # Print environment variables for a specific database engine set by DB format-code: # Format Python code with black. @black --line-length $(shell grep "^max_line_length" tox.ini | cut -d' ' -f3) src/tcms tests -detach_testdb_container ?= false -gh_wf_unittests = .github/workflows/unittests.yaml -gh_wf_services = .jobs.unittests.services +ifeq ($(strip $(detach)),yes) +detach_opt=-d +else +detach_opt= +endif + +common_health_options = --health-interval=5s --health-timeout=2s --health-retries=3 + +mariadb_image = mariadb:10.11.8@sha256:75f6e61397758489d1dccf95db33b6b49ebfc7ec1253d40060fdf8ceb7f938a3 +mysql_image = mysql:8.0.22@sha256:0fd2898dc1c946b34dceaccc3b80d38b1049285c1dab70df7480de62265d6213 +postgres_image = postgres:16.3@sha256:0aafd2ae7e6c391f39fb6b7621632d79f54068faebc726caf469e87bd1d301c0 # MariaDB setup for testenv .PHONY: start-testdb-mariadb start-testdb-mariadb: - podman run --rm -p 33061:3306 -e MYSQL_ROOT_PASSWORD=pass \ - $(shell yq '$(gh_wf_services).mariadb.options' $(gh_wf_unittests)) \ - $(shell yq '$(gh_wf_services).mariadb.image' $(gh_wf_unittests)) + podman run --rm $(detach_opt) \ + --name=testdb-mariadb \ + -p 33061:3306 \ + -e MYSQL_ROOT_PASSWORD=pass \ + --health-cmd="mysqladmin ping -uroot -ppass" \ + $(common_health_options) \ + $(mariadb_image) .PHONY: stop-testdb-mariadb stop-testdb-mariadb: @@ -58,9 +70,13 @@ stop-testdb-mariadb: .PHONY: start-testdb-mysql start-testdb-mysql: - podman run --rm -p 33062:3306 -e MYSQL_ROOT_PASSWORD=pass \ - $(shell yq '$(gh_wf_services).mysql.options' $(gh_wf_unittests)) \ - $(shell yq '$(gh_wf_services).mysql.image' $(gh_wf_unittests)) + podman run --rm $(detach_opt) \ + --name=testdb-mysql \ + -p 33062:3306 \ + -e MYSQL_ROOT_PASSWORD=pass \ + --health-cmd="mysqladmin ping -uroot -ppass" \ + $(common_health_options) \ + $(mysql_image) .PHONY: stop-testdb-mysql stop-testdb-mysql: @@ -68,13 +84,34 @@ stop-testdb-mysql: # PostgreSQL setup for testenv -.PHONY: start-testdb-pgsql -start-testdb-pgsql: - podman run --name testdb-pgsql --rm -p 54321:5432 \ +.PHONY: start-testdb-postgres +start-testdb-postgres: + podman run --rm $(detach_opt) \ + --name=testdb-postgres \ + -p 54321:5432 \ -e POSTGRES_PASSWORD=pass \ - $(shell yq '$(gh_wf_services).postgres.options' $(gh_wf_unittests)) \ - $(shell yq '$(gh_wf_services).postgres.image' $(gh_wf_unittests)) - -.PHONY: stop-testdb-pgsql -stop-testdb-pgsql: - podman stop testdb-pgsql || : + --health-cmd="PGPASSWORD=pass psql -h 127.0.0.1 -U postgres -c 'SELECT 1'" \ + $(common_health_options) \ + $(postgres_image) + +.PHONY: stop-testdb-postgres +stop-testdb-postgres: + podman stop testdb-postgres || : + +.PHONY: check-testdb-health +check-testdb-health: + @test_db_engine=$(DB); \ + for i in {1..5}; do \ + health_status=$$(podman inspect testdb-$${DB/pgsql/postgres} | jq -r '.[].State.Health.Status'); \ + [ "x$$health_status" == "xhealthy" ] && break; \ + if [ $$i -eq 5 ]; then \ + echo "testdb $${DB/pgsql/postgres} container is not healthy. Seems failed to start." >&2; \ + echo "container inspect:" >&2; \ + podman inspect testdb-$${DB/pgsql/postgres} | jq '.[].State' >&2; \ + exit 1; \ + else \ + echo "Sleep 2s then have another health check" >&2; \ + sleep 2s; \ + fi; \ + done; \ + echo "Test database $${DB/pgsql/postgres} is healthy." diff --git a/renovate.json b/renovate.json index ef700c49..8bd42ec1 100644 --- a/renovate.json +++ b/renovate.json @@ -3,9 +3,21 @@ "extends": [ "config:best-practices" ], + "docker-compose": { "fileMatch": ["^container-compose\\.ya?ml$"] }, + "customManagers": [ + { + "customType": "regex", + "fileMatch": ["^Makefile$"], + "matchStrings": [ + "(mariadb|mysql|postgres)_image = (?mariadb|mysql|postgres?):(?[.0-9]+?)@sha256:[0-9a-f]{64}\\n" + ], + "datasourceTemplate": "docker" + } + ], + "packageRules": [ { "matchPackageNames": ["mariadb"],