From 1d2e351fd076c92a4ce12dae2eea9be0f7797a27 Mon Sep 17 00:00:00 2001 From: Chenxiong Qi Date: Sun, 7 Jul 2024 23:42:52 +0800 Subject: [PATCH] ci: fix tests with real databases tox-docker is removed. The CI process running in GitHub actions depends on database services. The local tests with real databases runs by the following workflow: * Run: make start-testdb-{mysql,mariadb,pgsql} * Run tests with tox, e.g. tox -e py311-django420-mariadb Every database container is initialized with most default values except the the password is set to "pass". Signed-off-by: Chenxiong Qi --- .github/workflows/unittests.yaml | 45 ++++++++++++++- Makefile | 41 ++++++++++++++ contrib/scripts/adjust_mysql_auth_plugin.sh | 18 +++--- pyproject.toml | 3 +- tests/core/test_db.py | 4 +- tox.ini | 63 ++++----------------- 6 files changed, 107 insertions(+), 67 deletions(-) diff --git a/.github/workflows/unittests.yaml b/.github/workflows/unittests.yaml index f0fa3d67..e04a777c 100644 --- a/.github/workflows/unittests.yaml +++ b/.github/workflows/unittests.yaml @@ -12,7 +12,48 @@ on: jobs: unittests: name: "py${{ matrix.python-version }}-${{ matrix.django-version }}-${{ matrix.db }}" - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + + services: + mariadb: + image: mariadb:10.4.14 + ports: + - 33061:3306 + env: + MYSQL_DATABASE: nitrate + MYSQL_ROOT_PASSWORD: pass + options: >- + --name=testdb-mariadb + --health-cmd="mysqladmin ping -uroot -ppass" + --health-interval=5s + --health-timeout=2s + --health-retries=3 + postgres: + image: postgres:12.4 + ports: + - 54321:5432 + env: + POSTGRES_PASSWORD: pass + options: >- + --name=testdb-pgsql + --health-cmd="echo pass | psql -U postgres -W --command 'SELECT 1'" + --health-interval=5s + --health-timeout=2s + --health-retries=3 + mysql: + image: mysql:8.0.22 + ports: + - 33062:3306 + env: + MYSQL_DATABASE: nitrate + MYSQL_ROOT_PASSWORD: pass + options: >- + --name=testdb-mysql + --health-cmd="mysqladmin ping -uroot -ppass" + --health-interval=5s + --health-timeout=2s + --health-retries=3 + strategy: max-parallel: 7 matrix: @@ -47,7 +88,7 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - python -m pip install tox "tox-docker>=2.0.0" coveralls + python -m pip install tox coveralls - run: | echo "tox_pyver=${pyver}" | sed "s/\.//" >> $GITHUB_ENV diff --git a/Makefile b/Makefile index 91af6af9..8b1aa2ea 100644 --- a/Makefile +++ b/Makefile @@ -37,3 +37,44 @@ db_envs: # Print environment variables for a specific database engine set by DB .PHONY: format-code format-code: # Format Python code with black. @black --line-length $(shell grep "^max_line_length" tox.ini | cut -d' ' -f3) src/tcms tests + +detach_testdb_container ?= false +gh_wf_unittests = .github/workflows/unittests.yaml +gh_wf_services = .jobs.unittests.services + +# MariaDB setup for testenv + +.PHONY: start-testdb-mariadb +start-testdb-mariadb: + podman run --rm -p 33061:3306 -e MYSQL_ROOT_PASSWORD=pass \ + $(shell yq '$(gh_wf_services).mariadb.options' $(gh_wf_unittests)) \ + $(shell yq '$(gh_wf_services).mariadb.image' $(gh_wf_unittests)) + +.PHONY: stop-testdb-mariadb +stop-testdb-mariadb: + podman stop testdb-mariadb || : + +# MySQL setup for testenv + +.PHONY: start-testdb-mysql +start-testdb-mysql: + podman run --rm -p 33062:3306 -e MYSQL_ROOT_PASSWORD=pass \ + $(shell yq '$(gh_wf_services).mysql.options' $(gh_wf_unittests)) \ + $(shell yq '$(gh_wf_services).mysql.image' $(gh_wf_unittests)) + +.PHONY: stop-testdb-mysql +stop-testdb-mysql: + podman stop testdb-mysql || : + +# PostgreSQL setup for testenv + +.PHONY: start-testdb-pgsql +start-testdb-pgsql: + podman run --name testdb-pgsql --rm -p 54321:5432 \ + -e POSTGRES_PASSWORD=pass \ + $(shell yq '$(gh_wf_services).postgres.options' $(gh_wf_unittests)) \ + $(shell yq '$(gh_wf_services).postgres.image' $(gh_wf_unittests)) + +.PHONY: stop-testdb-pgsql +stop-testdb-pgsql: + podman stop testdb-pgsql || : diff --git a/contrib/scripts/adjust_mysql_auth_plugin.sh b/contrib/scripts/adjust_mysql_auth_plugin.sh index 2b9cd5b2..7f85b313 100644 --- a/contrib/scripts/adjust_mysql_auth_plugin.sh +++ b/contrib/scripts/adjust_mysql_auth_plugin.sh @@ -1,16 +1,14 @@ set -ex + username=$1 password=$2 -tempfile=$(mktemp) -# Guess container name of the test database -if ! docker ps -a --format "{{ json .Names }}" | tr -d '"' | grep "^testdb_mysql" >"$tempfile" 2>&1; then - echo "error: cannot find container testdb_mysql" - exit 1 +CONTAINER_NAME=testdb-mysql + +if ! docker inspect "$CONTAINER_NAME" >/dev/null 2>&1; then + echo "error: cannot find container $CONTAINER_NAME" >&2 + exit 1 fi -c_name=$(cat "$tempfile") -docker exec "$c_name" /bin/bash -c " -mysql -u${username} -p${password} \ --e \"ALTER USER '${username}'@'%' IDENTIFIED WITH mysql_native_password BY '${password}'\" -" +ALTER_USER_CMD="ALTER USER '${username}'@'%' IDENTIFIED WITH mysql_native_password BY '${password}'" +docker exec "$CONTAINER_NAME" /bin/bash -c "mysql -u${username} -p${password} -e \"${ALTER_USER_CMD}\"" diff --git a/pyproject.toml b/pyproject.toml index 1b4948ce..06c57003 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,7 +42,7 @@ dynamic = ["version"] [project.optional-dependencies] mysql = ["mysqlclient==2.1.1"] -pgsql = ["psycopg2-binary==2.9.5"] +pgsql = ["psycopg2==2.9.9"] krbauth = ["kerberos==1.3.1"] bugzilla = ["python-bugzilla==3.2.0"] socialauth = ["social-auth-app-django==5.0.0"] @@ -65,7 +65,6 @@ tests = [ "pytest-django", "sqlparse", "tox", - "tox-docker>=2.0.0", ] [project.urls] diff --git a/tests/core/test_db.py b/tests/core/test_db.py index 78909c03..6f3a7e77 100644 --- a/tests/core/test_db.py +++ b/tests/core/test_db.py @@ -13,14 +13,14 @@ [ [ True, - "SELECT summary FROM test_cases", + "SELECT summary FROM test_cases ORDER BY summary", None, [{"summary": "case 1"}, {"summary": "case 2"}], "case 1", ], [ False, - "SELECT summary, script FROM test_cases", + "SELECT summary, script FROM test_cases ORDER BY summary", None, [("case 1", "echo hello"), ("case 2", "find /tmp")], "case 1", diff --git a/tox.ini b/tox.ini index 2ec87fed..22663e34 100644 --- a/tox.ini +++ b/tox.ini @@ -25,85 +25,46 @@ extras = tests commands = py.test {posargs:tests/} -[docker:testdb_mysql] -image = mysql:8.0.22 -ports = - 33061:3306/tcp -environment = - MYSQL_ROOT_PASSWORD=pass - MYSQL_DATABASE=nitrate - -[docker:testdb_mariadb] -image = mariadb:10.4.14 -ports = - 33061:3306/tcp -environment = - MYSQL_ROOT_PASSWORD=pass - MYSQL_DATABASE=nitrate - -[testenv:{py39,py310}-django{420}-mysql] -docker = - testdb_mysql +[testenv:py311-django{420}-mysql] +description = Run tests with a real MySQL database instance. Start the container by running "make start-testdb-mysql". allowlist_externals = bash - sleep extras = {[testenv]extras} mysql setenv = NITRATE_DB_ENGINE = mysql - NITRATE_DB_NAME = nitrate - NITRATE_DB_PORT = 33061 + NITRATE_DB_HOST = 127.0.0.1 + NITRATE_DB_PORT = 33062 NITRATE_DB_USER = root NITRATE_DB_PASSWORD = pass commands_pre = - sleep 25 bash {toxinidir}/contrib/scripts/adjust_mysql_auth_plugin.sh root pass -[testenv:{py39,py310}-django{420}-mariadb] -docker = - testdb_mariadb -allowlist_externals = - bash - sleep +[testenv:py311-django{420}-mariadb] +description = Run tests with a real MariaDB database instance. Start the container by running "make start-testdb-mariadb". extras = {[testenv]extras} mysql setenv = NITRATE_DB_ENGINE = mysql - NITRATE_DB_NAME = nitrate + NITRATE_DB_HOST = 127.0.0.1 NITRATE_DB_PORT = 33061 NITRATE_DB_USER = root NITRATE_DB_PASSWORD = pass -commands_pre = - sleep 5 - -[docker:testdb_postgres] -image = postgres:12.4-alpine -ports = - 54321:5432/tcp -environment = - POSTGRES_USER=postgres - POSTGRES_PASSWORD=nitrate - POSTGRES_DB=testdb -[testenv:{py39,py310}-django{420}-postgres] -docker = - testdb_postgres -allowlist_externals = - sleep +[testenv:py311-django{420}-postgres] +description = Run tests with a real PostgreSQL database instance. Start the container by running "make start-testdb-pgsql". extras = {[testenv]extras} pgsql setenv = NITRATE_DB_ENGINE = pgsql + NITRATE_DB_HOST = 127.0.0.1 NITRATE_DB_PORT = 54321 NITRATE_DB_USER = postgres - NITRATE_DB_PASSWORD = nitrate - NITRATE_DB_NAME = nitrate -commands_pre = -; Sleeping 2 is enough to for the container is started completely - sleep 5 + NITRATE_DB_PASSWORD = pass + NITRATE_DB_NAME = postgres [testenv:flake8] basepython = python3