From eed0d1fdabf6370e91f0b3ee2cf7c73a91c82415 Mon Sep 17 00:00:00 2001 From: David Poltorak Date: Mon, 9 Oct 2023 13:07:46 +0100 Subject: [PATCH] feat: docker-compose for testing a single-tenant full stack Signed-off-by: David Poltorak --- .../single-tenant-testing-stack/.env | 3 + .../apisix/conf/apisix.yaml | 60 ++++ .../apisix/conf/config.yaml | 27 ++ .../docker-compose.yml | 304 ++++++++++++++++++ .../postgres/init-script.sh | 24 ++ .../postgres/max_conns.sql | 1 + .../run-e2e-tests-local.sh | 32 ++ .../run-performance-tests-local.sh | 43 +++ 8 files changed, 494 insertions(+) create mode 100644 infrastructure/single-tenant-testing-stack/.env create mode 100644 infrastructure/single-tenant-testing-stack/apisix/conf/apisix.yaml create mode 100644 infrastructure/single-tenant-testing-stack/apisix/conf/config.yaml create mode 100644 infrastructure/single-tenant-testing-stack/docker-compose.yml create mode 100755 infrastructure/single-tenant-testing-stack/postgres/init-script.sh create mode 100644 infrastructure/single-tenant-testing-stack/postgres/max_conns.sql create mode 100755 infrastructure/single-tenant-testing-stack/run-e2e-tests-local.sh create mode 100755 infrastructure/single-tenant-testing-stack/run-performance-tests-local.sh diff --git a/infrastructure/single-tenant-testing-stack/.env b/infrastructure/single-tenant-testing-stack/.env new file mode 100644 index 0000000000..00710460b2 --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/.env @@ -0,0 +1,3 @@ +PRISM_AGENT_VERSION=1.16.0 +PRISM_NODE_VERSION=2.2.1 +VAULT_DEV_ROOT_TOKEN_ID=root diff --git a/infrastructure/single-tenant-testing-stack/apisix/conf/apisix.yaml b/infrastructure/single-tenant-testing-stack/apisix/conf/apisix.yaml new file mode 100644 index 0000000000..fd7ea3c557 --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/apisix/conf/apisix.yaml @@ -0,0 +1,60 @@ +plugins: + - name: proxy-rewrite + +routes: + - uri: /issuer/prism-agent/* + upstream_id: 1 + plugins: + proxy-rewrite: + regex_uri: ["^/issuer/prism-agent/(.*)", "/$1"] + - uri: /issuer/didcomm* + upstream_id: 2 + plugins: + proxy-rewrite: + regex_uri: ["^/issuer/didcomm(.*)", "/$1"] + - uri: /verifier/prism-agent/* + upstream_id: 3 + plugins: + proxy-rewrite: + regex_uri: ["^/verifier/prism-agent/(.*)", "/$1"] + - uri: /verifier/didcomm* + upstream_id: 4 + plugins: + proxy-rewrite: + regex_uri: ["^/verifier/didcomm(.*)", "/$1"] + - uri: /holder/prism-agent/* + upstream_id: 5 + plugins: + proxy-rewrite: + regex_uri: ["^/holder/prism-agent/(.*)", "/$1"] + - uri: /holder/didcomm* + upstream_id: 6 + plugins: + proxy-rewrite: + regex_uri: ["^/holder/didcomm(.*)", "/$1"] +upstreams: + - id: 1 + nodes: + "issuer-oea:8085": 1 # tapir + type: roundrobin + - id: 2 + nodes: + "issuer-oea:8090": 1 # didcom + type: roundrobin + - id: 3 + nodes: + "verifier-oea:8085": 1 # tapir + type: roundrobin + - id: 4 + nodes: + "verifier-oea:8090": 1 # didcom + type: roundrobin + - id: 5 + nodes: + "holder-oea:8085": 1 # tapir + type: roundrobin + - id: 6 + nodes: + "holder-oea:8090": 1 # didcom + type: roundrobin +#END diff --git a/infrastructure/single-tenant-testing-stack/apisix/conf/config.yaml b/infrastructure/single-tenant-testing-stack/apisix/conf/config.yaml new file mode 100644 index 0000000000..7385594d84 --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/apisix/conf/config.yaml @@ -0,0 +1,27 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apisix: + node_listen: 9080 # APISIX listening port + enable_ipv6: false + enable_admin: false + config_center: yaml + +deployment: + role: data_plane + role_data_plane: + config_provider: yaml diff --git a/infrastructure/single-tenant-testing-stack/docker-compose.yml b/infrastructure/single-tenant-testing-stack/docker-compose.yml new file mode 100644 index 0000000000..b5ca092a70 --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/docker-compose.yml @@ -0,0 +1,304 @@ +--- +version: "3.8" + +services: + issuer-db: + image: postgres:13 + environment: + POSTGRES_MULTIPLE_DATABASES: "castor,pollux,connect,agent" + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - issuer_pg_data_db:/var/lib/postgresql/data + - ./postgres/init-script.sh:/docker-entrypoint-initdb.d/init-script.sh + - ./postgres/max_conns.sql:/docker-entrypoint-initdb.d/max_conns.sql + ports: + - 5432:5432 + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres", "-d", "castor"] + interval: 10s + timeout: 5s + retries: 5 + + verifier-db: + image: postgres:13 + environment: + POSTGRES_MULTIPLE_DATABASES: "castor,pollux,connect,agent" + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - verifier_pg_data_db:/var/lib/postgresql/data + - ./postgres/init-script.sh:/docker-entrypoint-initdb.d/init-script.sh + - ./postgres/max_conns.sql:/docker-entrypoint-initdb.d/max_conns.sql + ports: + - 5433:5432 + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres", "-d", "castor"] + interval: 10s + timeout: 5s + retries: 5 + + holder-db: + image: postgres:13 + environment: + POSTGRES_MULTIPLE_DATABASES: "castor,pollux,connect,agent" + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - holder_pg_data_db:/var/lib/postgresql/data + - ./postgres/init-script.sh:/docker-entrypoint-initdb.d/init-script.sh + - ./postgres/max_conns.sql:/docker-entrypoint-initdb.d/max_conns.sql + ports: + - 5434:5432 + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres", "-d", "castor"] + interval: 10s + timeout: 5s + retries: 5 + + node-db: + image: postgres:13 + environment: + POSTGRES_MULTIPLE_DATABASES: "node_db" + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + volumes: + - node_pg_data_db:/var/lib/postgresql/data + - ./postgres/init-script.sh:/docker-entrypoint-initdb.d/init-script.sh + - ./postgres/max_conns.sql:/docker-entrypoint-initdb.d/max_conns.sql + healthcheck: + test: ["CMD", "pg_isready", "-U", "postgres", "-d", "node_db"] + interval: 10s + timeout: 5s + retries: 5 + + prism-node: + image: ghcr.io/input-output-hk/prism-node:${PRISM_NODE_VERSION} + environment: + NODE_PSQL_HOST: node-db:5432 + NODE_LEDGER: in-memory + NODE_REFRESH_AND_SUBMIT_PERIOD: 1s + NODE_MOVE_SCHEDULED_TO_PENDING_PERIOD: 1s + NODE_WALLET_MAX_TPS: 1000 + depends_on: + node-db: + condition: service_healthy + + issuer-oea: + image: ghcr.io/input-output-hk/prism-agent:${PRISM_AGENT_VERSION} + environment: + IRIS_HOST: iris + IRIS_PORT: 8081 + CASTOR_DB_HOST: issuer-db + CASTOR_DB_PORT: 5432 + CASTOR_DB_NAME: castor + CASTOR_DB_USER: postgres + CASTOR_DB_PASSWORD: postgres + POLLUX_DB_HOST: issuer-db + POLLUX_DB_PORT: 5432 + POLLUX_DB_NAME: pollux + POLLUX_DB_USER: postgres + POLLUX_DB_PASSWORD: postgres + CONNECT_DB_HOST: issuer-db + CONNECT_DB_PORT: 5432 + CONNECT_DB_NAME: connect + CONNECT_DB_USER: postgres + CONNECT_DB_PASSWORD: postgres + AGENT_DB_HOST: issuer-db + AGENT_DB_PORT: 5432 + AGENT_DB_NAME: agent + AGENT_DB_USER: postgres + AGENT_DB_PASSWORD: postgres + DIDCOMM_SERVICE_URL: http://host.docker.internal:${PORT}/issuer/didcomm + PRISM_NODE_HOST: prism-node + PRISM_NODE_PORT: 50053 + SECRET_STORAGE_BACKEND: postgres + DEV_MODE: true + DEFAULT_WALLET_ENABLED: + DEFAULT_WALLET_SEED: + # DEFAULT_WALLET_WEBHOOK_URL: + # DEFAULT_WALLET_WEBHOOK_API_KEY: + # DEFAULT_WALLET_AUTH_API_KEY: + # GLOBAL_WEBHOOK_URL: + # GLOBAL_WEBHOOK_API_KEY: + # WEBHOOK_PARALLELISM: + ADMIN_TOKEN: + API_KEY_SALT: + API_KEY_ENABLED: + API_KEY_AUTHENTICATE_AS_DEFAULT_USER: + API_KEY_AUTO_PROVISIONING: + ISSUE_BG_JOB_RECORDS_LIMIT: 25 + ISSUE_BG_JOB_RECURRENCE_DELAY: 2 seconds + ISSUE_BG_JOB_PROCESSING_PARALLELISM: 5 + PRESENTATION_BG_JOB_RECORDS_LIMIT: 25 + PRESENTATION_BG_JOB_RECURRENCE_DELAY: 2 seconds + PRESENTATION_BG_JOB_PROCESSING_PARALLELISM: 5 + CONNECT_BG_JOB_RECORDS_LIMIT: 25 + CONNECT_BG_JOB_RECURRENCE_DELAY: 2 seconds + CONNECT_BG_JOB_PROCESSING_PARALLELISM: 5 + depends_on: + issuer-db: + condition: service_healthy + prism-node: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://issuer-oea:8085/_system/health"] + interval: 30s + timeout: 10s + retries: 5 + extra_hosts: + - "host.docker.internal:host-gateway" + + verifier-oea: + image: ghcr.io/input-output-hk/prism-agent:${PRISM_AGENT_VERSION} + environment: + IRIS_HOST: iris + IRIS_PORT: 8081 + CASTOR_DB_HOST: verifier-db + CASTOR_DB_PORT: 5432 + CASTOR_DB_NAME: castor + CASTOR_DB_USER: postgres + CASTOR_DB_PASSWORD: postgres + POLLUX_DB_HOST: verifier-db + POLLUX_DB_PORT: 5432 + POLLUX_DB_NAME: pollux + POLLUX_DB_USER: postgres + POLLUX_DB_PASSWORD: postgres + CONNECT_DB_HOST: verifier-db + CONNECT_DB_PORT: 5432 + CONNECT_DB_NAME: connect + CONNECT_DB_USER: postgres + CONNECT_DB_PASSWORD: postgres + AGENT_DB_HOST: verifier-db + AGENT_DB_PORT: 5432 + AGENT_DB_NAME: agent + AGENT_DB_USER: postgres + AGENT_DB_PASSWORD: postgres + DIDCOMM_SERVICE_URL: http://host.docker.internal:${PORT}/verifier/didcomm + PRISM_NODE_HOST: prism-node + PRISM_NODE_PORT: 50053 + SECRET_STORAGE_BACKEND: postgres + DEV_MODE: true + DEFAULT_WALLET_ENABLED: + DEFAULT_WALLET_SEED: + # DEFAULT_WALLET_WEBHOOK_URL: + # DEFAULT_WALLET_WEBHOOK_API_KEY: + # DEFAULT_WALLET_AUTH_API_KEY: + # GLOBAL_WEBHOOK_URL: + # GLOBAL_WEBHOOK_API_KEY: + # WEBHOOK_PARALLELISM: + ADMIN_TOKEN: + API_KEY_SALT: + API_KEY_ENABLED: + API_KEY_AUTHENTICATE_AS_DEFAULT_USER: + API_KEY_AUTO_PROVISIONING: + ISSUE_BG_JOB_RECORDS_LIMIT: 25 + ISSUE_BG_JOB_RECURRENCE_DELAY: 2 seconds + ISSUE_BG_JOB_PROCESSING_PARALLELISM: 5 + PRESENTATION_BG_JOB_RECORDS_LIMIT: 25 + PRESENTATION_BG_JOB_RECURRENCE_DELAY: 2 seconds + PRESENTATION_BG_JOB_PROCESSING_PARALLELISM: 5 + CONNECT_BG_JOB_RECORDS_LIMIT: 25 + CONNECT_BG_JOB_RECURRENCE_DELAY: 2 seconds + CONNECT_BG_JOB_PROCESSING_PARALLELISM: 5 + depends_on: + verifier-db: + condition: service_healthy + prism-node: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://verifier-oea:8085/_system/health"] + interval: 30s + timeout: 10s + retries: 5 + extra_hosts: + - "host.docker.internal:host-gateway" + + holder-oea: + image: ghcr.io/input-output-hk/prism-agent:${PRISM_AGENT_VERSION} + environment: + IRIS_HOST: iris + IRIS_PORT: 8081 + CASTOR_DB_HOST: holder-db + CASTOR_DB_PORT: 5432 + CASTOR_DB_NAME: castor + CASTOR_DB_USER: postgres + CASTOR_DB_PASSWORD: postgres + POLLUX_DB_HOST: holder-db + POLLUX_DB_PORT: 5432 + POLLUX_DB_NAME: pollux + POLLUX_DB_USER: postgres + POLLUX_DB_PASSWORD: postgres + CONNECT_DB_HOST: holder-db + CONNECT_DB_PORT: 5432 + CONNECT_DB_NAME: connect + CONNECT_DB_USER: postgres + CONNECT_DB_PASSWORD: postgres + AGENT_DB_HOST: holder-db + AGENT_DB_PORT: 5432 + AGENT_DB_NAME: agent + AGENT_DB_USER: postgres + AGENT_DB_PASSWORD: postgres + DIDCOMM_SERVICE_URL: http://host.docker.internal:${PORT}/holder/didcomm + PRISM_NODE_HOST: prism-node + PRISM_NODE_PORT: 50053 + SECRET_STORAGE_BACKEND: postgres + DEV_MODE: true + DEFAULT_WALLET_ENABLED: + DEFAULT_WALLET_SEED: + # DEFAULT_WALLET_WEBHOOK_URL: + # DEFAULT_WALLET_WEBHOOK_API_KEY: + # DEFAULT_WALLET_AUTH_API_KEY: + # GLOBAL_WEBHOOK_URL: + # GLOBAL_WEBHOOK_API_KEY: + # WEBHOOK_PARALLELISM: + ISSUE_BG_JOB_RECORDS_LIMIT: 25 + ISSUE_BG_JOB_RECURRENCE_DELAY: 2 seconds + ISSUE_BG_JOB_PROCESSING_PARALLELISM: 5 + PRESENTATION_BG_JOB_RECORDS_LIMIT: 25 + PRESENTATION_BG_JOB_RECURRENCE_DELAY: 2 seconds + PRESENTATION_BG_JOB_PROCESSING_PARALLELISM: 5 + CONNECT_BG_JOB_RECORDS_LIMIT: 25 + CONNECT_BG_JOB_RECURRENCE_DELAY: 2 seconds + CONNECT_BG_JOB_PROCESSING_PARALLELISM: 5 + ADMIN_TOKEN: + API_KEY_SALT: + API_KEY_ENABLED: + API_KEY_AUTHENTICATE_AS_DEFAULT_USER: + API_KEY_AUTO_PROVISIONING: + depends_on: + holder-db: + condition: service_healthy + prism-node: + condition: service_started + healthcheck: + test: ["CMD", "curl", "-f", "http://holder-oea:8085/_system/health"] + interval: 30s + timeout: 10s + retries: 5 + extra_hosts: + - "host.docker.internal:host-gateway" + + apisix: + image: apache/apisix:2.15.0-alpine + volumes: + - ./apisix/conf/apisix.yaml:/usr/local/apisix/conf/apisix.yaml:ro + - ./apisix/conf/config.yaml:/usr/local/apisix/conf/config.yaml:ro + ports: + - "${PORT}:9080/tcp" + depends_on: + - issuer-oea + - verifier-oea + - holder-oea + +volumes: + issuer_pg_data_db: + verifier_pg_data_db: + holder_pg_data_db: + node_pg_data_db: + pgadmin: +# Temporary commit network setting due to e2e CI bug +# to be enabled later after debugging +#networks: +# default: +# name: ${NETWORK} diff --git a/infrastructure/single-tenant-testing-stack/postgres/init-script.sh b/infrastructure/single-tenant-testing-stack/postgres/init-script.sh new file mode 100755 index 0000000000..408264cf1e --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/postgres/init-script.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e +set -u + +function create_user_and_database() { + local database=$1 + local app_user=${database}-application-user + echo " Creating user and database '$database'" + psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL + CREATE USER "$app_user" WITH PASSWORD 'password'; + CREATE DATABASE $database; + \c $database + ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO "$app_user"; + EOSQL +} + +if [ -n "$POSTGRES_MULTIPLE_DATABASES" ]; then + echo "Multiple database creation requested: $POSTGRES_MULTIPLE_DATABASES" + for db in $(echo $POSTGRES_MULTIPLE_DATABASES | tr ',' ' '); do + create_user_and_database $db + done + echo "Multiple databases created" +fi diff --git a/infrastructure/single-tenant-testing-stack/postgres/max_conns.sql b/infrastructure/single-tenant-testing-stack/postgres/max_conns.sql new file mode 100644 index 0000000000..f2a343e505 --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/postgres/max_conns.sql @@ -0,0 +1 @@ +ALTER SYSTEM SET max_connections = 500; diff --git a/infrastructure/single-tenant-testing-stack/run-e2e-tests-local.sh b/infrastructure/single-tenant-testing-stack/run-e2e-tests-local.sh new file mode 100755 index 0000000000..98dcd604ee --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/run-e2e-tests-local.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +ENV_FILE="${ENV_FILE:=${SCRIPT_DIR}/.env}" +PORT="${PORT:=9500}" + +echo "ENV_FILE = ${ENV_FILE}" + +echo "--------------------------------------" +echo "Starting stack using docker compose" +echo "--------------------------------------" + +PORT=${PORT} docker compose -f ${SCRIPT_DIR}/docker-compose.yml \ + --env-file ${ENV_FILE} up -d --wait + +export AGENT_AUTH_REQUIRED=true +export ACME_AGENT_URL=http://localhost:${PORT}/issuer/prism-agent +export ACME_AUTH_KEY=default +export BOB_AGENT_URL=http://localhost:${PORT}/verifier/prism-agent +export BOB_AUTH_KEY=default +export MALLORY_AGENT_URL=http://localhost:${PORT}/holder/prism-agent +export MALLORY_AUTH_KEY=default +export FABER_AGENT_URL=http://localhost:${PORT}/holder/prism-agent +export FABER_AUTH_KEY=default + +( + cd ${SCRIPT_DIR}/../../tests/e2e-tests/ + ./gradlew test reports +) diff --git a/infrastructure/single-tenant-testing-stack/run-performance-tests-local.sh b/infrastructure/single-tenant-testing-stack/run-performance-tests-local.sh new file mode 100755 index 0000000000..57166c29e9 --- /dev/null +++ b/infrastructure/single-tenant-testing-stack/run-performance-tests-local.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +set -e + +SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) + +ENV_FILE="${ENV_FILE:=${SCRIPT_DIR}/.env}" +PORT="${PORT:=9500}" + +echo "ENV_FILE = ${ENV_FILE}" + +echo "--------------------------------------" +echo "Starting stack using docker compose" +echo "--------------------------------------" + +PORT=${PORT} docker compose -f ${SCRIPT_DIR}/docker-compose.yml \ + --env-file ${ENV_FILE} up -d --wait + +export ISSUER_AGENT_URL=http://localhost:${PORT}/issuer/prism-agent +export ISSUER_AGENT_API_KEY=default +export HOLDER_AGENT_URL=http://localhost:${PORT}/holder/prism-agent +export HOLDER_AGENT_API_KEY=default +export VERIFIER_AGENT_URL=http://localhost:${PORT}/verifier/prism-agent +export VERIFIER_AGENT_API_KEY=default + +echo "--------------------------------------" +echo "Run perf tests" +echo "--------------------------------------" + +( + export K6_PROMETHEUS_RW_SERVER_URL=http://localhost:9090/api/v1/write + export K6_PROMETHEUS_RW_TREND_AS_NATIVE_HISTOGRAM=true + cd ${SCRIPT_DIR}/../../tests/performance-tests/atala-performance-tests-k6 + yarn install + yarn webpack + k6 run -e SCENARIO_LABEL=create-prism-did-smoke dist/create-prism-did-test.js -o experimental-prometheus-rw + k6 run -e SCENARIO_LABEL=credential-offer-smoke dist/credential-offer-test.js -o experimental-prometheus-rw + k6 run -e SCENARIO_LABEL=credential-schema-smoke dist/credential-schema-test.js -o experimental-prometheus-rw + k6 run -e SCENARIO_LABEL=did-publishing-smoke dist/did-publishing-test.js -o experimental-prometheus-rw + k6 run -e SCENARIO_LABEL=connection-flow-smoke dist/connection-flow-test.js -o experimental-prometheus-rw + k6 run -e SCENARIO_LABEL=issuance-flow-smoke dist/issuance-flow-test.js -o experimental-prometheus-rw + k6 run -e SCENARIO_LABEL=present-proof-flow-smoke dist/present-proof-flow-test.js -o experimental-prometheus-rw +)