tox.ini

[tox]
envlist =
    lint
    grid.test.backend
    grid.test.frontend
    hagrid.publish
    stack.test.course
    stack.test.integration
    stack.test.integration.tls
    stack.test.integration.k8s
    stack.test.integration.smpc
    stack.test.integration.tff
    syft.benchmark
    syft.docs
    syft.jupyter
    syft.publish
    syft.test.fast
    syft.test.fast.coverage
    syft.test.security
    padawan.trials
requires =
    tox-run-command
    pip >= 22.0.4
skipsdist = True


[testenv]
basepython = python3
deps = pip
commands =
    python --version
    python -c "import platform; import os; os.system('pip install jaxlib==0.3.14 -f https://whls.blob.core.windows.net/unstable/index.html') if platform.system().lower() == 'windows' else ''"

[testenv:lint]
; setupdir = {toxinidir}
; changedir = {toxinidir}
description = Linting
allowlist_externals =
    bash
deps =
    black[python2]
    isort
    protoc-wheel-0==3.19.4
    pre-commit
commands =
    pip install protoc-wheel-0==3.19.4 --force
    bash -c "cd {toxinidir}/packages/syft; ./scripts/build_proto.sh"
    black .
    isort .
    pre-commit run --all-files

[testenv:padawan.trials]
description = Padawan Trials
allowlist_externals =
    bash
deps =
    pytest
commands =
    pytest --noconftest packages/syft/tests/trials

# Syft
[testenv:syft]
deps =
    -e{toxinidir}/packages/syft[dev]
changedir = {toxinidir}/packages/syft
description = Syft
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
commands =
    pip list

[testenv:syft.publish]
changedir = {toxinidir}/packages/syft
description = Build and Publish Syft Wheel
commands =
    python -m pip install --upgrade pip
    pip install --upgrade setuptools wheel twine tox build
    python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)'
    python -m build .

[testenv:hagrid.publish]
changedir = {toxinidir}/packages/hagrid
description = Build and Publish Hagrid Wheel
commands =
    python -m pip install --upgrade pip
    pip install --upgrade setuptools wheel twine tox build
    python -c 'from shutil import rmtree; rmtree("build", True); rmtree("dist", True)'
    python -m build .

[testenv:syft.jupyter]
description = Jupyter Notebook with Editable Syft
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
deps =
    {[testenv:syft]deps}
    jupyter
    jupyterlab
commands =
    pip install -e packages/hagrid
    pip install jupyter jupyterlab --upgrade
    jupyter lab --ip 0.0.0.0 --ServerApp.token={posargs}

[testenv:syft.test.fast]
description = Syft Unit Tests Fast
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}/packages/syft
commands =
    pip list
    pytest -m fast -n auto

[testenv:syft.test.fast.coverage]
description = Syft Unit Tests Fast
deps =
    {[testenv:syft]deps}
    pytest-cov
changedir = {toxinidir}/packages/syft
commands =
    pytest --cov=src -m fast -n auto


[testenv:syft.test.security]
description = Security Checks for Syft
changedir = {toxinidir}/packages/syft
deps =
    {[testenv:syft]deps}
commands =
    pip install --upgrade pip
    pip install importlib-metadata==4.13.0
    bandit -r src
    safety check

[testenv:syft.benchmark]
description = Benchmark  for Syft
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
setenv =
    HAGRID_ART = false
    PYTHONIOENCODING = utf-8
commands =
    python --version
    pip install --upgrade pip
    pip install --upgrade pyperf
    pip install -e packages/syft
    pip install -e packages/hagrid
    docker --version
    docker compose version
    bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --tag=local --tail=false --test --health_checks=false'
    docker ps

    bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    sleep 10
    python benchmarks/macro_executor.py
    python benchmarks/executor.py --select_tests phitensor --fast -n 26 -o micro_benchmark.json
    pyperf stats micro_benchmark.json
    pyperf hist micro_benchmark.json

[testenv:syft.docs]
description = Build Docs for Syft
changedir = {toxinidir}/docs
deps = -r {toxinidir}/docs/requirements.txt
allowlist_externals =
    make
    echo
commands =
    python --version
    make html
    echo "Open: {toxinidir}/docs/build/html/index.html"

[testenv:stack.test.integration]
description = Integration Tests for Core Stack
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
    EMULATION = {env:EMULATION:false}
    HAGRID_ART = false
    PYTHONIOENCODING = utf-8
    PYTEST_MODULES = {env:PYTEST_MODULES:frontend network e2e security redis}
commands =
    bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date"

    ; install syft and hagrid
    bash -c 'if [[ "$HAGRID_FLAGS" == *"latest"* ]]; then \
        pip install --pre --force pytest hagrid syft; \
    else \
        pip install -e packages/hagrid -e packages/syft[dev]; \
    fi'

    ; fix windows encoding
    - chcp 65001

    ; check docker versions
    bash -c "docker --version"
    bash -c "docker compose version"

    ; reset volumes and create nodes
    bash -c "echo Starting Nodes; date"
    bash -c "docker rm -f $(docker ps -a -q) || true"
    bash -c "docker volume rm test_domain_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_2_app-db-data --force || true"
    bash -c "docker volume rm test_network_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_2_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_2_app-redis-data --force || true"
    bash -c "docker volume rm test_network_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c "docker volume rm test_domain_2_tailscale-data --force || true"
    bash -c "docker volume rm test_network_1_tailscale-data --force || true"
    bash -c "docker volume rm test_network_1_headscale-data --force || true"
    bash -c 'HAGRID_ART=$HAGRID_ART NETWORK_CHECK_INTERVAL=5 hagrid launch test_network_1 network to docker:9081 --tail=false $HAGRID_FLAGS --health_checks=false'
    bash -c 'HAGRID_ART=$HAGRID_ART DOMAIN_CHECK_INTERVAL=5 hagrid launch test_domain_1 domain to docker:9082 --tail=false $HAGRID_FLAGS --health_checks=false'
    bash -c 'HAGRID_ART=$HAGRID_ART DOMAIN_CHECK_INTERVAL=5 hagrid launch test_domain_2 domain to docker:9083 --headless=true --tail=false $HAGRID_FLAGS --vpn=false --health_checks=false'

    ; wait for nodes to start
    docker ps
    bash -c "echo Waiting for Nodes; date"
    bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q "event - compiled .* successfully\|nginx" || true'
    bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_domain_2-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_network_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'

    ; frontend
    bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
        echo "Starting frontend"; date; \
        pytest tests/integration -m frontend -p no:randomly --co; \
        pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        docker stop test_domain_1-frontend-1 || true; \
        echo "Finished frontend"; date; \
        exit $return; \
    fi'

    ; network
    bash -c 'if [[ "$PYTEST_MODULES" == *"network"* ]]; then \
        echo "Starting network"; date; \
        pytest tests/integration -m network -p no:randomly --co; \
        pytest tests/integration -m network -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished network"; date; \
        exit $return; \
    fi'

    ; e2e
    bash -c 'if [[ "$PYTEST_MODULES" == *"e2e"* ]]; then \
        echo "Starting e2e"; date; \
        pytest tests/integration -m e2e -p no:randomly --co; \
        pytest tests/integration -m e2e -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished e2e"; date; \
        exit $return; \
    fi'

    ; security
    bash -c 'if [[ "$PYTEST_MODULES" == *"security"* ]]; then \
        echo "Starting security"; date; \
        pytest tests/integration -m security -p no:randomly --co; \
        pytest tests/integration -m security -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished security"; date; \
        exit $return; \
    fi'

    ; redis
    bash -c 'if [[ "$PYTEST_MODULES" == *"redis"* ]]; then \
        echo "Starting redis database tests"; date; \
        pytest tests/integration -m redis -p no:randomly --co; \
        pytest tests/integration -m redis -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished Database tests"; date; \
        exit $return; \
    fi'

    # ; shutdown
    bash -c "echo Killing Nodes; date"
    bash -c 'HAGRID_ART=false hagrid land all --force'

[testenv:stack.test.integration.tls]
description = Integration Tests for Core Stack with TLS
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
    mkcert
    mkdir
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
    EMULATION = {env:EMULATION:false}
    HAGRID_ART = false
    PYTHONIOENCODING = utf-8
    PYTEST_MODULES = {env:PYTEST_MODULES:frontend network e2e security redis}
    IGNORE_TLS_ERRORS = True
    CAROOT = {toxinidir}/packages/grid/tls
    CERTS = {toxinidir}/packages/grid/traefik/certs
commands =
    bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date"

    bash -c "mkdir -p ./packages/grid/tls"
    bash -c "mkcert -cert-file={env:CERTS}/cert.pem -key-file={env:CERTS}/key.pem '*.openmined.grid' docker-host localhost 127.0.0.1 ::1"

    ; install syft and hagrid
    bash -c 'if [[ "$HAGRID_FLAGS" == *"latest"* ]]; then \
        pip install --pre --force pytest hagrid syft; \
    else \
        pip install -e packages/hagrid -e packages/syft[dev]; \
    fi'

    ; fix windows encoding
    - chcp 65001

    ; check docker versions
    bash -c "docker --version"
    bash -c "docker compose version"

    ; reset volumes and create nodes
    bash -c "echo Starting Nodes; date"
    bash -c "docker rm -f $(docker ps -a -q) || true"
    bash -c "docker volume rm test_domain_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_2_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_2_seaweedfs-data --force || true"
    bash -c "docker volume rm test_network_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_2_app-redis-data --force || true"
    bash -c "docker volume rm test_network_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c "docker volume rm test_domain_2_tailscale-data --force || true"
    bash -c "docker volume rm test_network_1_tailscale-data --force || true"
    bash -c "docker volume rm test_network_1_headscale-data --force || true"
    bash -c 'HAGRID_ART=$HAGRID_ART NETWORK_CHECK_INTERVAL=5 hagrid launch test_network_1 network to docker:9081 --tail=false $HAGRID_FLAGS --tls --test --cert_store_path=./traefik/certs --health_checks=false'
    bash -c 'HAGRID_ART=$HAGRID_ART DOMAIN_CHECK_INTERVAL=5 hagrid launch test_domain_1 domain to docker:9082 --tail=false $HAGRID_FLAGS --tls --test --cert_store_path=./traefik/certs --health_checks=false'
    bash -c 'HAGRID_ART=$HAGRID_ART DOMAIN_CHECK_INTERVAL=5 hagrid launch test_domain_2 domain to docker:9083 --headless=true --tail=false $HAGRID_FLAGS --vpn=false --tls --test --cert_store_path=./traefik/certs --health_checks=false'

    ; wait for nodes to start
    docker ps
    bash -c "echo Waiting for Nodes; date"
    bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q "event - compiled .* successfully\|nginx" || true'
    bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_domain_2-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_network_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'

    ; frontend
    bash -c 'if [[ "$PYTEST_MODULES" == *"frontend"* ]]; then \
        echo "Starting frontend"; date; \
        pytest tests/integration -m frontend -p no:randomly --co; \
        REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        docker stop test_domain_1-frontend-1 || true; \
        echo "Finished frontend"; date; \
        exit $return; \
    fi'

    ; network
    bash -c 'if [[ "$PYTEST_MODULES" == *"network"* ]]; then \
        echo "Starting network"; date; \
        pytest tests/integration -m network -p no:randomly --co; \
        REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m network -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished network"; date; \
        exit $return; \
    fi'

    ; e2e
    bash -c 'if [[ "$PYTEST_MODULES" == *"e2e"* ]]; then \
        echo "Starting e2e"; date; \
        pytest tests/integration -m e2e -p no:randomly --co; \
        REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m e2e -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished e2e"; date; \
        exit $return; \
    fi'

    ; security
    bash -c 'if [[ "$PYTEST_MODULES" == *"security"* ]]; then \
        echo "Starting security"; date; \
        pytest tests/integration -m security -p no:randomly --co; \
        REQUESTS_CA_BUNDLE={env:CAROOT}/rootCA.pem pytest tests/integration -m security -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished security"; date; \
        exit $return; \
    fi'

    ; redis
    bash -c 'if [[ "$PYTEST_MODULES" == *"redis"* ]]; then \
        echo "Starting redis"; date; \
        pytest tests/integration -m redis -p no:randomly --co; \
        pytest tests/integration -m redis -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished Database tests"; date; \
        exit $return; \
    fi'

    ; shutdown
    bash -c "echo Killing Nodes; date"
    bash -c 'HAGRID_ART=false hagrid land all --force'

[testenv:stack.test.integration.smpc]
description = Integration Tests for Core Stack
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
    EMULATION = {env:EMULATION:false}
    HAGRID_ART = false
    PYTHONIOENCODING = utf-8
    PYTEST_MODULES = {env:PYTEST_MODULES:smpc_np smpc_abstract smpc_share_tensor smpc_mpc_tensor}
commands =
    bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date"

    ; install syft and hagrid
    bash -c 'if [[ "$HAGRID_FLAGS" == *"latest"* ]]; then \
        pip install --pre --force pytest hagrid syft; \
    else \
        pip install -e packages/hagrid -e packages/syft[dev]; \
    fi'

    ; fix windows encoding
    - chcp 65001

    ; check docker versions
    bash -c "docker --version"
    bash -c "docker compose version"

    ; reset volumes and create nodes
    bash -c "echo Starting Nodes; date"
    bash -c "docker rm -f $(docker ps -a -q) || true"
    bash -c "docker volume rm test_domain_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_2_app-db-data --force || true"
    bash -c "docker volume rm test_domain_3_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_2_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_3_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_2_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_3_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c "docker volume rm test_domain_2_tailscale-data --force || true"
    bash -c "docker volume rm test_domain_3_tailscale-data --force || true"
    bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --tag=local --tail=false --headless=true --test --health_checks=false'
    bash -c 'HAGRID_ART=false hagrid launch test_domain_2 domain to docker:9083 --tag=local --tail=false --headless=true --test --health_checks=false'
    bash -c 'HAGRID_ART=false hagrid launch test_domain_3 domain to docker:9084 --tag=local --tail=false --headless=true --test --health_checks=false'

    ; wait for nodes to start
    docker ps
    bash -c "echo Waiting for Nodes; date"
    bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_domain_2-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_domain_3-backend_stream-1 -f &) | grep -q "Application startup complete" || true'

    ; smpc_np
    bash -c 'if [[ "$PYTEST_MODULES" == *"smpc_np"* ]]; then \
        echo "Starting smpc_np"; date; \
        pytest tests/integration -m smpc_np -p no:randomly --co; \
        pytest tests/integration -m smpc_np -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished smpc_np"; date; \
        exit $return; \
    fi'

    ; smpc_abstract
    bash -c 'if [[ "$PYTEST_MODULES" == *"smpc_abstract"* ]]; then \
        echo "Starting smpc_abstract"; date; \
        pytest tests/integration -m smpc_abstract -p no:randomly --co; \
        pytest tests/integration -m smpc_abstract -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Starting smpc_abstract"; date; \
        exit $return; \
    fi'

    ; smpc_share_tensor
    bash -c 'if [[ "$PYTEST_MODULES" == *"smpc_share_tensor"* ]]; then \
        echo "Starting smpc_share_tensor"; date; \
        pytest tests/integration -m smpc_share_tensor -p no:randomly --co; \
        pytest tests/integration -m smpc_share_tensor -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Starting smpc_share_tensor"; date; \
        exit $return; \
    fi'

    ; smpc_mpc_tensor
    bash -c 'if [[ "$PYTEST_MODULES" == *"smpc_mpc_tensor"* ]]; then \
        echo "Starting smpc_mpc_tensor"; date; \
        pytest tests/integration -m smpc_mpc_tensor -p no:randomly --co; \
        pytest tests/integration -m smpc_mpc_tensor -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Starting smpc_mpc_tensor"; date; \
        exit $return; \
    fi'

    ; shutdown
    bash -c "echo Killing Nodes; date"
    bash -c 'HAGRID_ART=false hagrid land all --force'

[testenv:stack.test.integration.tff]
description = Integration Tests for PySyTFF
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    HAGRID_FLAGS = {env:HAGRID_FLAGS:--tag=local --test}
    EMULATION = {env:EMULATION:false}
    HAGRID_ART = false
    PYTHONIOENCODING = utf-8
    PYTEST_MODULES = {env:PYTEST_MODULES:tff}
commands =
    bash -c "echo Running with HAGRID_FLAGS=$HAGRID_FLAGS EMULATION=$EMULATION PYTEST_MODULES=$PYTEST_MODULES; date"

    ; temporarily use our custom wheel
    pip install ./packages/grid/backend/wheels/tensorflow_federated-0.36.0-py2.py3-none-any.whl

    ; install syft and hagrid
    bash -c 'if [[ "$HAGRID_FLAGS" == *"latest"* ]]; then \
        pip install --pre --force pytest hagrid syft; \
    else \
        pip install -e packages/hagrid -e packages/syft[dev,tff]; \
    fi'

    ; fix windows encoding
    - chcp 65001

    ; check docker versions
    bash -c "docker --version"
    bash -c "docker compose version"

    ; reset volumes and create nodes
    bash -c "echo Starting Nodes; date"
    bash -c "docker rm -f $(docker ps -a -q) || true"
    bash -c "docker volume rm test_domain_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c 'HAGRID_ART=$HAGRID_ART DOMAIN_CHECK_INTERVAL=5 hagrid launch test_domain_1 domain to docker:9081 --tail=false $HAGRID_FLAGS --headless=true --health_checks=false'

    ; wait for nodes to start
    docker ps
    bash -c "echo Waiting for Nodes; date"
    bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'

    ; network
    bash -c 'if [[ "$PYTEST_MODULES" == *"tff"* ]]; then \
        echo "Starting tff"; date; \
        pytest tests/integration -m tff -p no:randomly --co; \
        pytest tests/integration -m tff -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no; \
        return=$?; \
        echo "Finished tff"; date; \
        exit $return; \
    fi'

    ; shutdown
    bash -c "echo Killing Nodes; date"
    bash -c 'HAGRID_ART=false hagrid land all --force'

[testenv:stack.test.integration.k8s]
description = Integration Tests for Core Stack
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
passenv=HOME
allowlist_externals =
    devspace
    kubectl
    grep
    sleep
    bash
    kubectx
    k3d
    echo
setenv =
    PIP_FIND_LINKS=https://whls.blob.core.windows.net/unstable/index.html
    CONTAINER_HOST = kubernetes
commands =
    k3d version

    bash -c "docker rm $(docker ps -aq) --force || true"
    bash -c "k3d cluster delete test-network-1 || true"
    bash -c "k3d cluster delete test-domain-1 || true"
    bash -c "k3d cluster delete test-domain-2 || true"
    bash -c "k3d registry delete k3d-registry.localhost || true"
    bash -c "docker volume rm k3d-test-network-1-images --force || true"
    bash -c "docker volume rm k3d-test-domain-1-images --force || true"
    bash -c "docker volume rm k3d-test-domain-2-images --force || true"

    bash -c 'k3d registry create registry.localhost --port 12345  -v `pwd`/k3d-registry:/var/lib/registry || true'

    bash -c 'NODE_NAME=test-network-1 NODE_PORT=9081 && \
        k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
        k3d cluster start $NODE_NAME'

    bash -c 'NODE_NAME=test-network-1 NODE_PORT=9081 && \
        cd packages/grid && \
        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
        --var DOMAIN_NAME=$NODE_NAME \
        --var NETWORK_CHECK_INTERVAL=5 \
        --var TEST_MODE=1 \
        --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
        build -b'

    bash -c 'NODE_NAME=test-network-1 NODE_PORT=9081 && \
        cd packages/grid && \
        (r=5;while ! \
        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
        --var DOMAIN_NAME=$NODE_NAME \
        --var NETWORK_CHECK_INTERVAL=5 \
        --var TEST_MODE=1 \
        --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
        deploy -b -p network; \
        do ((--r))||exit;echo "retrying" && sleep 20;done)'

    bash -c 'NODE_NAME=test-domain-1 NODE_PORT=9082 && \
        k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
        k3d cluster start $NODE_NAME'

    bash -c 'NODE_NAME=test-domain-1 NODE_PORT=9082 && \
        cd packages/grid && \
        (r=5;while ! \
        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
        --var DOMAIN_NAME=$NODE_NAME \
        --var DOMAIN_CHECK_INTERVAL=5 \
        --var TEST_MODE=1 \
        --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
        deploy -b -p domain; \
        do ((--r))||exit;echo "retrying" && sleep 20;done)'

    bash -c 'NODE_NAME=test-domain-2 NODE_PORT=9083 && \
        k3d cluster create $NODE_NAME -p "$NODE_PORT:80@loadbalancer" --registry-use k3d-registry.localhost || true \
        k3d cluster start $NODE_NAME'

    bash -c 'NODE_NAME=test-domain-2 NODE_PORT=9083 && \
        cd packages/grid && \
        (r=5;while ! \
        devspace --no-warn --kube-context "k3d-$NODE_NAME" --namespace $NODE_NAME \
        --var DOMAIN_NAME=$NODE_NAME \
        --var DOMAIN_CHECK_INTERVAL=5 \
        --var TEST_MODE=1 \
        --var CONTAINER_REGISTRY=k3d-registry.localhost:12345/ \
        deploy -b -p domain; \
        do ((--r))||exit;echo "retrying" && sleep 20;done)'

    sleep 10

    ; wait for front end
    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-1 --namespace test-domain-1
    bash -c '(kubectl logs deployment.apps/frontend --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q "event - compiled .* successfully\|nginx" || true'

    ; wait for everything else to be loaded
    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-network-1 --namespace test-network-1
    bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-network-1 --namespace test-network-1
    bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-network-1 --namespace test-network-1
    bash packages/grid/scripts/wait_for.sh service db --context k3d-test-network-1 --namespace test-network-1
    bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-network-1 --namespace test-network-1
    bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-network-1 --namespace test-network-1
    bash packages/grid/scripts/wait_for.sh service headscale --context k3d-test-network-1 --namespace test-network-1

    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service db --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-domain-1 --namespace test-domain-1
    bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-1 --namespace test-domain-1

    bash packages/grid/scripts/wait_for.sh service frontend --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service proxy --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service queue --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service redis --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service db --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service backend --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service backend-stream --context k3d-test-domain-2 --namespace test-domain-2
    bash packages/grid/scripts/wait_for.sh service seaweedfs --context k3d-test-domain-2 --namespace test-domain-2

    pytest tests/integration -m frontend -p no:randomly --co
    bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m frontend -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no"

    bash -c '(kubectl logs deployment.apps/backend-stream --context k3d-test-network-1 --namespace test-network-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(kubectl logs deployment.apps/backend-stream --context k3d-test-domain-1 --namespace test-domain-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(kubectl logs deployment.apps/backend-stream --context k3d-test-domain-2 --namespace test-domain-2 -f &) | grep -q "Application startup complete" || true'

    pytest tests/integration -m network -p no:randomly --co
    bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m network -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no"

    pytest tests/integration -m e2e -p no:randomly --co
    bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m e2e -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no"

    pytest tests/integration -m security -p no:randomly --co
    bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m security -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no"

    pytest tests/integration -m security -p no:randomly --co
    bash -c "CONTAINER_HOST=$CONTAINER_HOST pytest tests/integration -m redis -vvvv -p no:randomly -p no:benchmark -o log_cli=True --capture=no"

    ; bash -c "k3d cluster delete test-network-1 || true"
    ; bash -c "k3d cluster delete test-domain-1 || true"
    ; bash -c "k3d cluster delete test-domain-2 || true"
    ; bash -c "k3d registry delete k3d-registry.localhost || true"
    bash -c "docker rm $(docker ps -aq) --force || true"
    bash -c "docker volume rm k3d-test-network-1-images --force || true"
    bash -c "docker volume rm k3d-test-domain-1-images --force || true"
    bash -c "docker volume rm k3d-test-domain-2-images --force || true"


[testenv:stack.test.course]
description = Integration Tests for Course Notebooks
deps =
    {[testenv:syft]deps}
changedir = {toxinidir}
allowlist_externals =
    docker
    grep
    sleep
    bash
commands =
    pip install -e packages/hagrid
    pip install testbook prompt-toolkit jupyter
    docker --version
    docker compose version
    bash -c "docker volume rm test_domain_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:8081 --tag=local --tail=false --headless=true --test --health_checks=false --health_checks=false'
    bash -c 'rm -rf tests/course/courses'
    bash -c 'git clone https://github.com/OpenMined/courses.git tests/course/courses/ || true'
    bash -c 'cd tests/course/courses && git fetch && git checkout introduction-to-remote-data-science-dev && git pull || true'

    bash -c 'cp -r tests/course/tests/ tests/course/courses/'

    docker ps
    bash -c '(docker logs test_domain_1-backend_stream-1 -f &) | grep -q "Application startup complete" || true'
    ; perform course notebook test
    bash -c 'cd tests/course/courses && pytest -p no:randomly -vvvv'

    ; stop containers
    bash -c 'HAGRID_ART=false hagrid land test_domain_1 --force'
    bash -c 'rm -rf tests/course/courses'

[testenv:grid.test.backend]
description = Tests for Grid Backend
deps =
    {[testenv:syft]deps}
    -r {toxinidir}/packages/grid/backend/requirements.txt
    -r {toxinidir}/packages/grid/backend/requirements.dev.txt
changedir = {toxinidir}/packages/grid/backend
allowlist_externals =
    bash
commands =
    python --version
    bash -c 'USE_NEW_SERVICE=False SQLALCHEMY_DATABASE_URI="sqlite:///file:test_db?mode=memory&cache=shared&uri=true" LOGURU_SINK="./grid.log" pytest grid/tests'
    # Test New Service
    bash -c 'USE_NEW_SERVICE=True SQLALCHEMY_DATABASE_URI="sqlite:///file:test_db?mode=memory&cache=shared&uri=true" LOGURU_SINK="./grid.log" pytest grid/tests/api/users/user_routes_test.py'

[testenv:grid.test.frontend]
description = Tests for Grid Frontend
deps =
changedir = {toxinidir}/packages/grid/frontend
allowlist_externals =
    bash
    echo
    yarn
    docker
commands =
    ; check yarn is installed
    bash ./scripts/check_yarn.sh

    ; install hagrid
    pip install -e {toxinidir}/packages/hagrid

    ; launch a domain
    bash -c "docker volume rm test_domain_1_app-db-data --force || true"
    bash -c "docker volume rm test_domain_1_seaweedfs-data --force || true"
    bash -c "docker volume rm test_domain_1_app-redis-data --force || true"
    bash -c "docker volume rm test_domain_1_tailscale-data --force || true"
    bash -c 'HAGRID_ART=false hagrid launch test_domain_1 domain to docker:9082 --tag=local --build=false --tail=false --test --health_checks=false'
    docker ps

    ; install packages
    yarn install

    ; wait for server to be up
    bash -c '(docker logs test_domain_1-backend-1 -f &) | grep -q "Application startup complete" || true'
    bash -c '(docker logs test_domain_1-frontend-1 -f &) | grep -q "event - compiled .* successfully\|nginx" || true'

    ; run cypress
    bash -c 'HTTP_PORT=9082 yarn cypress:run'

    bash -c 'HAGRID_ART=false hagrid land test_domain_1 --force'

[flake8]
ignore =
    W503
max-line-length = 120
exclude =
    .tox

[mypy]
python_version = 3.10