fix migrations and add airflow tutorial (#3)

.gitignore

@@ -10,7 +10,7 @@ __pycache__/
 .Python
 build/
 develop-eggs/
-dist/
+lab/airflow/dist/
 downloads/
 eggs/
 .eggs/

README.md

@@ -26,6 +26,11 @@ With CanaryPy, you gain the ability to test the waters with new features or upda
 
 ## How it works
 
+CanaryPy has three main entities: 
+* `Product`: It is an application that will have releases. 
+* `Release`: It is any version of your Product. It can be of two types: `active` and `canary`. A release is created as canary which will be tested against the `active` using a linear growth rollout strategy. If the canary performs well during all the phases, it'll be promoted to `active`. The release performance is measure based on its signals.
+* `Signal`: It the execution result of a release. It can have a `success` or `failed` state. Typically, is the result of your job.
+
 CanaryPy also includes a CLI to manage the products, releases, and signals. The CLI is built using the Click library.
 
 The CLI uses the FastAPI application as the backend and therefore the following environment variables need to be set to run the CLI:
@@ -69,6 +74,14 @@ client = CanaryPyClient(base_url="http://localhost:8000")
 client.send_signal_to_canary(artifact_url, version, instance_id, description, status)
 ```
 
+# Airflow Integration
+
+`CanaryPy` provides a plugin to integrate your Airflow tasks with CanaryPy backend with a minimal effort. The plugin can be installed by executing:
+
+`pip install canarypy-airflow-plugin`
+
+Check this [tutorial](examples/airflow-tutorial.md) to understand how this integration works. The plugin code can be found [here](integrations/airflow).
+
 # CanaryPy Deployment
 
 CanaryPy is built around a modular architecture composed by an FastAPI application that host the rest APIs and a Streamlist application that show metrics about release performance. Both applications use a shared PostgreSQL database to store the data.

alembic.ini → canarypy/alembic.ini

@@ -2,7 +2,7 @@
 
 [alembic]
 # path to migration scripts
-script_location = alembic
+script_location = %(here)s/migrations
 
 # template used to generate migration files
 # file_template = %%(rev)s_%%(slug)s

canarypy/cli/main.py

@@ -6,6 +6,7 @@
 from canarypy.cli.services.product import ProductService
 from canarypy.cli.services.release import ReleaseService
 from canarypy.cli.services.signal import SignalService
+from canarypy.config import MIGRATION_PATH
 
 
 @click.group()
@@ -122,5 +123,5 @@ def init():
     from alembic import command
     from alembic.config import Config
 
-    alembic_cfg = Config("alembic.ini")
+    alembic_cfg = Config(MIGRATION_PATH)
     command.upgrade(alembic_cfg, "head")

canarypy/config.py

@@ -0,0 +1,4 @@
+from pathlib import Path
+
+BASE_DIR = Path(__file__).resolve().parent
+MIGRATION_PATH = BASE_DIR.parent / "canarypy" / "alembic.ini"

alembic/env.py → canarypy/migrations/env.py

@@ -2,9 +2,8 @@
 import sys
 from logging.config import fileConfig
 
-from sqlalchemy import create_engine, pool
-
 from alembic import context
+from sqlalchemy import create_engine, pool
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

alembic/versions/1c26034449b9_add_band_number.py → canarypy/migrations/versions/1c26034449b9_add_band_number.py

@@ -5,7 +5,6 @@
 Create Date: 2023-05-13 20:32:34.412577
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/33873c5a7bbe_change_canary_period_type.py → canarypy/migrations/versions/33873c5a7bbe_change_canary_period_type.py

@@ -5,7 +5,6 @@
 Create Date: 2023-05-05 15:49:31.100669
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/4a95c3cf7adc_initial_tables.py → canarypy/migrations/versions/4a95c3cf7adc_initial_tables.py

@@ -5,9 +5,8 @@
 Create Date: 2023-04-22 14:27:28.070017
 """
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "4a95c3cf7adc"

alembic/versions/5a588045e274_add_release_date.py → canarypy/migrations/versions/5a588045e274_add_release_date.py

@@ -5,7 +5,6 @@
 Create Date: 2023-04-22 18:57:49.724856
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/652e9c47c653_add_is_canary_to_signal.py → canarypy/migrations/versions/652e9c47c653_add_is_canary_to_signal.py

@@ -5,7 +5,6 @@
 Create Date: 2023-05-28 14:55:01.811426
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/67c9708da025_add_release_canary_band_table.py → canarypy/migrations/versions/67c9708da025_add_release_canary_band_table.py

@@ -5,9 +5,8 @@
 Create Date: 2023-05-13 15:40:29.225959
 """
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "67c9708da025"

alembic/versions/738957fcc3bc_add_signal_period_column.py → canarypy/migrations/versions/738957fcc3bc_add_signal_period_column.py

@@ -5,7 +5,6 @@
 Create Date: 2023-04-30 15:33:22.382549
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/84a61d063ad6_add_release_status_columns.py → canarypy/migrations/versions/84a61d063ad6_add_release_status_columns.py

@@ -5,7 +5,6 @@
 Create Date: 2023-04-30 14:16:05.993738
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/9261790e46ac_change_canary_period_type.py → canarypy/migrations/versions/9261790e46ac_change_canary_period_type.py

@@ -5,7 +5,6 @@
 Create Date: 2023-05-20 06:01:28.735292
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/9505cfab1f85_refactor_bands.py → canarypy/migrations/versions/9505cfab1f85_refactor_bands.py

@@ -5,7 +5,6 @@
 Create Date: 2023-05-13 16:29:05.937050
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/9dbfd3ac874b_add_release_period_columns.py → canarypy/migrations/versions/9dbfd3ac874b_add_release_period_columns.py

@@ -5,7 +5,6 @@
 Create Date: 2023-04-30 14:43:49.077569
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/aebc398fc592_change_mandatory_field.py → canarypy/migrations/versions/aebc398fc592_change_mandatory_field.py

@@ -5,9 +5,8 @@
 Create Date: 2023-05-13 15:51:09.967636
 """
 import sqlalchemy as sa
-from sqlalchemy.dialects import postgresql
-
 from alembic import op
+from sqlalchemy.dialects import postgresql
 
 # revision identifiers, used by Alembic.
 revision = "aebc398fc592"

alembic/versions/d1b816f53004_change_product_name.py → canarypy/migrations/versions/d1b816f53004_change_product_name.py

@@ -5,7 +5,6 @@
 Create Date: 2023-05-03 20:34:52.882065
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

alembic/versions/ea0d1d149192_change_status_field.py → canarypy/migrations/versions/ea0d1d149192_change_status_field.py

@@ -5,7 +5,6 @@
 Create Date: 2023-04-22 16:05:23.603570
 """
 import sqlalchemy as sa
-
 from alembic import op
 
 # revision identifiers, used by Alembic.

examples/Dockerfile

@@ -0,0 +1,20 @@
+FROM python:3.9-slim
+
+# Required to run get SSL certs and perform health checks when locally run in docker-compose
+RUN apt-get update -q && apt-get install -yq curl netcat-openbsd
+
+RUN useradd -ms /bin/bash canarypy
+RUN mkdir /var/log/myapp
+WORKDIR /home/canarypy
+
+USER canarypy:canarypy
+
+ENV PATH="/home/canarypy/.local/bin:${PATH}"
+ENV PYTHONPATH="/home/canarypy/.local/lib/python3.9/site-packages:${PYTHONPATH:-}"
+
+RUN pip install canarypy
+
+ENV PORT=8080
+EXPOSE $PORT
+
+CMD ["canarypy", "api", "start"]

examples/airflow-tutorial.md

@@ -0,0 +1,68 @@
+## Canarypy Airflow Tutorial
+
+This page provides a tutorial on how to use `canarypy` with Apache Airflow. It assumes that you have a working knowledge of Airflow.
+
+### Environment setup
+
+1. From the root of the `examples` directory, run `docker-compose up -d` to start the `canarypy` backend. This will create the folowing containers:
+* `API`: The `canarypy` backend API.
+* `DB`: The `canarypy` backend database.
+* `WEB`: The `canarypy` backend dashboard.
+* `INIT`: Temporary container that runs the `canarypy` backend database migrations.
+
+2. From the root of the `examples/airflow` directory, run `docker-compose up -d` to start the Airflow instance. This will create the local Airflow setup with the `canarypy` plugin package already installed.
+
+If you can access http://localhost:8080, then the Airflow instance is running correctly. The airflow user/passwrod is: airflow/airflow.
+
+In the Airflow DAG list, you should see a DAG called `docker_operator_demo`. This DAG contains three `DockerOperator` tasks that uses the `python` image to print basic commands. On this tutorial, we'll create a few releases to demo the version switching automatically.
+
+### Canarypy setup
+
+The very first thing to do within Canarypy is to create a Product. In our example is Python.
+
+### 1. Create a product in the `canarypy` backend
+
+The `canarypy` command-line interface (CLI) provides commands to interact with the `canarypy` backend. The `product create` command is used to create a new product.
+
+Before running the command, we need to specify the `canarypy` backend URL by setting the `CANARYPY_URL` environment variable. This can be done directly in the command line like so:
+
+```bash
+export CANARYPY_URL=http://localhost:9090
+```
+
+This sets the backend URL to `http://localhost:9090`. Ensure that your `canarypy` backend is running on this URL. If it's running on a different URL, replace `http://localhost:9090` with the correct URL.
+
+After setting the `CANARYPY_URL`, you can create the product:
+
+```bash
+canarypy product create
+```
+
+This command will prompt you a few questions:
+* `Enter the name of this product?` super python
+* `What the git repository url of this product?` https://github.com/python/cpython
+* `What the artifact url of this product?` python
+
+The artifact url is what it will be used by Canarypy Airflow plugin to fetch the correct artifact.  
+
+### 2. Create a release for the product
+
+The `canarypy release create` command is used to create a new release for a product. This command takes a `--semver-version` argument to specify the semantic version of the release, and an `--artifact-url` argument to specify the URL where the release's artifact can be downloaded.
+
+In a real world scenario, the `canarypy release create` command can be integrated into your CI/CD pipeline. After your product is built and deployed, the command can be executed to create a new release in the `canarypy` backend:
+
+```bash
+canarypy release create --semver-version 3.10-bullseye --artifact-url python
+```
+
+After creating the release, trigger the DAG in the Airflow UI at `http://localhost:8080`. You can check the logs of the Docker operator to confirm that it's using the correct version.
+
+### 5. Create another release and check the DAG running on the newest version
+
+To create another release for your product, use the `canarypy release create` command again with the new version and artifact URL:
+
+```bash
+canarypy release create --semver-version 3.11-bullseye --artifact-url python
+```
+
+Now check that the DAG will run using the newest release version of your product.

examples/airflow/DockerfileAirflow

@@ -0,0 +1,5 @@
+FROM apache/airflow:2.6.2-python3.10
+
+USER airflow
+
+RUN pip install canarypy-airflow-plugin

lab/airflow/dags/docker_example.py → examples/airflow/dags/docker_example.py

@@ -1,7 +1,6 @@
 from datetime import datetime, timedelta
 
 from airflow import DAG
-from airflow.operators.bash import BashOperator
 from airflow.providers.docker.operators.docker import DockerOperator
 
 default_args = {
@@ -21,11 +20,19 @@
     schedule_interval="5 * * * *",
     catchup=False,
 ) as dag:
-
-    t1 = BashOperator(task_id="print_current_date", bash_command="date")
+    t1 = DockerOperator(
+        task_id="docker_command_starting",
+        image='{{ macros.canarypy_plugin.get_latest_stable_version("python") }}',
+        container_name="task___command_sleep",
+        api_version="auto",
+        auto_remove=True,
+        command="echo starting",
+        docker_url="unix://var/run/docker.sock",
+        network_mode="bridge",
+    )
 
     t2 = DockerOperator(
-        task_id="docker_command_sleep",
+        task_id="docker_command_hello",
         image='{{ macros.canarypy_plugin.get_latest_stable_version("python") }}',
         container_name="task___command_sleep",
         api_version="auto",
@@ -35,6 +42,15 @@
         network_mode="bridge",
     )
 
-    t4 = BashOperator(task_id="print_hello", bash_command='echo "hello world"')
+    t3 = DockerOperator(
+        task_id="docker_command_world",
+        image='{{ macros.canarypy_plugin.get_latest_stable_version("python") }}',
+        container_name="task___command_sleep",
+        api_version="auto",
+        auto_remove=True,
+        command="echo world",
+        docker_url="unix://var/run/docker.sock",
+        network_mode="bridge",
+    )
 
-    t1 >> t2 >> t4
+    t1 >> t2 >> t3