From a9f7d2e8e29832c748c11e5893f137100b014e9f Mon Sep 17 00:00:00 2001 From: earmenda Date: Wed, 1 Dec 2021 14:38:32 -0800 Subject: [PATCH] Integrate with Conda and a publish through CI (#249) * Create metadata file to integrate with conda * Add github action for publishing to conda * Attempt to install Conda manually * Install conda-build with conda * Install anaconda manually again as miniconda is not enough * Install anaconda client as part of workflow * Test if conda activate makes anaconda available * Run conda init as well * Trying some changes out to fix upload action * Add anaconda to PATH * Testing anaconda bin output * Enable auto activation * Remove ls * Test with setup-miniconda again * Fix syntax erorrs in workflow * Use bash as workflow shell * Confirm fix by using shell * Only push to conda on tag releases * Explicitly add optional dependencies for conda compatibility * Add documentation for working with conda * Small grammar fix * docs nits, merge pypi/conda publish workflows Co-authored-by: Eduardo Armendariz Co-authored-by: Thomas La Piana --- .github/workflows/publish_package.yaml | 42 ++++++++++++++++ .github/workflows/publish_pypi.yaml | 24 --------- .gitignore | 1 + README.md | 8 +-- docs/fides/docs/cli/generate-dataset.md | 53 -------------------- docs/fides/docs/development/releases.md | 2 +- docs/fides/docs/installation/conda.md | 17 +++++++ docs/fides/docs/installation/installation.md | 14 ++++-- docs/fides/docs/installation/pypi.md | 6 --- docs/fides/mkdocs.yml | 1 + fidesctl/conda/meta.yaml | 45 +++++++++++++++++ fidesctl/requirements.txt | 2 +- fidesctl/setup.py | 3 +- 13 files changed, 123 insertions(+), 95 deletions(-) create mode 100644 .github/workflows/publish_package.yaml delete mode 100644 .github/workflows/publish_pypi.yaml delete mode 100644 docs/fides/docs/cli/generate-dataset.md create mode 100644 docs/fides/docs/installation/conda.md create mode 100644 fidesctl/conda/meta.yaml diff --git a/.github/workflows/publish_package.yaml b/.github/workflows/publish_package.yaml new file mode 100644 index 0000000000..cbdd76e8ee --- /dev/null +++ b/.github/workflows/publish_package.yaml @@ -0,0 +1,42 @@ +name: Publish fidesctl + +on: + push: + tags: + - "*" + +jobs: + upload_to_pypi: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + + - name: Install Twine + run: pip install twine + + - name: Twine Upload + run: | + cd fidesctl/ + python setup.py sdist + twine upload dist/* + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + + upload_to_conda: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Install Anaconda + uses: conda-incubator/setup-miniconda@v2 + with: + auto-activate-base: true + activate-environment: "" + - name: Conda Upload + shell: bash -l {0} + run: | + conda install conda-build + conda install anaconda-client + conda install conda-verify + conda build --output-folder ./conda-out/ ./fidesctl/conda/ -c plotly -c conda-forge + anaconda -t "${{ secrets.CONDA_TOKEN }}" upload -u ethyca ./conda-out/noarch/fidesctl*.bz2 diff --git a/.github/workflows/publish_pypi.yaml b/.github/workflows/publish_pypi.yaml deleted file mode 100644 index 4d357eb140..0000000000 --- a/.github/workflows/publish_pypi.yaml +++ /dev/null @@ -1,24 +0,0 @@ -name: Publish the CLI on PyPI - -on: - push: - tags: - - "*" - -jobs: - upload_to_pypi: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Install Twine - run: pip install twine - - - name: Twine Upload - run: | - cd fidesctl/ - python setup.py sdist - twine upload dist/* - env: - TWINE_USERNAME: __token__ - TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore index 5435b8020f..c67ca6003c 100644 --- a/.gitignore +++ b/.gitignore @@ -122,6 +122,7 @@ wheels/ .installed.cfg *.egg MANIFEST +conda-out/ # PyInstaller # Usually these files are written by a python script from a template diff --git a/README.md b/README.md index 84a80f1969..c2ef13d2c2 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Meet Fidesctl: Privacy Policies as Code -[![Latest Version][pypi-image]][pypi-url] +[![Latest Release Version][release-image]][release-url] [![License][license-image]][license-url] [![Code style: black][black-image]][black-url] [![Checked with mypy][mypy-image]][mypy-url] @@ -223,12 +223,12 @@ Read about the [Fides community](https://ethyca.github.io/fides/community/hints_ ## :balance_scale: License The Fides ecosystem of tools ([Fidesops](https://github.com/ethyca/fidesops) and [Fidesctl](https://github.com/ethyca/fides)) are licensed under the [Apache Software License Version 2.0](https://www.apache.org/licenses/LICENSE-2.0). -Fides tools are built on [Fideslang](https://github.com/ethyca/privacy-taxonomy), the Fides language specification, which is licensed under [CC by 4](https://github.com/ethyca/privacy-taxonomy/blob/main/LICENSE). +Fides tools are built on [Fideslang](https://github.com/ethyca/privacy-taxonomy), the Fides language specification, which is licensed under [CC by 4](https://github.com/ethyca/privacy-taxonomy/blob/main/LICENSE). Fides is created and sponsored by Ethyca: a developer tools company building the trust infrastructure of the internet. If you have questions or need assistance getting started, let us know at fides@ethyca.com! -[pypi-image]: https://img.shields.io/pypi/v/fidesctl.svg -[pypi-url]: https://pypi.python.org/pypi/fidesctl/ +[release-image]: https://img.shields.io/github/release/ethyca/fides.svg +[release-url]: https://github.com/ethyca/fides/releases [license-image]: https://img.shields.io/:license-Apache%202-blue.svg [license-url]: https://www.apache.org/licenses/LICENSE-2.0.txt [black-image]: https://img.shields.io/badge/code%20style-black-000000.svg diff --git a/docs/fides/docs/cli/generate-dataset.md b/docs/fides/docs/cli/generate-dataset.md deleted file mode 100644 index d32b205bca..0000000000 --- a/docs/fides/docs/cli/generate-dataset.md +++ /dev/null @@ -1,53 +0,0 @@ -
-

generate-dataset

- -
NAME
-
- annotate-dataset — guided dataset annotation -
-
SYNOPSIS
-
- 
fidesctl generate-dataset connection_string output_filename
-
- -
DESCRIPTION
-
- The generate-dataset command reads one more schemas from a database by executing the connection_string statement, transforms the schemas into Dataset resources, and writes the Datasets (as YAML) to output_filename. - - The connection to your database isn't dynamic. If you update your database schemas, you have to re-generate (or modify) your Dataset objects. - -
- -
ARGUMENTS
-
-
- connection_string -
-
- An SQLAlchemy-compatible database connection URL that connects to your database and reads one or more schemas. -
-
-
-
- output_filename -
-
-
- The name of the Dataset manifest file that the command will write to. The value can be an absolute or relative path, and should include the .yml or .yaml extension. - -
-
-
- -
OPTIONS
-
-
- -h/--help -
-
- Prints a synopsis of this command. -
-
-
- - diff --git a/docs/fides/docs/development/releases.md b/docs/fides/docs/development/releases.md index 82f3076429..23312b36b8 100644 --- a/docs/fides/docs/development/releases.md +++ b/docs/fides/docs/development/releases.md @@ -14,7 +14,7 @@ Fidesctl does not follow a set release schedule, but instead ships versions base For each release a corresponding GitHub Project is created. These projects can be found [here](https://github.com/ethyca/fides/projects). Issues are then added to release projects as a way to organize what will be included in each release. -Once a release project is complete and the core team signs off on the readiness of the release, a new version is cut using GitHub releases. You can see all fidesctl releases [here](https://github.com/ethyca/fides/releases). Each new release triggers a GitHub Action that pushes the new version to PyPI as well as pushes a clean version to DockerHub. The release project is then marked as `closed`. +Once a release project is complete and the core team signs off on the readiness of the release, a new version is cut using GitHub releases. You can see all fidesctl releases [here](https://github.com/ethyca/fides/releases). Each new release triggers a GitHub Action that pushes the new version to PyPI and Conda as well as pushes a clean version to DockerHub. The release project is then marked as `closed`. Hotfixes are an exception to this and can be added and pushed as patch versions when needed. diff --git a/docs/fides/docs/installation/conda.md b/docs/fides/docs/installation/conda.md new file mode 100644 index 0000000000..3a84e4edd5 --- /dev/null +++ b/docs/fides/docs/installation/conda.md @@ -0,0 +1,17 @@ +# Installation from Conda + +This page describes installations using the `fidesctl` package [published on Conda](https://anaconda.org/ethyca/fidesctl). + +## Installation + +To install fidesctl, first create an environment with the fidesctl package and necessary channels: + +```bash +conda create --name fidesctl-environment fidesctl \ + --channel ethyca \ + --channel plotly \ + --channel conda-forge +``` + +Then activate your environment to begin using the `fidesctl` cli: +`conda activate fidesctl-environment` diff --git a/docs/fides/docs/installation/installation.md b/docs/fides/docs/installation/installation.md index 714e901491..d2791930ce 100644 --- a/docs/fides/docs/installation/installation.md +++ b/docs/fides/docs/installation/installation.md @@ -6,14 +6,18 @@ You should also check-out the [prerequisites](prerequisites_dependencies.md) tha When you install fidesctl, you need to [setup the database](database.md) which must also be kept updated when fidesctl is upgraded. -## Using PyPI +## Installation Tools -More details: [Installation from PyPI](pypi.md) +Only `pip` and `conda` installations are currently officially supported. For more details see [Installation from PyPI](pypi.md) or [Installation from Conda](conda.md) + +In some cases a lightweight installation might be desired, for instance, if the webserver is not needed. If this is the case, our `pip` installation supports optional dependencies. + +While there are some successes with using other tools like poetry or pip-tools, they do not share the same workflow as the supported tools - especially when it comes to constraint vs. requirements management. Installing via Poetry or pip-tools is not currently supported. If you wish to install fidesctl using those tools you do so at your own discretion. **When this option works best** * This installation method is useful when you are not familiar with containers and Docker and want to install fidesctl on physical or virtual machines and you are used to installing and running software using custom deployment mechanism. -* The only officially supported mechanism of installation is via pip. +* The only officially supported mechanisms of installation are pip and conda. **Intended users** @@ -27,7 +31,7 @@ More details: [Installation from PyPI](pypi.md) **What the Fidesctl community provides for this method** -* You have [Installation from PyPI](pypi.md) on how to install the software but due to various environments and tools you might want to use, you might expect that there will be problems which are specific to your deployment and environment that you will have to diagnose and solve. +* You have [Installation from PyPI](pypi.md) and [Installation from Conda](conda.md) on how to install the software but due to various environments and tools you might want to use, you might expect that there will be problems which are specific to your deployment and environment that you will have to diagnose and solve. * You have the [Running fidesctl Locally](../quickstart/local_full.md) guide where you can see an example of running fidesctl with minimal dependencies and setup. You can use this guide to start fidesctl quickly for local testing and development, however this is only intended to provide inspiration, not to represent a production-grade installation. **Where to ask for help** @@ -37,7 +41,7 @@ More details: [Installation from PyPI](pypi.md) ## Using Production Docker Images -More details: [Installation from Docker](pypi.md) +More details: [Installation from Docker](docker.md) **When this option works best** diff --git a/docs/fides/docs/installation/pypi.md b/docs/fides/docs/installation/pypi.md index fed81cffc9..e369c14139 100644 --- a/docs/fides/docs/installation/pypi.md +++ b/docs/fides/docs/installation/pypi.md @@ -2,12 +2,6 @@ This page describes installations using the `fidesctl` package [published on PyPI](https://pypi.org/project/fidesctl/). -## Installation Tools - -Only `pip` installation is currently officially supported. - -While there are some successes with using other tools like poetry or pip-tools, they do not share the same workflow as pip - especially when it comes to constraint vs. requirements management. Installing via Poetry or pip-tools is not currently supported. If you wish to install fidesctl using those tools you do so at your own discretion. - ## Basic Installation To install Fidesctl, run: diff --git a/docs/fides/mkdocs.yml b/docs/fides/mkdocs.yml index 5c45e71315..d928f9970c 100644 --- a/docs/fides/mkdocs.yml +++ b/docs/fides/mkdocs.yml @@ -11,6 +11,7 @@ nav: - Installation Overview: installation/installation.md - Prerequisites & Dependencies: installation/prerequisites_dependencies.md - Installation from PyPI: installation/pypi.md + - Installation from Conda: installation/conda.md - Installation from Docker: installation/docker.md - Setting up the database: installation/database.md - Configuration: installation/configuration.md diff --git a/fidesctl/conda/meta.yaml b/fidesctl/conda/meta.yaml new file mode 100644 index 0000000000..1ec05e03ae --- /dev/null +++ b/fidesctl/conda/meta.yaml @@ -0,0 +1,45 @@ +{% set data = load_setup_py_data(setup_file='../setup.py', from_recipe_dir=True) %} + +package: + name: "{{ data.get('name')|lower }}" + version: "{{ data.get('version') }}" + +source: + path: "../.." + +build: + number: 0 + entry_points: + - fidesctl=fidesctl.cli:cli + script: | + cd fidesctl + {{ PYTHON }} -m pip install . -vv + noarch: python + +requirements: + host: + - pip + - python >=3.7, <4 + run: + - python >=3.7, <4 + {% for package in data.get('install_requires') %} + - {{ package }} + {% endfor %} + {% for package in data.get('extras_require')['all'] %} + - {{ package }} + {% endfor %} + +test: + imports: + - fidesctl + commands: + - fidesctl --help + +about: + home: "{{ data.get('url') }}" + license: "{{ data.get('license') }}" + license_family: APACHE + license_file: LICENSE + summary: "{{ data.get('description') }}" + description: "{{ data.get('long_description') }}" + doc_url: "https://ethyca.github.io/fides/" diff --git a/fidesctl/requirements.txt b/fidesctl/requirements.txt index 5eb6b29469..44ad9b6592 100644 --- a/fidesctl/requirements.txt +++ b/fidesctl/requirements.txt @@ -4,8 +4,8 @@ colorama==0.4.4 deepdiff==5.5.0 pandas==1.3.3 plotly==5.3.1 -PyJWT==2.1.0 pydantic==1.8.2 +PyJWT==2.1.0 pyyaml==5.4.1 requests==2.25.1 sqlalchemy==1.4.14 diff --git a/fidesctl/setup.py b/fidesctl/setup.py index ba793214ff..7f5f23ba68 100644 --- a/fidesctl/setup.py +++ b/fidesctl/setup.py @@ -6,13 +6,14 @@ long_description = open("README.md").read() # Requirements +# Explicitly add optional dependencies for conda compatiblity, for instance, avoid using fastapi[all] install_requires = open("requirements.txt").read().strip().split("\n") dev_requires = open("dev-requirements.txt").read().strip().split("\n") extras = { "postgres": ["psycopg2-binary==2.9.1"], "mysql": ["pymysql==1.0.2"], - "webserver": ["fastapi[all]==0.68.1", "psycopg2-binary==2.9.1"], + "webserver": ["fastapi==0.68.1", "uvicorn==0.15.0", "psycopg2-binary==2.9.1"], } extras["all"] = sum(extras.values(), [])