diff --git a/.github/workflows/pytest.yaml b/.github/workflows/pytest.yaml index 41caa049..ef2a3b02 100644 --- a/.github/workflows/pytest.yaml +++ b/.github/workflows/pytest.yaml @@ -59,15 +59,18 @@ jobs: # pip-extra: "'sqlalchemy>2'" # - python-version: "3.8" # test-db-env: "sqlite" + - python-version: "3.9" test-db-env: "postgres" pip-extra: '"sqlalchemy>2"' # - python-version: "3.9" # test-db-env: "sqlite" + # - python-version: "3.10" # test-db-env: "postgres" # - python-version: "3.10" # test-db-env: "sqlite" + - python-version: "3.11" test-db-env: "postgres" pip-extra: '"sqlalchemy<2" "pandas<2.2"' @@ -78,6 +81,11 @@ jobs: test-db-env: "sqlite" pip-extra: '"sqlalchemy>2"' + - python-version: "3.12" + test-db-env: "postgres" + - python-version: "3.12" + test-db-env: "sqlite" + services: # Label used to access the service container postgres: diff --git a/.github/workflows/test_examples.yaml b/.github/workflows/test_examples.yaml index 7eee97c0..5fc3d59b 100644 --- a/.github/workflows/test_examples.yaml +++ b/.github/workflows/test_examples.yaml @@ -15,9 +15,10 @@ jobs: matrix: python-version: # - "3.8" - # - "3.9" + - "3.9" - "3.10" # - "3.11" + - "3.12" example: - datatable_batch_transform - image_resize diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c76d9dd..48bbb2a5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # WIP 0.14.0 Changes: +* Enable Python 3.12 support * `DatatableTansform` can become `BatchTransform` with empty indices * SQLAlchemy tables can be used directly without duplication in Catalog * `datapipe.compute.Table` can be used directly without Catalog diff --git a/datapipe/cli.py b/datapipe/cli.py index 753b2fbe..39fbc430 100644 --- a/datapipe/cli.py +++ b/datapipe/cli.py @@ -513,8 +513,14 @@ def migrate_transform_tables(ctx: click.Context, labels: str, name: str) -> None return migrations_v013.migrate_transform_tables(app, batch_transforms_steps) -for entry_point in metadata.entry_points().get("datapipe.cli", []): - register_commands = entry_point.load() +try: + entry_points = metadata.entry_points(group="datapipe.cli") # type: ignore +except TypeError: + # Compatibility with older versions of importlib.metadata (Python 3.8-3.9) + entry_points = metadata.entry_points().get("datapipe.cli", []) # type: ignore + +for entry_point in entry_points: + register_commands = entry_point.load() # type: ignore register_commands(cli) diff --git a/docs/source/extending-cli.md b/docs/source/extending-cli.md index bba0eae0..2a79321b 100644 --- a/docs/source/extending-cli.md +++ b/docs/source/extending-cli.md @@ -1,5 +1,7 @@ # Extending `datapipe` cli +## Entry point + Datapipe offers a way to add additional cli commands. It is achieved by utilizing Python entrypoints mechanism. @@ -13,5 +15,17 @@ def register_commands(cli: click.Group) -> None: ... ``` +## Context + +Plugin can expect some information in `click.Context`: + +* `ctx.obj["pipeline"]`: `datapipe.compute.DatapipeApp` instance of DatapipeApp + with all necessary initialization steps performed + +* `ctx.obj["executor"]`: `datapipe.executor.Executor` contains an instance of + Executor which will be used to perform computation + +## Example + To see example of extending `datapipe` cli see -[`datapipe-app`](https://github.com/epoch8/datapipe-app) +[`datapipe_app.cli`](https://github.com/epoch8/datapipe-app/blob/master/datapipe_app/cli.py) diff --git a/pyproject.toml b/pyproject.toml index 814d9141..99f8b4e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datapipe-core" -version = "0.14.0-alpha.1" +version = "0.14.0-alpha.2" description = "`datapipe` is a realtime incremental ETL library for Python application" readme = "README.md" repository = "https://github.com/epoch8/datapipe" @@ -12,7 +12,7 @@ packages = [ include = ["datapipe/py.typed"] [tool.poetry.dependencies] -python = ">=3.9,<3.12" +python = ">=3.9,<3.13" fsspec = ">=2021.11.1" gcsfs = {version=">=2021.11.1", optional=true}