Merge pull request #336 from epoch8/elephantum/issue313

Enable python 3.12

.github/workflows/pytest.yaml

@@ -59,15 +59,18 @@ jobs:
           #   pip-extra: "'sqlalchemy>2'"
           # - python-version: "3.8"
           #   test-db-env: "sqlite"
+
           - python-version: "3.9"
             test-db-env: "postgres"
             pip-extra: '"sqlalchemy>2"'
           # - python-version: "3.9"
           #   test-db-env: "sqlite"
+
           # - python-version: "3.10"
           #   test-db-env: "postgres"
           # - python-version: "3.10"
           #   test-db-env: "sqlite"
+
           - python-version: "3.11"
             test-db-env: "postgres"
             pip-extra: '"sqlalchemy<2" "pandas<2.2"'
@@ -78,6 +81,11 @@ jobs:
             test-db-env: "sqlite"
             pip-extra: '"sqlalchemy>2"'
 
+          - python-version: "3.12"
+            test-db-env: "postgres"
+          - python-version: "3.12"
+            test-db-env: "sqlite"
+
     services:
       # Label used to access the service container
       postgres:

.github/workflows/test_examples.yaml

@@ -15,9 +15,10 @@ jobs:
       matrix:
         python-version:
           # - "3.8"
-          # - "3.9"
+          - "3.9"
           - "3.10"
           # - "3.11"
+          - "3.12"
         example:
           - datatable_batch_transform
           - image_resize

CHANGELOG.md

@@ -1,6 +1,7 @@
 # WIP 0.14.0
 
 Changes:
+* Enable Python 3.12 support
 * `DatatableTansform` can become `BatchTransform` with empty indices
 * SQLAlchemy tables can be used directly without duplication in Catalog
 * `datapipe.compute.Table` can be used directly without Catalog

datapipe/cli.py

@@ -513,8 +513,14 @@ def migrate_transform_tables(ctx: click.Context, labels: str, name: str) -> None
     return migrations_v013.migrate_transform_tables(app, batch_transforms_steps)
 
 
-for entry_point in metadata.entry_points().get("datapipe.cli", []):
-    register_commands = entry_point.load()
+try:
+    entry_points = metadata.entry_points(group="datapipe.cli")  # type: ignore
+except TypeError:
+    # Compatibility with older versions of importlib.metadata (Python 3.8-3.9)
+    entry_points = metadata.entry_points().get("datapipe.cli", [])  # type: ignore
+
+for entry_point in entry_points:
+    register_commands = entry_point.load()  # type: ignore
     register_commands(cli)
 
 

docs/source/extending-cli.md

@@ -1,5 +1,7 @@
 # Extending `datapipe` cli
 
+## Entry point
+
 Datapipe offers a way to add additional cli commands. It is achieved by
 utilizing Python entrypoints mechanism.
 
@@ -13,5 +15,17 @@ def register_commands(cli: click.Group) -> None:
     ...
 ```
 
+## Context
+
+Plugin can expect some information in `click.Context`:
+
+* `ctx.obj["pipeline"]`: `datapipe.compute.DatapipeApp` instance of DatapipeApp
+  with all necessary initialization steps performed
+
+* `ctx.obj["executor"]`: `datapipe.executor.Executor` contains an instance of
+  Executor which will be used to perform computation
+
+## Example
+
 To see example of extending `datapipe` cli see
-[`datapipe-app`](https://github.com/epoch8/datapipe-app)
+[`datapipe_app.cli`](https://github.com/epoch8/datapipe-app/blob/master/datapipe_app/cli.py)

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "datapipe-core"
-version = "0.14.0-alpha.1"
+version = "0.14.0-alpha.2"
 description = "`datapipe` is a realtime incremental ETL library for Python application"
 readme = "README.md"
 repository = "https://github.com/epoch8/datapipe"
@@ -12,7 +12,7 @@ packages = [
 include = ["datapipe/py.typed"]
 
 [tool.poetry.dependencies]
-python = ">=3.9,<3.12"
+python = ">=3.9,<3.13"
 fsspec = ">=2021.11.1"
 
 gcsfs = {version=">=2021.11.1", optional=true}