From 1d01330e9720124f596fbecbc20c044960a4f146 Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 9 Jan 2024 17:08:44 +0100 Subject: [PATCH 1/3] parse sub-model for table_settings --- tests/conftest.py | 22 ++++++++++++++++++++++ tests/routes/test_item.py | 13 +++++++++++++ tipg/collections.py | 18 ++++++------------ tipg/settings.py | 25 ++++++++++++++++++------- 4 files changed, 59 insertions(+), 19 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 6f1fa2b9..db5753ab 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -558,3 +558,25 @@ def app_functions(database_url, monkeypatch): with TestClient(app) as client: yield client + + +@pytest.fixture(autouse=True) +def app_public_table(database_url, monkeypatch): + """Create app with connection to the pytest database.""" + monkeypatch.setenv("TIPG_TABLE_CONFIG__public_landsat_wrs__properties", '["pr"]') + + postgres_settings = PostgresSettings(database_url=database_url) + db_settings = DatabaseSettings( + schemas=["public"], + functions=[], + ) + sql_settings = CustomSQLSettings(custom_sql_directory=None) + + app = create_tipg_app( + postgres_settings=postgres_settings, + db_settings=db_settings, + sql_settings=sql_settings, + ) + + with TestClient(app) as client: + yield client diff --git a/tests/routes/test_item.py b/tests/routes/test_item.py index 07e471bd..1e05d52c 100644 --- a/tests/routes/test_item.py +++ b/tests/routes/test_item.py @@ -38,3 +38,16 @@ def test_item(app): # not found response = app.get("/collections/public.landsat_wrs/items/50000") assert response.status_code == 404 + + +def test_item_with_property_config(app_public_table): + """Test /items/{item id} endpoint.""" + response = app_public_table.get("/collections/public.landsat_wrs/items/1") + assert response.status_code == 200 + assert response.headers["content-type"] == "application/geo+json" + body = response.json() + assert body["type"] == "Feature" + assert body["id"] == 1 + assert body["links"] + print(body["properties"]) + Item.model_validate(body) diff --git a/tipg/collections.py b/tipg/collections.py index 1164ca1b..fc2b1515 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -25,7 +25,7 @@ from tipg.filter.filters import bbox_to_wkt from tipg.logger import logger from tipg.model import Extent -from tipg.settings import FeaturesSettings, MVTSettings, TableSettings +from tipg.settings import FeaturesSettings, MVTSettings, TableConfig, TableSettings from fastapi import FastAPI @@ -958,16 +958,16 @@ async def get_collection_index( # noqa: C901 if table_id == "pg_temp.tipg_catalog": continue - table_conf = table_confs.get(confid, {}) + table_conf = table_confs.get(confid, TableConfig()) # Make sure that any properties set in conf exist in table properties = sorted(table.get("properties", []), key=lambda d: d["name"]) - properties_setting = table_conf.get("properties", []) + properties_setting = table_conf.properties or [] if properties_setting: properties = [p for p in properties if p["name"] in properties_setting] # ID Column - id_column = table_conf.get("pk") or table.get("pk") + id_column = table_conf.pk or table.get("pk") if not id_column and fallback_key_names: for p in properties: if p["name"] in fallback_key_names: @@ -979,17 +979,11 @@ async def get_collection_index( # noqa: C901 for c in properties: if c.get("type") in ("timestamp", "timestamptz", "date"): - if ( - table_conf.get("datetimecol") == c["name"] - or datetime_column is None - ): + if table_conf.datetimecol == c["name"] or datetime_column is None: datetime_column = c if c.get("type") in ("geometry", "geography"): - if ( - table_conf.get("geomcol") == c["name"] - or geometry_column is None - ): + if table_conf.geomcol == c["name"] or geometry_column is None: geometry_column = c catalog[table_id] = Collection( diff --git a/tipg/settings.py b/tipg/settings.py index 46091b41..154adb15 100644 --- a/tipg/settings.py +++ b/tipg/settings.py @@ -1,9 +1,11 @@ """tipg config.""" +import json import pathlib -from typing import Dict, List, Optional +from typing import Any, Dict, List, Optional from pydantic import ( + BaseModel, DirectoryPath, Field, PostgresDsn, @@ -12,7 +14,6 @@ model_validator, ) from pydantic_settings import BaseSettings -from typing_extensions import TypedDict class APISettings(BaseSettings): @@ -36,13 +37,23 @@ def parse_cors_origin(cls, v): return [origin.strip() for origin in v.split(",")] -class TableConfig(TypedDict, total=False): +class TableConfig(BaseModel): """Configuration to add table options with env variables.""" - geomcol: Optional[str] - datetimecol: Optional[str] - pk: Optional[str] - properties: Optional[List[str]] + geomcol: Optional[str] = None + datetimecol: Optional[str] = None + pk: Optional[str] = None + properties: Optional[List[str]] = None + + model_config = {"extra": "ignore"} + + @field_validator("properties", mode="before") + def _properties(cls, v: Any) -> Any: + """set geometry from geo interface or input""" + if isinstance(v, str): + return json.loads(v) + else: + return v class TableSettings(BaseSettings): From 11f8977ecaf9e30eb09a809f3feb2494e3fca0df Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 9 Jan 2024 18:31:43 +0100 Subject: [PATCH 2/3] forward table columns in collection object --- tests/routes/test_item.py | 2 +- tipg/collections.py | 55 ++++++++++++++++++--------------------- 2 files changed, 26 insertions(+), 31 deletions(-) diff --git a/tests/routes/test_item.py b/tests/routes/test_item.py index 1e05d52c..44124535 100644 --- a/tests/routes/test_item.py +++ b/tests/routes/test_item.py @@ -49,5 +49,5 @@ def test_item_with_property_config(app_public_table): assert body["type"] == "Feature" assert body["id"] == 1 assert body["links"] - print(body["properties"]) + assert list(body["properties"]) == ["pr"] Item.model_validate(body) diff --git a/tipg/collections.py b/tipg/collections.py index fc2b1515..cdb26524 100644 --- a/tipg/collections.py +++ b/tipg/collections.py @@ -163,8 +163,9 @@ class Collection(BaseModel): dbschema: str = Field(..., alias="schema") title: Optional[str] = None description: Optional[str] = None + table_columns: List[Column] = [] properties: List[Column] = [] - id_column: Optional[str] = None + id_column: Optional[Column] = None geometry_column: Optional[Column] = None datetime_column: Optional[Column] = None parameters: List[Parameter] = [] @@ -237,12 +238,12 @@ def crs(self): @property def geometry_columns(self) -> List[Column]: """Return geometry columns.""" - return [c for c in self.properties if c.is_geometry] + return [c for c in self.table_columns if c.is_geometry] @property def datetime_columns(self) -> List[Column]: """Return datetime columns.""" - return [c for c in self.properties if c.is_datetime] + return [c for c in self.table_columns if c.is_datetime] def get_geometry_column(self, name: Optional[str] = None) -> Optional[Column]: """Return the name of the first geometry column.""" @@ -272,13 +273,6 @@ def get_datetime_column(self, name: Optional[str] = None) -> Optional[Column]: return None - @property - def id_column_info(self) -> Column: # type: ignore - """Return Column for a unique identifier.""" - for col in self.properties: - if col.name == self.id_column: - return col - def columns(self, properties: Optional[List[str]] = None) -> List[str]: """Return table columns optionally filtered to only include columns from properties.""" if properties in [[], [""]]: @@ -311,7 +305,7 @@ def _select_no_geo(self, properties: Optional[List[str]], addid: bool = True): if addid: if self.id_column: - id_clause = logic.V(self.id_column).as_("tipg_id") + id_clause = logic.V(self.id_column.name).as_("tipg_id") else: id_clause = raw(" ROW_NUMBER () OVER () AS tipg_id ") if nocomma: @@ -480,18 +474,14 @@ def _where( # noqa: C901 if ids is not None: if len(ids) == 1: wheres.append( - logic.V(self.id_column) - == pg_funcs.cast( - pg_funcs.cast(ids[0], "text"), self.id_column_info.type - ) + logic.V(self.id_column.name) + == pg_funcs.cast(pg_funcs.cast(ids[0], "text"), self.id_column.type) ) else: w = [ - logic.V(self.id_column) + logic.V(self.id_column.name) == logic.S( - pg_funcs.cast( - pg_funcs.cast(i, "text"), self.id_column_info.type - ) + pg_funcs.cast(pg_funcs.cast(i, "text"), self.id_column.type) ) for i in ids ] @@ -626,7 +616,7 @@ def _sortby(self, sortby: Optional[str]): else: if self.id_column is not None: - sorts.append(logic.V(self.id_column)) + sorts.append(logic.V(self.id_column.name)) else: sorts.append(logic.V(self.properties[0].name)) @@ -961,23 +951,27 @@ async def get_collection_index( # noqa: C901 table_conf = table_confs.get(confid, TableConfig()) # Make sure that any properties set in conf exist in table - properties = sorted(table.get("properties", []), key=lambda d: d["name"]) - properties_setting = table_conf.properties or [] - if properties_setting: - properties = [p for p in properties if p["name"] in properties_setting] + columns = sorted(table.get("properties", []), key=lambda d: d["name"]) + properties_setting = table_conf.properties or [c["name"] for c in columns] # ID Column - id_column = table_conf.pk or table.get("pk") - if not id_column and fallback_key_names: - for p in properties: + id_column = None + if id_name := table_conf.pk or table.get("pk"): + for p in columns: + if id_name == p["name"]: + id_column = p + break + + if id_column is None and fallback_key_names: + for p in columns: if p["name"] in fallback_key_names: - id_column = p["name"] + id_column = p break datetime_column = None geometry_column = None - for c in properties: + for c in columns: if c.get("type") in ("timestamp", "timestamptz", "date"): if table_conf.datetimecol == c["name"] or datetime_column is None: datetime_column = c @@ -992,8 +986,9 @@ async def get_collection_index( # noqa: C901 table=table["name"], schema=table["schema"], description=table.get("description", None), + table_columns=columns, + properties=[p for p in columns if p["name"] in properties_setting], id_column=id_column, - properties=properties, datetime_column=datetime_column, geometry_column=geometry_column, parameters=table.get("parameters") or [], From 04bca7e2ccb569cec28fabaf3155a8e50ad41e1f Mon Sep 17 00:00:00 2001 From: vincentsarago Date: Tue, 9 Jan 2024 21:58:20 +0100 Subject: [PATCH 3/3] update changelog --- CHANGES.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGES.md b/CHANGES.md index 61693558..a10658dd 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,10 +6,12 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/). Note: Minor version `0.X.0` update might break the API, It's recommended to pin `tipg` to minor version: `tipg>=0.1,<0.2` -## [unreleased] +## [0.6.0] - 2024-01-09 - update FastAPI version lower limit to `>0.107.0` and adapt for new starlette version - fix invalid streaming response formatting +- refactor internal table properties handling +- fix sub-model Table settings (https://github.com/developmentseed/tipg/issues/154) ## [0.5.7] - 2024-01-08