Merge branch 'geopython:master' into STAC_API_V1.0.0_Imp

.github/FUNDING.yml

@@ -1,4 +1,4 @@
 # These are supported funding model platforms
 
 #github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-custom: ['https://www.paypal.com/cgi-bin/webscr?cmd=_s-xclick&hosted_button_id=WT27AS28UFSNW&source=url'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
+custom: ['https://github.com/geopython/pygeoapi/wiki/Sponsorship'] # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']

.github/workflows/main.yml

@@ -94,7 +94,6 @@ jobs:
         pip3 install -r requirements-manager.txt
         pip3 install -r requirements-django.txt
         python3 setup.py install
-        pip3 install --upgrade "sqlalchemy<2"
         pip3 install --global-option=build_ext --global-option="-I/usr/include/gdal" GDAL==`gdal-config --version`
         #pip3 install --upgrade rasterio==1.1.8
     - name: setup test data ⚙️

.github/workflows/vulnerabilities.yml

@@ -22,7 +22,7 @@ jobs:
         working-directory: .
     steps:
     - name: Checkout pygeoapi
-      uses: actions/checkout@v4
+      uses: actions/checkout@master
     - name: Scan vulnerabilities with trivy
       uses: aquasecurity/trivy-action@master
       with:
@@ -37,6 +37,9 @@ jobs:
         docker buildx build -t ${{ github.repository }}:${{ github.sha }} --platform linux/amd64 --no-cache -f Dockerfile .
     - name: Scan locally built Docker image for vulnerabilities with trivy
       uses: aquasecurity/trivy-action@master
+      env:
+        TRIVY_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-db:2
+        TRIVY_JAVA_DB_REPOSITORY: public.ecr.aws/aquasecurity/trivy-java-db:1
       with:
         scan-type: image
         exit-code: 1

docs/source/admin-api.rst

@@ -14,6 +14,15 @@ The API is enabled with the following server configuration:
     server:
         admin: true # boolean on whether to enable Admin API.
 
+.. note::
+
+    If you generate the OpenAPI definition after enabling the admin API, the admin routes will be exposed on ``/openapi`` 
+
+    .. image:: /_static/openapi_admin.png
+        :alt: admin routes
+        :align: center
+
+
 Access control
 --------------
 

docs/source/conf.py

@@ -112,7 +112,7 @@ def __getattr__(cls, name):
 # built documents.
 #
 # The short X.Y version.
-version = '0.18.dev0'
+version = '0.19.dev0'
 # The full version, including alpha/beta/rc tags.
 release = version
 

docs/source/data-publishing/ogcapi-features.rst

@@ -145,7 +145,11 @@ To publish an ESRI `Feature Service`_ or `Map Service`_ specify the URL for the
 
 * ``id_field`` will often be ``OBJECTID``, ``objectid``, or ``FID``.
 * If the map or feature service is not shared publicly, the ``username`` and ``password`` fields can be set in the
-  configuration to authenticate into the service.
+  configuration to authenticate to the service.
+* If the map or feature service is self-hosted and not shared publicly, the ``token_service`` and optional ``referer`` fields
+  can be set in the configuration to authenticate to the service.
+
+To publish from an ArcGIS online hosted service:
 
 .. code-block:: yaml
 
@@ -158,6 +162,24 @@ To publish an ESRI `Feature Service`_ or `Map Service`_ specify the URL for the
          crs: 4326 # Optional crs (default is EPSG:4326)
          username: username # Optional ArcGIS username
          password: password # Optional ArcGIS password
+         token_service: https://your.server.com/arcgis/sharing/rest/generateToken  # optional URL to your generateToken service
+         referer: https://your.server.com  # optional referer, defaults to https://www.arcgis.com if not set
+
+To publish from a self-hosted service that is not publicly accessible, the ``token_service`` field is required:
+
+.. code-block:: yaml
+
+   providers:
+       - type: feature
+         name: ESRI
+         data: https://your.server.com/arcgis/rest/services/your-layer/MapServer/0
+         id_field: objectid
+         time_field: date_in_your_device_time_zone # Optional time field
+         crs: 4326 # Optional crs (default is EPSG:4326)
+         username: username # Optional ArcGIS username
+         password: password # Optional ArcGIS password
+         token_service: https://your.server.com/arcgis/sharing/rest/generateToken # Optional url to your generateToken service
+         referer: https://your.server.com # Optional referer, defaults to https://www.arcgis.com if not set
 
 GeoJSON
 ^^^^^^^

docs/source/security.rst

@@ -14,4 +14,5 @@ as required.
 The following projects provide security frameworks atop pygeoapi:
 
 * `fastgeoapi <https://github.com/geobeyond/fastgeoapi>`_
-* `pygeoapi-auth <https://github.com/cartologic/pygeoapi-auth>`_
+* `pygeoapi-auth-deployment <https://github.com/cartologic/pygeoapi-auth-deployment>`_
+* `pygeoapi-auth <https://github.com/geopython/pygeoapi-auth>`_ (Python package for use along with pygeoapi-auth-deployment)

pygeoapi/__init__.py

@@ -30,7 +30,7 @@
 #
 # =================================================================
 
-__version__ = '0.18.dev0'
+__version__ = '0.19.dev0'
 
 import click
 try:

pygeoapi/api/itemtypes.py

@@ -121,29 +121,22 @@ def get_collection_queryables(api: API, request: Union[APIRequest, Any],
             HTTPStatus.NOT_FOUND, headers, request.format, 'NotFound', msg)
 
     LOGGER.debug('Creating collection queryables')
-    try:
-        LOGGER.debug('Loading feature provider')
-        p = load_plugin('provider', get_provider_by_type(
-            api.config['resources'][dataset]['providers'], 'feature'))
-    except ProviderTypeError:
+
+    p = None
+    for pt in ['feature', 'coverage', 'record']:
         try:
-            LOGGER.debug('Loading coverage provider')
+            LOGGER.debug(f'Loading {pt} provider')
             p = load_plugin('provider', get_provider_by_type(
-                api.config['resources'][dataset]['providers'], 'coverage'))  # noqa
+                api.config['resources'][dataset]['providers'], pt))
+            break
         except ProviderTypeError:
-            LOGGER.debug('Loading record provider')
-            p = load_plugin('provider', get_provider_by_type(
-                api.config['resources'][dataset]['providers'], 'record'))
-        finally:
-            msg = 'queryables not available for this collection'
-            return api.get_exception(
-                HTTPStatus.BAD_REQUEST, headers, request.format,
-                'NoApplicableError', msg)
+            LOGGER.debug(f'Providing type {pt} not found')
 
-    except ProviderGenericError as err:
+    if p is None:
+        msg = 'queryables not available for this collection'
         return api.get_exception(
-            err.http_status_code, headers, request.format,
-            err.ogc_exception_code, err.message)
+            HTTPStatus.BAD_REQUEST, headers, request.format,
+            'NoApplicableError', msg)
 
     queryables = {
         'type': 'object',

pygeoapi/flask_app.py

@@ -279,11 +279,7 @@ def collection_items(collection_id, item_id=None):
     """
 
     if item_id is None:
-        if request.method == 'GET':  # list items
-            return execute_from_flask(itemtypes_api.get_collection_items,
-                                      request, collection_id,
-                                      skip_valid_check=True)
-        elif request.method == 'POST':  # filter or manage items
+        if request.method == 'POST':  # filter or manage items
             if request.content_type is not None:
                 if request.content_type == 'application/geo+json':
                     return execute_from_flask(
@@ -298,6 +294,10 @@ def collection_items(collection_id, item_id=None):
             return execute_from_flask(
                     itemtypes_api.manage_collection_item, request, 'options',
                     collection_id, skip_valid_check=True)
+        else:  # GET: list items
+            return execute_from_flask(itemtypes_api.get_collection_items,
+                                      request, collection_id,
+                                      skip_valid_check=True)
 
     elif request.method == 'DELETE':
         return execute_from_flask(itemtypes_api.manage_collection_item,

pygeoapi/openapi.py

@@ -134,6 +134,52 @@ def gen_response_object(description: str, media_type: str,
     return response
 
 
+def gen_contact(cfg: dict) -> dict:
+    """
+    Generates an OpenAPI contact object with OGC extensions
+    based on OGC API - Records contact
+
+    :param cfg: `dict` of configuration
+
+    :returns: `dict` of OpenAPI contact object
+    """
+
+    contact = {
+        'name': cfg['metadata']['provider']['name'],
+        'url': cfg['metadata']['provider']['url'],
+        'email': cfg['metadata']['contact']['email']
+    }
+
+    contact['x-ogc-serviceContact'] = {
+        'name': cfg['metadata']['contact']['name'],
+        'position': cfg['metadata']['contact']['position'],
+        'addresses': [{
+            'deliveryPoint': [cfg['metadata']['contact']['address']],
+            'city': cfg['metadata']['contact']['city'],
+            'administrativeArea': cfg['metadata']['contact']['stateorprovince'],  # noqa
+            'postalCode': cfg['metadata']['contact']['postalcode'],
+            'country': cfg['metadata']['contact']['country']
+        }],
+        'phones': [{
+            'type': 'main', 'value': cfg['metadata']['contact']['phone']
+            }, {
+            'type': 'fax', 'value': cfg['metadata']['contact']['fax']
+        }],
+        'emails': [{
+            'value': cfg['metadata']['contact']['email']
+        }],
+        'contactInstructions': cfg['metadata']['contact']['instructions'],
+        'links': [{
+            'type': 'text/html',
+            'href': cfg['metadata']['contact']['url']
+        }],
+        'hoursOfService': cfg['metadata']['contact']['hours'],
+        'roles': [cfg['metadata']['contact']['role']]
+    }
+
+    return contact
+
+
 def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict:
     """
     Generates an OpenAPI 3.0 Document
@@ -167,11 +213,7 @@ def get_oas_30(cfg: dict, fail_on_invalid_collection: bool = True) -> dict:
         'x-keywords': l10n.translate(cfg['metadata']['identification']['keywords'], locale_),  # noqa
         'termsOfService':
             cfg['metadata']['identification']['terms_of_service'],
-        'contact': {
-            'name': cfg['metadata']['provider']['name'],
-            'url': cfg['metadata']['provider']['url'],
-            'email': cfg['metadata']['contact']['email']
-        },
+        'contact': gen_contact(cfg),
         'license': {
             'name': cfg['metadata']['license']['name'],
             'url': cfg['metadata']['license']['url']

pygeoapi/process/manager/dummy.py

@@ -116,7 +116,8 @@ def execute_process(
         self._send_in_progress_notification(subscriber)
         processor = self.get_processor(process_id)
         try:
-            jfmt, outputs = processor.execute(data_dict)
+            jfmt, outputs = processor.execute(
+                data_dict, outputs=requested_outputs)
             current_status = JobStatus.successful
             self._send_success_notification(subscriber, outputs)
         except Exception as err:

pygeoapi/process/manager/mongodb_.py

@@ -32,6 +32,7 @@
 
 from pymongo import MongoClient
 
+from pygeoapi.api import FORMAT_TYPES, F_JSON, F_JSONLD
 from pygeoapi.process.base import (
     JobNotFoundError,
     JobResultNotFoundError,
@@ -151,8 +152,16 @@ def get_job_result(self, job_id):
             if entry["status"] != "successful":
                 LOGGER.info("JOBMANAGER - job not finished or failed")
                 return (None,)
-            with open(entry["location"], "r") as file:
-                data = json.load(file)
+            if not entry["location"]:
+                LOGGER.warning(f"job {job_id!r} -  unknown result location")
+                raise JobResultNotFoundError()
+            if entry["mimetype"] in (None, FORMAT_TYPES[F_JSON],
+                                     FORMAT_TYPES[F_JSONLD]):
+                with open(entry["location"], "r") as file:
+                    data = json.load(file)
+            else:
+                with open(entry["location"], "rb") as file:
+                    data = file.read()
             LOGGER.info("JOBMANAGER - MongoDB job result queried")
             return entry["mimetype"], data
         except Exception as err:

pygeoapi/process/manager/postgresql.py

@@ -49,6 +49,7 @@
 from sqlalchemy.engine import make_url
 from sqlalchemy.orm import Session
 
+from pygeoapi.api import FORMAT_TYPES, F_JSON, F_JSONLD
 from pygeoapi.process.base import (
     JobNotFoundError,
     JobResultNotFoundError,
@@ -292,8 +293,13 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]:
         else:
             try:
                 location = Path(location)
-                with location.open(encoding='utf-8') as fh:
-                    result = json.load(fh)
+                if mimetype in (None, FORMAT_TYPES[F_JSON],
+                                FORMAT_TYPES[F_JSONLD]):
+                    with location.open('r', encoding='utf-8') as fh:
+                        result = json.load(fh)
+                else:
+                    with location.open('rb') as fh:
+                        result = fh.read()
             except (TypeError, FileNotFoundError, json.JSONDecodeError):
                 raise JobResultNotFoundError()
             else:

pygeoapi/process/manager/tinydb_.py

@@ -37,6 +37,7 @@
 import tinydb
 from filelock import FileLock
 
+from pygeoapi.api import FORMAT_TYPES, F_JSON, F_JSONLD
 from pygeoapi.process.base import (
     JobNotFoundError,
     JobResultNotFoundError,
@@ -211,8 +212,13 @@ def get_job_result(self, job_id: str) -> Tuple[str, Any]:
         else:
             try:
                 location = Path(location)
-                with location.open('r', encoding='utf-8') as filehandler:
-                    result = json.load(filehandler)
+                if mimetype in (None, FORMAT_TYPES[F_JSON],
+                                FORMAT_TYPES[F_JSONLD]):
+                    with location.open('r', encoding='utf-8') as filehandler:
+                        result = json.load(filehandler)
+                else:
+                    with location.open('rb') as filehandler:
+                        result = filehandler.read()
             except (TypeError, FileNotFoundError, json.JSONDecodeError):
                 raise JobResultNotFoundError()
             else:

pygeoapi/provider/esri.py

@@ -62,8 +62,9 @@ def __init__(self, provider_def):
         self.crs = provider_def.get('crs', '4326')
         self.username = provider_def.get('username')
         self.password = provider_def.get('password')
+        self.token_url = provider_def.get('token_service', ARCGIS_URL)
+        self.token_referer = provider_def.get('referer', GENERATE_TOKEN_URL)
         self.token = None
-
         self.session = Session()
 
         self.login()
@@ -194,16 +195,15 @@ def login(self):
                 msg = 'Missing ESRI login information, not setting token'
                 LOGGER.debug(msg)
                 return
-
             params = {
                 'f': 'pjson',
                 'username': self.username,
                 'password': self.password,
-                'referer': ARCGIS_URL
+                'referer': self.token_referer
             }
 
             LOGGER.debug('Logging in')
-            with self.session.post(GENERATE_TOKEN_URL, data=params) as r:
+            with self.session.post(self.token_url, data=params) as r:
                 self.token = r.json().get('token')
                 # https://enterprise.arcgis.com/en/server/latest/administer/windows/about-arcgis-tokens.htm
                 self.session.headers.update({

pygeoapi/provider/postgresql.py

@@ -62,7 +62,8 @@
 import shapely
 from sqlalchemy import create_engine, MetaData, PrimaryKeyConstraint, asc, desc
 from sqlalchemy.engine import URL
-from sqlalchemy.exc import InvalidRequestError, OperationalError
+from sqlalchemy.exc import ConstraintColumnNotFoundError, \
+    InvalidRequestError, OperationalError
 from sqlalchemy.ext.automap import automap_base
 from sqlalchemy.orm import Session, load_only
 from sqlalchemy.sql.expression import and_
@@ -515,7 +516,7 @@ def get_table_model(
     sqlalchemy_table_def = metadata.tables[f'{schema}.{table_name}']
     try:
         sqlalchemy_table_def.append_constraint(PrimaryKeyConstraint(id_field))
-    except KeyError:
+    except (ConstraintColumnNotFoundError, KeyError):
         raise ProviderQueryError(
             f"No such id_field column ({id_field}) on {schema}.{table_name}.")