add enhanced support for limits (RFC5) (#1856)

.github/workflows/containers.yml

@@ -25,7 +25,7 @@ jobs:
       contents: read
     steps:
       - name: Check out the repo
-        uses: actions/checkout@v3
+        uses: actions/checkout@master
 
       - name: Set up QEMU
         uses: docker/[email protected]

.github/workflows/docs.yml

@@ -23,8 +23,8 @@ jobs:
         include:
           - python-version: '3.10'
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
+    - uses: actions/checkout@master
+    - uses: actions/setup-python@v5
       name: Setup Python ${{ matrix.python-version }}
       with:
         python-version: ${{ matrix.python-version }}

.github/workflows/flake8.yml

@@ -7,8 +7,8 @@ jobs:
   flake8_py3:
     runs-on: ubuntu-22.04
     steps:
-      - uses: actions/checkout@v3
-      - uses: actions/setup-python@v3
+      - uses: actions/checkout@master
+      - uses: actions/setup-python@v5
         name: setup Python
         with:
           python-version: '3.10'

docker/default.config.yml

@@ -46,7 +46,9 @@ server:
     cors: true
     pretty_print: true
     admin: ${PYGEOAPI_SERVER_ADMIN:-false}
-    limit: 10
+    limits:
+        defaultitems: 10
+        maxitems: 50
     # templates: /path/to/templates
     map:
         url: https://tile.openstreetmap.org/{z}/{x}/{y}.png

docs/source/configuration.rst

@@ -49,7 +49,13 @@ For more information related to API design rules (the ``api_rules`` property in
     gzip: false # default server config to gzip/compress responses to requests with gzip in the Accept-Encoding header
     cors: true  # boolean on whether server should support CORS
     pretty_print: true  # whether JSON responses should be pretty-printed
-    limit: 10  # server limit on number of items to return
+
+    limits:  # server limits on number of items to return.  This property can also be defined at the resource level to override global server settings
+        defaultitems: 10
+        maxitems: 100
+        maxdistance: [25, 25]
+        on_exceed: throttle  # throttle or error (default=throttle)
+
     admin: false  # whether to enable the Admin API
 
     # optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates

pygeoapi/api/__init__.py

@@ -40,7 +40,7 @@
 Returns content from plugins and sets responses.
 """
 
-from collections import OrderedDict
+from collections import ChainMap, OrderedDict
 from copy import deepcopy
 from datetime import datetime
 from functools import partial
@@ -1590,3 +1590,43 @@ def validate_subset(value: str) -> dict:
         subsets[subset_name] = list(map(get_typed_value, values))
 
     return subsets
+
+
+def evaluate_limit(requested: Union[None, int], server_limits: dict,
+                   collection_limits: dict) -> int:
+    """
+    Helper function to evaluate limit parameter
+
+    :param requested: the limit requested by the client
+    :param server_limits: `dict` of server limits
+    :param collection_limits: `dict` of collection limits
+
+    :returns: `int` of evaluated limit
+    """
+
+    effective_limits = ChainMap(collection_limits, server_limits)
+
+    default = effective_limits.get('defaultitems', 10)
+    max_ = effective_limits.get('maxitems', 10)
+    on_exceed = effective_limits.get('on_exceed', 'throttle')
+
+    LOGGER.debug(f'Requested limit: {requested}')
+    LOGGER.debug(f'Default limit: {default}')
+    LOGGER.debug(f'Maximum limit: {max_}')
+    LOGGER.debug(f'On exceed: {on_exceed}')
+
+    if requested is None:
+        LOGGER.debug('no limit requested; returning default')
+        return default
+
+    requested2 = get_typed_value(requested)
+    if not isinstance(requested2, int):
+        raise ValueError('limit value should be an integer')
+
+    if requested2 <= 0:
+        raise ValueError('limit value should be strictly positive')
+    elif requested2 > max_ and on_exceed == 'error':
+        raise RuntimeError('Limit exceeded; throwing errror')
+    else:
+        LOGGER.debug('limit requested')
+        return min(requested2, max_)

pygeoapi/api/environmental_data_retrieval.py

@@ -47,6 +47,7 @@
 from shapely.wkt import loads as shapely_loads
 
 from pygeoapi import l10n
+from pygeoapi.api import evaluate_limit
 from pygeoapi.plugin import load_plugin, PLUGINS
 from pygeoapi.provider.base import ProviderGenericError
 from pygeoapi.util import (
@@ -175,6 +176,16 @@ def get_collection_edr_query(api: API, request: APIRequest,
             HTTPStatus.BAD_REQUEST, headers, request.format,
             'InvalidParameterValue', msg)
 
+    LOGGER.debug('Processing limit parameter')
+    try:
+        limit = evaluate_limit(request.params.get('limit'),
+                               api.config['server'].get('limits', {}),
+                               collections[dataset].get('limits', {}))
+    except ValueError as err:
+        return api.get_exception(
+            HTTPStatus.BAD_REQUEST, headers, request.format,
+            'InvalidParameterValue', str(err))
+
     query_args = dict(
         query_type=query_type,
         instance=instance,
@@ -186,8 +197,8 @@ def get_collection_edr_query(api: API, request: APIRequest,
         bbox=bbox,
         within=within,
         within_units=within_units,
-        limit=int(api.config['server']['limit']),
-        location_id=location_id,
+        limit=limit,
+        location_id=location_id
     )
 
     try:

pygeoapi/api/itemtypes.py

@@ -48,6 +48,7 @@
 from pyproj.exceptions import CRSError
 
 from pygeoapi import l10n
+from pygeoapi.api import evaluate_limit
 from pygeoapi.formatter.base import FormatterSerializationError
 from pygeoapi.linked_data import geojson2jsonld
 from pygeoapi.plugin import load_plugin, PLUGINS
@@ -240,33 +241,24 @@ def get_collection_items(
             return api.get_exception(
                 HTTPStatus.BAD_REQUEST, headers, request.format,
                 'InvalidParameterValue', msg)
-    except TypeError as err:
-        LOGGER.warning(err)
-        offset = 0
     except ValueError:
         msg = 'offset value should be an integer'
         return api.get_exception(
             HTTPStatus.BAD_REQUEST, headers, request.format,
             'InvalidParameterValue', msg)
+    except TypeError as err:
+        LOGGER.warning(err)
+        offset = 0
 
     LOGGER.debug('Processing limit parameter')
     try:
-        limit = int(request.params.get('limit'))
-        # TODO: We should do more validation, against the min and max
-        #       allowed by the server configuration
-        if limit <= 0:
-            msg = 'limit value should be strictly positive'
-            return api.get_exception(
-                HTTPStatus.BAD_REQUEST, headers, request.format,
-                'InvalidParameterValue', msg)
-    except TypeError as err:
-        LOGGER.warning(err)
-        limit = int(api.config['server']['limit'])
-    except ValueError:
-        msg = 'limit value should be an integer'
+        limit = evaluate_limit(request.params.get('limit'),
+                               api.config['server'].get('limits', {}),
+                               collections[dataset].get('limits', {}))
+    except ValueError as err:
         return api.get_exception(
             HTTPStatus.BAD_REQUEST, headers, request.format,
-            'InvalidParameterValue', msg)
+            'InvalidParameterValue', str(err))
 
     resulttype = request.params.get('resulttype') or 'results'
 
@@ -693,22 +685,13 @@ def post_collection_items(
 
     LOGGER.debug('Processing limit parameter')
     try:
-        limit = int(request.params.get('limit'))
-        # TODO: We should do more validation, against the min and max
-        # allowed by the server configuration
-        if limit <= 0:
-            msg = 'limit value should be strictly positive'
-            return api.get_exception(
-                HTTPStatus.BAD_REQUEST, headers, request.format,
-                'InvalidParameterValue', msg)
-    except TypeError as err:
-        LOGGER.warning(err)
-        limit = int(api.config['server']['limit'])
-    except ValueError:
-        msg = 'limit value should be an integer'
+        limit = evaluate_limit(request.params.get('limit'),
+                               api.config['server'].get('limits', {}),
+                               collections[dataset].get('limits', {}))
+    except ValueError as err:
         return api.get_exception(
             HTTPStatus.BAD_REQUEST, headers, request.format,
-            'InvalidParameterValue', msg)
+            'InvalidParameterValue', str(err))
 
     resulttype = request.params.get('resulttype') or 'results'
 

pygeoapi/api/processes.py

@@ -49,6 +49,7 @@
 import urllib.parse
 
 from pygeoapi import l10n
+from pygeoapi.api import evaluate_limit
 from pygeoapi.util import (
     json_serial, render_j2_template, JobStatus, RequestedProcessExecutionMode,
     to_json, DATETIME_FORMAT)
@@ -101,23 +102,14 @@ def describe_processes(api: API, request: APIRequest,
         else:
             LOGGER.debug('Processing limit parameter')
             try:
-                limit = int(request.params.get('limit'))
-
-                if limit <= 0:
-                    msg = 'limit value should be strictly positive'
-                    return api.get_exception(
-                        HTTPStatus.BAD_REQUEST, headers, request.format,
-                        'InvalidParameterValue', msg)
-
+                limit = evaluate_limit(request.params.get('limit'),
+                                       api.config['server'].get('limits', {}),
+                                       {})
                 relevant_processes = list(api.manager.processes)[:limit]
-            except TypeError:
-                LOGGER.debug('returning all processes')
-                relevant_processes = api.manager.processes.keys()
-            except ValueError:
-                msg = 'limit value should be an integer'
+            except ValueError as err:
                 return api.get_exception(
                     HTTPStatus.BAD_REQUEST, headers, request.format,
-                    'InvalidParameterValue', msg)
+                    'InvalidParameterValue', str(err))
 
         for key in relevant_processes:
             p = api.manager.get_processor(key)
@@ -244,21 +236,13 @@ def get_jobs(api: API, request: APIRequest,
                                            **api.api_headers)
     LOGGER.debug('Processing limit parameter')
     try:
-        limit = int(request.params.get('limit'))
-
-        if limit <= 0:
-            msg = 'limit value should be strictly positive'
-            return api.get_exception(
-                HTTPStatus.BAD_REQUEST, headers, request.format,
-                'InvalidParameterValue', msg)
-    except TypeError:
-        limit = int(api.config['server']['limit'])
-        LOGGER.debug('returning all jobs')
-    except ValueError:
-        msg = 'limit value should be an integer'
+        limit = evaluate_limit(request.params.get('limit'),
+                               api.config['server'].get('limits', {}),
+                               {})
+    except ValueError as err:
         return api.get_exception(
             HTTPStatus.BAD_REQUEST, headers, request.format,
-            'InvalidParameterValue', msg)
+            'InvalidParameterValue', str(err))
 
     LOGGER.debug('Processing offset parameter')
     try:

pygeoapi/schemas/config/pygeoapi-config-0.x.yml

@@ -60,8 +60,36 @@ properties:
                 default: false
             limit:
                 type: integer
-                description: server limit on number of items to return
                 default: 10
+                description: "limit of items to return.  DEPRECATED: use limits instead"
+            limits: &x-limits
+                type: object
+                description: server level limits on number of items to return
+                properties:
+                    maxitems:
+                        type: integer
+                        description: maximum limit of items to return for feature and record providers
+                        minimum: 1
+                        default: 10
+                    defaultitems:
+                        type: integer
+                        description: default limit of items to return for feature and record providers
+                        minimum: 1
+                        default: 10
+                    maxdistance:
+                        type: array
+                        description: maximum distance in x and y for all data providers
+                        minItems: 2
+                        maxItems: 2
+                        items:
+                            type: number
+                    on_exceed:
+                        type: string
+                        description: how to handle limit exceeding
+                        default: throttle
+                        enum:
+                            - error
+                            - throttle
             templates:
                 type: object
                 description: optional configuration to specify a different set of templates for HTML pages. Recommend using absolute paths. Omit this to use the default provided templates
@@ -417,6 +445,9 @@ properties:
                                               default: 'http://www.opengis.net/def/uom/ISO-8601/0/Gregorian'
                               required:
                                   - spatial
+                          limits:
+                              <<: *x-limits
+                              description: collection level limits on number of items to return
                           providers:
                               type: array
                               description: required connection information

tests/api/test_api.py

@@ -40,7 +40,7 @@
 
 from pygeoapi.api import (
     API, APIRequest, FORMAT_TYPES, F_HTML, F_JSON, F_JSONLD, F_GZIP,
-    __version__, validate_bbox, validate_datetime,
+    __version__, validate_bbox, validate_datetime, evaluate_limit,
     validate_subset, landing_page, openapi_, conformance, describe_collections,
     get_collection_schema,
 )
@@ -848,3 +848,42 @@ def test_get_exception(config, api_):
     assert content['description'] == 'oops'
 
     d = api_.get_exception(500, {}, 'html', 'NoApplicableCode', 'oops')
+
+
+def test_evaluate_limit():
+    collection = {}
+    server = {}
+
+    with pytest.raises(ValueError):
+        assert evaluate_limit('1.1', server, collection) == 10
+
+    with pytest.raises(ValueError):
+        assert evaluate_limit('-12', server, collection) == 10
+
+    assert evaluate_limit('1', server, collection) == 1
+
+    collection = {}
+    server = {'defaultitems': 2, 'maxitems': 3}
+
+    assert evaluate_limit(None, server, collection) == 2
+    assert evaluate_limit('1', server, collection) == 1
+    assert evaluate_limit('4', server, collection) == 3
+
+    collection = {'defaultitems': 10, 'maxitems': 50}
+    server = {'defaultitems': 100, 'maxitems': 1000}
+
+    assert evaluate_limit(None, server, collection) == 10
+    assert evaluate_limit('40', server, collection) == 40
+    assert evaluate_limit('60', server, collection) == 50
+
+    collection = {}
+    server = {'defaultitems': 2, 'maxitems': 3, 'on_exceed': 'error'}
+
+    with pytest.raises(RuntimeError):
+        assert evaluate_limit('40', server, collection) == 40
+
+    collection = {'defaultitems': 10}
+    server = {'defaultitems': 2, 'maxitems': 3}
+
+    assert evaluate_limit(None, server, collection) == 10
+    assert evaluate_limit('40', server, collection) == 3