From 77df58839efea9e0ce9ecc96926c96ab60f53434 Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Mon, 29 Sep 2025 18:43:40 +0200 Subject: [PATCH 1/5] fix: the default and max allowed returned values for collections and items Ensure collections and items endpoints use their respective default and maximum limit values. --- CHANGELOG.md | 3 + README.md | 6 +- stac_fastapi/core/stac_fastapi/core/core.py | 76 ++++++++------- .../core/stac_fastapi/core/utilities.py | 10 +- .../elasticsearch/database_logic.py | 4 +- .../stac_fastapi/opensearch/database_logic.py | 4 +- stac_fastapi/tests/api/test_api.py | 96 ++++++++++++------- .../tests/api/test_api_item_collection.py | 59 ++++++++++-- 8 files changed, 172 insertions(+), 86 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 94dbe9a97..2834a32f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +- Removed ENV_MAX_LIMIT environment variable; maximum limits are now handled by the default global limit environment variable. [#482](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/482) +- Changed the default and maximum pagination limits for collections/items endpoints. [#482](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/482) + ### Fixed [v6.5.0] - 2025-09-29 diff --git a/README.md b/README.md index a8e2a2973..5694a2133 100644 --- a/README.md +++ b/README.md @@ -299,9 +299,11 @@ You can customize additional settings in your `.env` file: | `ENABLE_COLLECTIONS_SEARCH` | Enable collection search extensions (sort, fields, free text search, structured filtering, and datetime filtering) on the core `/collections` endpoint. | `true` | Optional | | `ENABLE_COLLECTIONS_SEARCH_ROUTE` | Enable the custom `/collections-search` endpoint (both GET and POST methods). When disabled, the custom endpoint will not be available, but collection search extensions will still be available on the core `/collections` endpoint if `ENABLE_COLLECTIONS_SEARCH` is true. | `false` | Optional | | `ENABLE_TRANSACTIONS_EXTENSIONS` | Enables or disables the Transactions and Bulk Transactions API extensions. This is useful for deployments where mutating the catalog via the API should be prevented. If set to `true`, the POST `/collections` route for search will be unavailable in the API. | `true` | Optional | -| `STAC_ITEM_LIMIT` | Sets the environment variable for result limiting to SFEOS for the number of returned items and STAC collections. | `10` | Optional | +| `STAC_GLOBAL_COLLECTION_MAX_LIMIT` | Configures the maximum number of STAC collections that can be returned in a single search request. | N/A | Optional | +| `STAC_DEFAULT_COLLECTION_LIMIT` | Configures the default number of STAC collections returned when no limit parameter is specified in the request. | `300` | Optional | +| `STAC_GLOBAL_ITEM_MAX_LIMIT` | Configures the maximum number of STAC items that can be returned in a single search request. | N/A | Optional | +| `STAC_DEFAULT_ITEM_LIMIT` | Configures the default number of STAC items returned when no limit parameter is specified in the request. | `10` | Optional | | `STAC_INDEX_ASSETS` | Controls if Assets are indexed when added to Elasticsearch/Opensearch. This allows asset fields to be included in search queries. | `false` | Optional | -| `ENV_MAX_LIMIT` | Configures the environment variable in SFEOS to override the default `MAX_LIMIT`, which controls the limit parameter for returned items and STAC collections. | `10,000` | Optional | | `USE_DATETIME` | Configures the datetime search behavior in SFEOS. When enabled, searches both datetime field and falls back to start_datetime/end_datetime range for items with null datetime. When disabled, searches only by start_datetime/end_datetime range. | `true` | Optional | > [!NOTE] diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index ac2f228d2..726ca4e3a 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -269,34 +269,31 @@ async def all_collections( request = kwargs["request"] base_url = str(request.base_url) - # Get the global limit from environment variable - global_limit = None - env_limit = os.getenv("STAC_ITEM_LIMIT") - if env_limit: - try: - global_limit = int(env_limit) - except ValueError: - # Handle invalid integer in environment variable - pass - - # Apply global limit if it exists - if global_limit is not None: - # If a limit was provided, use the smaller of the two - if limit is not None: - limit = min(limit, global_limit) - else: - limit = global_limit + global_max_limit = ( + int(os.getenv("STAC_GLOBAL_COLLECTION_MAX_LIMIT")) + if os.getenv("STAC_GLOBAL_COLLECTION_MAX_LIMIT") + else None + ) + default_limit = int(os.getenv("STAC_DEFAULT_COLLECTION_LIMIT", 300)) + query_limit = request.query_params.get("limit") + + body_limit = None + try: + if request.method == "POST" and request.body(): + body_data = await request.json() + body_limit = body_data.get("limit") + except Exception: + pass + + if body_limit is not None: + limit = int(body_limit) + elif query_limit: + limit = int(query_limit) else: - # No global limit, use provided limit or default - if limit is None: - query_limit = request.query_params.get("limit") - if query_limit: - try: - limit = int(query_limit) - except ValueError: - limit = 10 - else: - limit = 10 + limit = default_limit + + if global_max_limit is not None: + limit = min(limit, global_max_limit) token = request.query_params.get("token") @@ -562,7 +559,7 @@ async def item_collection( request (Request): FastAPI Request object. bbox (Optional[BBox]): Optional bounding box filter. datetime (Optional[str]): Optional datetime or interval filter. - limit (Optional[int]): Optional page size. Defaults to env ``STAC_ITEM_LIMIT`` when unset. + limit (Optional[int]): Optional page size. Defaults to env `STAC_DEFAULT_ITEM_LIMIT` when unset. sortby (Optional[str]): Optional sort specification. Accepts repeated values like ``sortby=-properties.datetime`` or ``sortby=+id``. Bare fields (e.g. ``sortby=id``) imply ascending order. @@ -653,15 +650,12 @@ async def get_search( q (Optional[List[str]]): Free text query to filter the results. intersects (Optional[str]): GeoJSON geometry to search in. kwargs: Additional parameters to be passed to the API. - Returns: ItemCollection: Collection of `Item` objects representing the search results. Raises: HTTPException: If any error occurs while searching the catalog. """ - limit = int(request.query_params.get("limit", os.getenv("STAC_ITEM_LIMIT", 10))) - base_args = { "collections": collections, "ids": ids, @@ -736,6 +730,25 @@ async def post_search( Raises: HTTPException: If there is an error with the cql2_json filter. """ + global_max_limit = ( + int(os.getenv("STAC_GLOBAL_ITEM_MAX_LIMIT")) + if os.getenv("STAC_GLOBAL_ITEM_MAX_LIMIT") + else None + ) + default_limit = int(os.getenv("STAC_DEFAULT_ITEM_LIMIT", 10)) + + requested_limit = getattr(search_request, "limit", None) + + if requested_limit is None: + limit = default_limit + else: + limit = requested_limit + + if global_max_limit: + limit = min(limit, global_max_limit) + + search_request.limit = limit + base_url = str(request.base_url) search = self.database.make_search() @@ -812,7 +825,6 @@ async def post_search( if hasattr(search_request, "sortby") and getattr(search_request, "sortby"): sort = self.database.populate_sort(getattr(search_request, "sortby")) - limit = 10 if search_request.limit: limit = search_request.limit diff --git a/stac_fastapi/core/stac_fastapi/core/utilities.py b/stac_fastapi/core/stac_fastapi/core/utilities.py index c54348af0..de6536567 100644 --- a/stac_fastapi/core/stac_fastapi/core/utilities.py +++ b/stac_fastapi/core/stac_fastapi/core/utilities.py @@ -10,15 +10,7 @@ from stac_fastapi.types.stac import Item - -def get_max_limit(): - """ - Retrieve a MAX_LIMIT value from an environment variable. - - Returns: - int: The int value parsed from the environment variable. - """ - return int(os.getenv("ENV_MAX_LIMIT", 10000)) +MAX_LIMIT = 10000 def get_bool_env(name: str, default: Union[bool, str] = False) -> bool: diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index f4f33cb97..faa5175ff 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -17,7 +17,7 @@ from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer -from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit +from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, get_bool_env from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings from stac_fastapi.elasticsearch.config import ( ElasticsearchSettings as SyncElasticsearchSettings, @@ -818,7 +818,7 @@ async def execute_search( index_param = ITEM_INDICES query = add_collections_to_body(collection_ids, query) - max_result_window = get_max_limit() + max_result_window = MAX_LIMIT size_limit = min(limit + 1, max_result_window) diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 8791390bb..02cb1e4b1 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -17,7 +17,7 @@ from stac_fastapi.core.base_database_logic import BaseDatabaseLogic from stac_fastapi.core.serializers import CollectionSerializer, ItemSerializer -from stac_fastapi.core.utilities import bbox2polygon, get_bool_env, get_max_limit +from stac_fastapi.core.utilities import MAX_LIMIT, bbox2polygon, get_bool_env from stac_fastapi.extensions.core.transaction.request import ( PartialCollection, PartialItem, @@ -810,7 +810,7 @@ async def execute_search( search_body["sort"] = sort if sort else DEFAULT_SORT - max_result_window = get_max_limit() + max_result_window = MAX_LIMIT size_limit = min(limit + 1, max_result_window) diff --git a/stac_fastapi/tests/api/test_api.py b/stac_fastapi/tests/api/test_api.py index 6fdc2fb60..0b0733825 100644 --- a/stac_fastapi/tests/api/test_api.py +++ b/stac_fastapi/tests/api/test_api.py @@ -1475,70 +1475,102 @@ def create_items(date_prefix: str, start_day: int, count: int) -> dict: @pytest.mark.asyncio -async def test_collections_limit_env_variable(app_client, txn_client, load_test_data): - limit = "5" - os.environ["STAC_ITEM_LIMIT"] = limit - item = load_test_data("test_collection.json") +async def test_global_collection_max_limit_set(app_client, txn_client, load_test_data): + """Test with global collection max limit set, expect cap the limit""" + os.environ["STAC_GLOBAL_COLLECTION_MAX_LIMIT"] = "5" for i in range(10): - test_collection = item.copy() - test_collection["id"] = f"test-collection-env-{i}" - test_collection["title"] = f"Test Collection Env {i}" + test_collection = load_test_data("test_collection.json") + test_collection_id = f"test-collection-global-{i}" + test_collection["id"] = test_collection_id + await create_collection(txn_client, test_collection) + + resp = await app_client.get("/collections?limit=10") + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["collections"]) == 5 + + del os.environ["STAC_GLOBAL_COLLECTION_MAX_LIMIT"] + + +@pytest.mark.asyncio +async def test_default_collection_limit(app_client, txn_client, load_test_data): + """Test default collection limit set, should use default when no limit provided""" + os.environ["STAC_DEFAULT_COLLECTION_LIMIT"] = "5" + + for i in range(10): + test_collection = load_test_data("test_collection.json") + test_collection_id = f"test-collection-default-{i}" + test_collection["id"] = test_collection_id await create_collection(txn_client, test_collection) resp = await app_client.get("/collections") assert resp.status_code == 200 resp_json = resp.json() - assert int(limit) == len(resp_json["collections"]) + assert len(resp_json["collections"]) == 5 + + del os.environ["STAC_DEFAULT_COLLECTION_LIMIT"] @pytest.mark.asyncio -async def test_search_collection_limit_env_variable( - app_client, txn_client, load_test_data -): - limit = "5" - os.environ["STAC_ITEM_LIMIT"] = limit +async def test_no_global_item_max_limit_set(app_client, txn_client, load_test_data): + """Test with no global max limit set for items""" + + if "STAC_GLOBAL_ITEM_MAX_LIMIT" in os.environ: + del os.environ["STAC_GLOBAL_ITEM_MAX_LIMIT"] test_collection = load_test_data("test_collection.json") - test_collection_id = "test-collection-search-limit" + test_collection_id = "test-collection-no-global-limit" test_collection["id"] = test_collection_id await create_collection(txn_client, test_collection) item = load_test_data("test_item.json") item["collection"] = test_collection_id - for i in range(10): + for i in range(20): test_item = item.copy() - test_item["id"] = f"test-item-search-{i}" + test_item["id"] = f"test-item-{i}" await create_item(txn_client, test_item) - resp = await app_client.get("/search", params={"collections": [test_collection_id]}) + resp = await app_client.get(f"/collections/{test_collection_id}/items?limit=20") + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 20 + + resp = await app_client.get(f"/search?collections={test_collection_id}&limit=20") assert resp.status_code == 200 resp_json = resp.json() - assert int(limit) == len(resp_json["features"]) + assert len(resp_json["features"]) == 20 + resp = await app_client.post( + "/search", json={"collections": [test_collection_id], "limit": 20} + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 20 -async def test_search_max_item_limit( - app_client, load_test_data, txn_client, monkeypatch -): - limit = "10" - monkeypatch.setenv("ENV_MAX_LIMIT", limit) - test_collection = load_test_data("test_collection.json") - await create_collection(txn_client, test_collection) +@pytest.mark.asyncio +async def test_no_global_collection_max_limit_set( + app_client, txn_client, load_test_data +): + """Test with no global max limit set for collections""" - item = load_test_data("test_item.json") + if "STAC_GLOBAL_COLLECTION_MAX_LIMIT" in os.environ: + del os.environ["STAC_GLOBAL_COLLECTION_MAX_LIMIT"] + test_collections = [] for i in range(20): - test_item = item.copy() - test_item["id"] = f"test-item-collection-{i}" - await create_item(txn_client, test_item) - - resp = await app_client.get("/search", params={"limit": 20}) + test_collection = load_test_data("test_collection.json") + test_collection_id = f"test-collection-no-global-limit-{i}" + test_collection["id"] = test_collection_id + await create_collection(txn_client, test_collection) + test_collections.append(test_collection_id) + resp = await app_client.get("/collections?limit=20") assert resp.status_code == 200 resp_json = resp.json() - assert int(limit) == len(resp_json["features"]) + assert len(resp_json["collections"]) == 20 @pytest.mark.asyncio diff --git a/stac_fastapi/tests/api/test_api_item_collection.py b/stac_fastapi/tests/api/test_api_item_collection.py index 2b1aa8e79..f0d07a744 100644 --- a/stac_fastapi/tests/api/test_api_item_collection.py +++ b/stac_fastapi/tests/api/test_api_item_collection.py @@ -10,29 +10,74 @@ @pytest.mark.asyncio -async def test_item_collection_limit_env_variable( +async def test_global_item_max_limit_set(app_client, txn_client, load_test_data): + """Test with global max limit set for items, expect cap the ?limit parameter""" + os.environ["STAC_GLOBAL_ITEM_MAX_LIMIT"] = "5" + + test_collection = load_test_data("test_collection.json") + test_collection_id = "test-collection-for-items" + test_collection["id"] = test_collection_id + await create_collection(txn_client, test_collection) + + item = load_test_data("test_item.json") + item["collection"] = test_collection_id + + for i in range(10): + test_item = item.copy() + test_item["id"] = f"test-item-{i}" + await create_item(txn_client, test_item) + + resp = await app_client.get(f"/collections/{test_collection_id}/items?limit=10") + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 5 + + resp = await app_client.get(f"/search?collections={test_collection_id}&limit=10") + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 5 + + del os.environ["STAC_GLOBAL_ITEM_MAX_LIMIT"] + + +@pytest.mark.asyncio +async def test_default_item_limit_without_limit_parameter_set( app_client, txn_client, load_test_data ): - limit = "5" - os.environ["STAC_ITEM_LIMIT"] = limit + """Test default item limit set, should use default when no limit provided""" + os.environ["STAC_DEFAULT_ITEM_LIMIT"] = "10" test_collection = load_test_data("test_collection.json") - test_collection_id = "test-collection-items-limit" + test_collection_id = "test-collection-items" test_collection["id"] = test_collection_id await create_collection(txn_client, test_collection) item = load_test_data("test_item.json") item["collection"] = test_collection_id - for i in range(10): + for i in range(15): test_item = item.copy() - test_item["id"] = f"test-item-collection-{i}" + test_item["id"] = f"test-item-{i}" await create_item(txn_client, test_item) resp = await app_client.get(f"/collections/{test_collection_id}/items") assert resp.status_code == 200 resp_json = resp.json() - assert int(limit) == len(resp_json["features"]) + assert len(resp_json["features"]) == 10 + + resp = await app_client.get(f"/search?collections={test_collection_id}") + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 10 + + # Also test POST search to compare + search_body = {"collections": [test_collection_id]} + resp = await app_client.post("/search", json=search_body) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 10 + + del os.environ["STAC_DEFAULT_ITEM_LIMIT"] @pytest.mark.asyncio From 9d584380448808ef3f0ea11cab9895d33be11961 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Tue, 30 Sep 2025 15:39:55 +0800 Subject: [PATCH 2/5] v6.5.1 post collections search fixes (#483) **Related Issue(s):** - None **Description:** Fixed - Issue where token, query param was not being passed to POST collections search logic. - Issue where datetime param was not being passed from POST collections search logic to Elasticsearch. - Collections search tests to ensure both GET /collections and GET/POST /collections-search endpoints are tested. **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 11 +- compose.yml | 2 + stac_fastapi/core/stac_fastapi/core/core.py | 13 +- .../core/extensions/collections_search.py | 4 + .../core/stac_fastapi/core/version.py | 2 +- stac_fastapi/elasticsearch/setup.py | 4 +- .../elasticsearch/database_logic.py | 2 - .../stac_fastapi/elasticsearch/version.py | 2 +- stac_fastapi/opensearch/setup.py | 4 +- .../stac_fastapi/opensearch/database_logic.py | 2 - .../stac_fastapi/opensearch/version.py | 2 +- stac_fastapi/sfeos_helpers/setup.py | 2 +- .../stac_fastapi/sfeos_helpers/version.py | 2 +- .../tests/api/test_api_search_collections.py | 609 +++++++++++------- 14 files changed, 421 insertions(+), 240 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2834a32f9..ed5cdeefc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed +[v6.5.1] - 2025-09-30 + +### Fixed + +- Issue where token, query param was not being passed to POST collections search logic [#483](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/483) +- Issue where datetime param was not being passed from POST collections search logic to Elasticsearch [#483](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/483) +- Collections search tests to ensure both GET /collections and GET/POST /collections-search endpoints are tested [#483](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/483) + [v6.5.0] - 2025-09-29 ### Added @@ -550,7 +558,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.5.0...main +[Unreleased]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.5.1...main +[v6.5.1]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.5.0...v6.5.1 [v6.5.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.4.0...v6.5.0 [v6.4.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.3.0...v6.4.0 [v6.3.0]: https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/compare/v6.2.1...v6.3.0 diff --git a/compose.yml b/compose.yml index 77d64198b..8c83ae127 100644 --- a/compose.yml +++ b/compose.yml @@ -22,6 +22,7 @@ services: - ES_VERIFY_CERTS=false - BACKEND=elasticsearch - DATABASE_REFRESH=true + - ENABLE_COLLECTIONS_SEARCH_ROUTE=true ports: - "8080:8080" volumes: @@ -56,6 +57,7 @@ services: - ES_VERIFY_CERTS=false - BACKEND=opensearch - STAC_FASTAPI_RATE_LIMIT=200/minute + - ENABLE_COLLECTIONS_SEARCH_ROUTE=true ports: - "8082:8082" volumes: diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index 726ca4e3a..c7d372041 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -240,14 +240,16 @@ async def landing_page(self, **kwargs) -> stac_types.LandingPage: async def all_collections( self, - datetime: Optional[str] = None, limit: Optional[int] = None, + datetime: Optional[str] = None, fields: Optional[List[str]] = None, sortby: Optional[Union[str, List[str]]] = None, filter_expr: Optional[str] = None, filter_lang: Optional[str] = None, q: Optional[Union[str, List[str]]] = None, query: Optional[str] = None, + request: Request = None, + token: Optional[str] = None, **kwargs, ) -> stac_types.Collections: """Read all collections from the database. @@ -266,7 +268,6 @@ async def all_collections( Returns: A Collections object containing all the collections in the database and links to various resources. """ - request = kwargs["request"] base_url = str(request.base_url) global_max_limit = ( @@ -295,7 +296,9 @@ async def all_collections( if global_max_limit is not None: limit = min(limit, global_max_limit) - token = request.query_params.get("token") + # Get token from query params only if not already provided (for GET requests) + if token is None: + token = request.query_params.get("token") # Process fields parameter for filtering collection properties includes, excludes = set(), set() @@ -496,6 +499,10 @@ async def post_all_collections( # Pass all parameters from search_request to all_collections return await self.all_collections( limit=search_request.limit if hasattr(search_request, "limit") else None, + datetime=search_request.datetime + if hasattr(search_request, "datetime") + else None, + token=search_request.token if hasattr(search_request, "token") else None, fields=fields, sortby=sortby, filter_expr=search_request.filter diff --git a/stac_fastapi/core/stac_fastapi/core/extensions/collections_search.py b/stac_fastapi/core/stac_fastapi/core/extensions/collections_search.py index 0ddbefeda..d36197d03 100644 --- a/stac_fastapi/core/stac_fastapi/core/extensions/collections_search.py +++ b/stac_fastapi/core/stac_fastapi/core/extensions/collections_search.py @@ -18,6 +18,10 @@ class CollectionsSearchRequest(ExtendedSearch): """Extended search model for collections with free text search support.""" q: Optional[Union[str, List[str]]] = None + token: Optional[str] = None + query: Optional[ + str + ] = None # Legacy query extension (deprecated but still supported) class CollectionsSearchEndpointExtension(ApiExtension): diff --git a/stac_fastapi/core/stac_fastapi/core/version.py b/stac_fastapi/core/stac_fastapi/core/version.py index d5564adc9..019563e86 100644 --- a/stac_fastapi/core/stac_fastapi/core/version.py +++ b/stac_fastapi/core/stac_fastapi/core/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.5.0" +__version__ = "6.5.1" diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index adef9b75a..1751df78f 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==6.5.0", - "sfeos-helpers==6.5.0", + "stac-fastapi-core==6.5.1", + "sfeos-helpers==6.5.1", "elasticsearch[async]~=8.18.0", "uvicorn~=0.23.0", "starlette>=0.35.0,<0.36.0", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index faa5175ff..87a98ceb2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -309,7 +309,6 @@ async def get_all_collections( query_parts.append(search_dict["query"]) except Exception as e: - logger = logging.getLogger(__name__) logger.error(f"Error converting query to Elasticsearch: {e}") # If there's an error, add a query that matches nothing query_parts.append({"bool": {"must_not": {"match_all": {}}}}) @@ -381,7 +380,6 @@ async def get_all_collections( try: matched = count_task.result().get("count") except Exception as e: - logger = logging.getLogger(__name__) logger.error(f"Count task failed: {e}") return collections, next_token, matched diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index d5564adc9..019563e86 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.5.0" +__version__ = "6.5.1" diff --git a/stac_fastapi/opensearch/setup.py b/stac_fastapi/opensearch/setup.py index e301addd6..d7727267f 100644 --- a/stac_fastapi/opensearch/setup.py +++ b/stac_fastapi/opensearch/setup.py @@ -6,8 +6,8 @@ desc = f.read() install_requires = [ - "stac-fastapi-core==6.5.0", - "sfeos-helpers==6.5.0", + "stac-fastapi-core==6.5.1", + "sfeos-helpers==6.5.1", "opensearch-py~=2.8.0", "opensearch-py[async]~=2.8.0", "uvicorn~=0.23.0", diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py index 02cb1e4b1..e993410d3 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py @@ -293,7 +293,6 @@ async def get_all_collections( query_parts.append(search_dict["query"]) except Exception as e: - logger = logging.getLogger(__name__) logger.error(f"Error converting query to OpenSearch: {e}") # If there's an error, add a query that matches nothing query_parts.append({"bool": {"must_not": {"match_all": {}}}}) @@ -365,7 +364,6 @@ async def get_all_collections( try: matched = count_task.result().get("count") except Exception as e: - logger = logging.getLogger(__name__) logger.error(f"Count task failed: {e}") return collections, next_token, matched diff --git a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py index d5564adc9..019563e86 100644 --- a/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py +++ b/stac_fastapi/opensearch/stac_fastapi/opensearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.5.0" +__version__ = "6.5.1" diff --git a/stac_fastapi/sfeos_helpers/setup.py b/stac_fastapi/sfeos_helpers/setup.py index 35306eb60..e7cdd84c6 100644 --- a/stac_fastapi/sfeos_helpers/setup.py +++ b/stac_fastapi/sfeos_helpers/setup.py @@ -6,7 +6,7 @@ desc = f.read() install_requires = [ - "stac-fastapi.core==6.5.0", + "stac-fastapi.core==6.5.1", ] setup( diff --git a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py index d5564adc9..019563e86 100644 --- a/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py +++ b/stac_fastapi/sfeos_helpers/stac_fastapi/sfeos_helpers/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "6.5.0" +__version__ = "6.5.1" diff --git a/stac_fastapi/tests/api/test_api_search_collections.py b/stac_fastapi/tests/api/test_api_search_collections.py index 8f5bed73b..029292ed0 100644 --- a/stac_fastapi/tests/api/test_api_search_collections.py +++ b/stac_fastapi/tests/api/test_api_search_collections.py @@ -8,7 +8,7 @@ @pytest.mark.asyncio async def test_collections_sort_id_asc(app_client, txn_client, ctx): - """Verify GET /collections honors ascending sort on id.""" + """Verify GET /collections, GET /collections-search, and POST /collections-search honor ascending sort on id.""" # Create multiple collections with different ids base_collection = ctx.collection @@ -25,29 +25,48 @@ async def test_collections_sort_id_asc(app_client, txn_client, ctx): await refresh_indices(txn_client) - # Test ascending sort by id - resp = await app_client.get( - "/collections", - params=[("sortby", "+id")], - ) - assert resp.status_code == 200 - resp_json = resp.json() - - # Filter collections to only include the ones we created for this test - test_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + # Define endpoints to test + endpoints = [ + {"method": "GET", "path": "/collections", "params": [("sortby", "+id")]}, + { + "method": "GET", + "path": "/collections-search", + "params": [("sortby", "+id")], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"sortby": [{"field": "id", "direction": "asc"}]}, + }, ] - # Collections should be sorted alphabetically by id - sorted_ids = sorted(collection_ids) - assert len(test_collections) == len(collection_ids) - for i, expected_id in enumerate(sorted_ids): - assert test_collections[i]["id"] == expected_id + for endpoint in endpoints: + # Test ascending sort by id + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) + + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + test_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Collections should be sorted alphabetically by id + sorted_ids = sorted(collection_ids) + assert len(test_collections) == len(collection_ids) + for i, expected_id in enumerate(sorted_ids): + assert test_collections[i]["id"] == expected_id @pytest.mark.asyncio async def test_collections_sort_id_desc(app_client, txn_client, ctx): - """Verify GET /collections honors descending sort on id.""" + """Verify GET /collections, GET /collections-search, and POST /collections-search honor descending sort on id.""" # Create multiple collections with different ids base_collection = ctx.collection @@ -64,24 +83,43 @@ async def test_collections_sort_id_desc(app_client, txn_client, ctx): await refresh_indices(txn_client) - # Test descending sort by id - resp = await app_client.get( - "/collections", - params=[("sortby", "-id")], - ) - assert resp.status_code == 200 - resp_json = resp.json() - - # Filter collections to only include the ones we created for this test - test_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + # Define endpoints to test + endpoints = [ + {"method": "GET", "path": "/collections", "params": [("sortby", "-id")]}, + { + "method": "GET", + "path": "/collections-search", + "params": [("sortby", "-id")], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"sortby": [{"field": "id", "direction": "desc"}]}, + }, ] - # Collections should be sorted in reverse alphabetical order by id - sorted_ids = sorted(collection_ids, reverse=True) - assert len(test_collections) == len(collection_ids) - for i, expected_id in enumerate(sorted_ids): - assert test_collections[i]["id"] == expected_id + for endpoint in endpoints: + # Test descending sort by id + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) + + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + test_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Collections should be sorted in reverse alphabetical order by id + sorted_ids = sorted(collection_ids, reverse=True) + assert len(test_collections) == len(collection_ids) + for i, expected_id in enumerate(sorted_ids): + assert test_collections[i]["id"] == expected_id @pytest.mark.asyncio @@ -245,7 +283,7 @@ async def test_collections_free_text_all_endpoints( @pytest.mark.asyncio async def test_collections_filter_search(app_client, txn_client, ctx): - """Verify GET /collections honors the filter parameter for structured search.""" + """Verify GET /collections, GET /collections-search, and POST /collections-search honor the filter parameter for structured search.""" # Create multiple collections with different content base_collection = ctx.collection @@ -287,52 +325,97 @@ async def test_collections_filter_search(app_client, txn_client, ctx): # Use the ID of the first test collection for the filter test_collection_id = test_collections[0]["id"] + # Test 1: CQL2-JSON format # Create a simple filter for exact ID match using CQL2-JSON filter_expr = {"op": "=", "args": [{"property": "id"}, test_collection_id]} # Convert to JSON string for URL parameter filter_json = json.dumps(filter_expr) - # Use CQL2-JSON format with explicit filter-lang - resp = await app_client.get( - f"/collections?filter={filter_json}&filter-lang=cql2-json", - ) + # Define endpoints to test + endpoints = [ + { + "method": "GET", + "path": "/collections", + "params": [("filter", filter_json), ("filter-lang", "cql2-json")], + }, + { + "method": "GET", + "path": "/collections-search", + "params": [("filter", filter_json), ("filter-lang", "cql2-json")], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"filter": filter_expr, "filter-lang": "cql2-json"}, + }, + ] - assert resp.status_code == 200 - resp_json = resp.json() + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) - # Should find exactly one collection with the specified ID - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id - ] + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() - assert ( - len(found_collections) == 1 - ), f"Expected 1 collection with ID {test_collection_id}, found {len(found_collections)}" - assert found_collections[0]["id"] == test_collection_id + # Should find exactly one collection with the specified ID + found_collections = [ + c for c in resp_json["collections"] if c["id"] == test_collection_id + ] - # Test 2: CQL2-text format with LIKE operator for more advanced filtering - # Use a filter that will match the test collection ID we created - filter_text = f"id LIKE '%{test_collection_id.split('-')[-1]}%'" + assert ( + len(found_collections) == 1 + ), f"Expected 1 collection with ID {test_collection_id}, found {len(found_collections)} for {endpoint['method']} {endpoint['path']}" + assert found_collections[0]["id"] == test_collection_id - resp = await app_client.get( - f"/collections?filter={filter_text}&filter-lang=cql2-text", - ) - assert resp.status_code == 200 - resp_json = resp.json() + # Test 2: CQL2-text format with LIKE operator + filter_text = f"id LIKE '%{test_collection_id.split('-')[-1]}%'" - # Should find the test collection we created - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id + endpoints = [ + { + "method": "GET", + "path": "/collections", + "params": [("filter", filter_text), ("filter-lang", "cql2-text")], + }, + { + "method": "GET", + "path": "/collections-search", + "params": [("filter", filter_text), ("filter-lang", "cql2-text")], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"filter": filter_text, "filter-lang": "cql2-text"}, + }, ] - assert ( - len(found_collections) >= 1 - ), f"Expected at least 1 collection with ID {test_collection_id} using LIKE filter" + + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) + + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() + + # Should find the test collection we created + found_collections = [ + c for c in resp_json["collections"] if c["id"] == test_collection_id + ] + assert ( + len(found_collections) >= 1 + ), f"Expected at least 1 collection with ID {test_collection_id} using LIKE filter for {endpoint['method']} {endpoint['path']}" @pytest.mark.asyncio async def test_collections_query_extension(app_client, txn_client, ctx): - """Verify GET /collections honors the query extension.""" + """Verify GET /collections, GET /collections-search, and POST /collections-search honor the query extension.""" # Create multiple collections with different content base_collection = ctx.collection # Use unique prefixes to avoid conflicts between tests @@ -370,75 +453,100 @@ async def test_collections_query_extension(app_client, txn_client, ctx): await refresh_indices(txn_client) - # Use the exact ID that was created + # Test 1: Query with equal operator sentinel_id = f"{test_prefix}-sentinel" - query = {"id": {"eq": sentinel_id}} - resp = await app_client.get( - "/collections", - params=[("query", json.dumps(query))], - ) - assert resp.status_code == 200 - resp_json = resp.json() - - # Filter collections to only include the ones we created for this test - found_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + endpoints = [ + { + "method": "GET", + "path": "/collections", + "params": [("query", json.dumps(query))], + }, + { + "method": "GET", + "path": "/collections-search", + "params": [("query", json.dumps(query))], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"query": json.dumps(query)}, + }, ] - # Should only find the sentinel collection - assert len(found_collections) == 1 - assert found_collections[0]["id"] == f"{test_prefix}-sentinel" - - # Test query extension with equal operator on ID - query = {"id": {"eq": f"{test_prefix}-sentinel"}} + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) - resp = await app_client.get( - "/collections", - params=[("query", json.dumps(query))], - ) - assert resp.status_code == 200 - resp_json = resp.json() + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() - # Filter collections to only include the ones we created for this test - found_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) - ] - found_ids = [c["id"] for c in found_collections] + # Filter collections to only include the ones we created for this test + found_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] - # Should find landsat and modis collections but not sentinel - assert len(found_collections) == 1 - assert f"{test_prefix}-sentinel" in found_ids - assert f"{test_prefix}-landsat" not in found_ids - assert f"{test_prefix}-modis" not in found_ids + # Should only find the sentinel collection + assert ( + len(found_collections) == 1 + ), f"Expected 1 collection for {endpoint['method']} {endpoint['path']}" + assert found_collections[0]["id"] == sentinel_id - # Test query extension with not-equal operator on ID + # Test 2: Query with not-equal operator query = {"id": {"neq": f"{test_prefix}-sentinel"}} - resp = await app_client.get( - "/collections", - params=[("query", json.dumps(query))], - ) - assert resp.status_code == 200 - resp_json = resp.json() - - # Filter collections to only include the ones we created for this test - found_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + endpoints = [ + { + "method": "GET", + "path": "/collections", + "params": [("query", json.dumps(query))], + }, + { + "method": "GET", + "path": "/collections-search", + "params": [("query", json.dumps(query))], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"query": json.dumps(query)}, + }, ] - found_ids = [c["id"] for c in found_collections] - # Should find landsat and modis collections but not sentinel - assert len(found_collections) == 2 - assert f"{test_prefix}-sentinel" not in found_ids - assert f"{test_prefix}-landsat" in found_ids - assert f"{test_prefix}-modis" in found_ids + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) + + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + found_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + found_ids = [c["id"] for c in found_collections] + + # Should find landsat and modis collections but not sentinel + assert ( + len(found_collections) == 2 + ), f"Expected 2 collections for {endpoint['method']} {endpoint['path']}" + assert f"{test_prefix}-sentinel" not in found_ids + assert f"{test_prefix}-landsat" in found_ids + assert f"{test_prefix}-modis" in found_ids @pytest.mark.asyncio async def test_collections_datetime_filter(app_client, load_test_data, txn_client): - """Test filtering collections by datetime.""" + """Test filtering collections by datetime across all endpoints.""" # Create a test collection with a specific temporal extent base_collection = load_test_data("test_collection.json") @@ -450,66 +558,71 @@ async def test_collections_datetime_filter(app_client, load_test_data, txn_clien await create_collection(txn_client, base_collection) await refresh_indices(txn_client) - # Test 1: Datetime range that overlaps with collection's temporal extent - resp = await app_client.get( - "/collections?datetime=2020-06-01T00:00:00Z/2021-01-01T00:00:00Z" - ) - assert resp.status_code == 200 - resp_json = resp.json() - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id - ] - assert ( - len(found_collections) == 1 - ), f"Expected to find collection {test_collection_id} with overlapping datetime range" - - # Test 2: Datetime range that is completely before collection's temporal extent - resp = await app_client.get( - "/collections?datetime=2019-01-01T00:00:00Z/2019-12-31T23:59:59Z" - ) - assert resp.status_code == 200 - resp_json = resp.json() - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id + # Test scenarios with different datetime ranges + test_scenarios = [ + { + "name": "overlapping range", + "datetime": "2020-06-01T00:00:00Z/2021-01-01T00:00:00Z", + "expected_count": 1, + }, + { + "name": "before range", + "datetime": "2019-01-01T00:00:00Z/2019-12-31T23:59:59Z", + "expected_count": 0, + }, + { + "name": "after range", + "datetime": "2021-01-01T00:00:00Z/2021-12-31T23:59:59Z", + "expected_count": 0, + }, + { + "name": "single datetime within range", + "datetime": "2020-06-15T12:00:00Z", + "expected_count": 1, + }, + { + "name": "open-ended future range", + "datetime": "2020-06-01T00:00:00Z/..", + "expected_count": 1, + }, ] - assert ( - len(found_collections) == 0 - ), f"Expected not to find collection {test_collection_id} with non-overlapping datetime range" - # Test 3: Datetime range that is completely after collection's temporal extent - resp = await app_client.get( - "/collections?datetime=2021-01-01T00:00:00Z/2021-12-31T23:59:59Z" - ) - assert resp.status_code == 200 - resp_json = resp.json() - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id - ] - assert ( - len(found_collections) == 0 - ), f"Expected not to find collection {test_collection_id} with non-overlapping datetime range" + for scenario in test_scenarios: + endpoints = [ + { + "method": "GET", + "path": "/collections", + "params": [("datetime", scenario["datetime"])], + }, + { + "method": "GET", + "path": "/collections-search", + "params": [("datetime", scenario["datetime"])], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"datetime": scenario["datetime"]}, + }, + ] - # Test 4: Single datetime that falls within collection's temporal extent - resp = await app_client.get("/collections?datetime=2020-06-15T12:00:00Z") - assert resp.status_code == 200 - resp_json = resp.json() - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id - ] - assert ( - len(found_collections) == 1 - ), f"Expected to find collection {test_collection_id} with datetime point within range" + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) - # Test 5: Open-ended range (from a specific date to the future) - resp = await app_client.get("/collections?datetime=2020-06-01T00:00:00Z/..") - assert resp.status_code == 200 - resp_json = resp.json() - found_collections = [ - c for c in resp_json["collections"] if c["id"] == test_collection_id - ] - assert ( - len(found_collections) == 1 - ), f"Expected to find collection {test_collection_id} with open-ended future range" + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']} with {scenario['name']}" + resp_json = resp.json() + found_collections = [ + c for c in resp_json["collections"] if c["id"] == test_collection_id + ] + assert len(found_collections) == scenario["expected_count"], ( + f"Expected {scenario['expected_count']} collection(s) for {scenario['name']} " + f"on {endpoint['method']} {endpoint['path']}, found {len(found_collections)}" + ) # Test 6: Open-ended range (from the past to a date within the collection's range) # TODO: This test is currently skipped due to an unresolved issue with open-ended past range queries. @@ -528,7 +641,7 @@ async def test_collections_datetime_filter(app_client, load_test_data, txn_clien @pytest.mark.asyncio async def test_collections_number_matched_returned(app_client, txn_client, ctx): - """Verify GET /collections returns correct numberMatched and numberReturned values.""" + """Verify GET /collections, GET /collections-search, and POST /collections-search return correct numberMatched and numberReturned values.""" # Create multiple collections with different ids base_collection = ctx.collection @@ -545,56 +658,91 @@ async def test_collections_number_matched_returned(app_client, txn_client, ctx): await refresh_indices(txn_client) - # Test with limit=5 - resp = await app_client.get( - "/collections", - params=[("limit", "5")], - ) - assert resp.status_code == 200 - resp_json = resp.json() - - # Filter collections to only include the ones we created for this test - test_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + # Test 1: With limit=5 + endpoints = [ + {"method": "GET", "path": "/collections", "params": [("limit", "5")]}, + {"method": "GET", "path": "/collections-search", "params": [("limit", "5")]}, + {"method": "POST", "path": "/collections-search", "body": {"limit": 5}}, ] - # Should return 5 collections - assert len(test_collections) == 5 + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) - # Check that numberReturned matches the number of collections returned - assert resp_json["numberReturned"] == len(resp_json["collections"]) + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() - # Check that numberMatched is greater than or equal to numberReturned - # (since there might be other collections in the database) - assert resp_json["numberMatched"] >= resp_json["numberReturned"] + # Filter collections to only include the ones we created for this test + test_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Should return 5 collections + assert ( + len(test_collections) == 5 + ), f"Expected 5 test collections for {endpoint['method']} {endpoint['path']}" - # Check that numberMatched includes at least all our test collections - assert resp_json["numberMatched"] >= len(collection_ids) + # Check that numberReturned matches the number of collections returned + assert resp_json["numberReturned"] == len(resp_json["collections"]) - # Now test with a query that should match only some collections + # Check that numberMatched is greater than or equal to numberReturned + assert resp_json["numberMatched"] >= resp_json["numberReturned"] + + # Check that numberMatched includes at least all our test collections + assert resp_json["numberMatched"] >= len(collection_ids) + + # Test 2: With a query that should match only one collection query = {"id": {"eq": f"{test_prefix}-1"}} - resp = await app_client.get( - "/collections", - params=[("query", json.dumps(query))], - ) - assert resp.status_code == 200 - resp_json = resp.json() - # Filter collections to only include the ones we created for this test - test_collections = [ - c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + endpoints = [ + { + "method": "GET", + "path": "/collections", + "params": [("query", json.dumps(query))], + }, + { + "method": "GET", + "path": "/collections-search", + "params": [("query", json.dumps(query))], + }, + { + "method": "POST", + "path": "/collections-search", + "body": {"query": json.dumps(query)}, + }, ] - # Should return only 1 collection - assert len(test_collections) == 1 - assert test_collections[0]["id"] == f"{test_prefix}-1" + for endpoint in endpoints: + if endpoint["method"] == "GET": + resp = await app_client.get(endpoint["path"], params=endpoint["params"]) + else: # POST + resp = await app_client.post(endpoint["path"], json=endpoint["body"]) - # Check that numberReturned matches the number of collections returned - assert resp_json["numberReturned"] == len(resp_json["collections"]) + assert ( + resp.status_code == 200 + ), f"Failed for {endpoint['method']} {endpoint['path']}" + resp_json = resp.json() + + # Filter collections to only include the ones we created for this test + test_collections = [ + c for c in resp_json["collections"] if c["id"].startswith(test_prefix) + ] + + # Should return only 1 collection + assert ( + len(test_collections) == 1 + ), f"Expected 1 test collection for {endpoint['method']} {endpoint['path']}" + assert test_collections[0]["id"] == f"{test_prefix}-1" + + # Check that numberReturned matches the number of collections returned + assert resp_json["numberReturned"] == len(resp_json["collections"]) - # Check that numberMatched matches the number of collections that match the query - # (should be 1 in this case) - assert resp_json["numberMatched"] >= 1 + # Check that numberMatched matches the number of collections that match the query + assert resp_json["numberMatched"] >= 1 @pytest.mark.asyncio @@ -787,17 +935,35 @@ async def test_collections_pagination_all_endpoints(app_client, txn_client, ctx) for i, expected_id in enumerate(expected_ids): assert test_found[i]["id"] == expected_id - # Test second page using the token from the first page - if "token" in resp_json and resp_json["token"]: - token = resp_json["token"] + # Test second page using the token from the next link + next_link = None + for link in resp_json.get("links", []): + if link.get("rel") == "next": + next_link = link + break - # Make the request with token + if next_link: + # Extract token based on method if endpoint["method"] == "GET": - params = [(endpoint["param"], str(limit)), ("token", token)] - resp = await app_client.get(endpoint["path"], params=params) + # For GET, token is in the URL query params + from urllib.parse import parse_qs, urlparse + + parsed_url = urlparse(next_link["href"]) + query_params = parse_qs(parsed_url.query) + token = query_params.get("token", [None])[0] + + if token: + params = [(endpoint["param"], str(limit)), ("token", token)] + resp = await app_client.get(endpoint["path"], params=params) + else: + continue # Skip if no token found else: # POST - body = {endpoint["body_key"]: limit, "token": token} - resp = await app_client.post(endpoint["path"], json=body) + # For POST, token is in the body + body = next_link.get("body", {}) + if "token" in body: + resp = await app_client.post(endpoint["path"], json=body) + else: + continue # Skip if no token found assert ( resp.status_code == 200 @@ -805,10 +971,7 @@ async def test_collections_pagination_all_endpoints(app_client, txn_client, ctx) resp_json = resp.json() # Filter to our test collections - if endpoint["path"] == "/collections": - found_collections = resp_json - else: # For collection-search endpoints - found_collections = resp_json["collections"] + found_collections = resp_json["collections"] test_found = [ c for c in found_collections if c["id"].startswith(test_prefix) From ec299838da02d27ea1f1eb2df8a15176407ec244 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Sun, 5 Oct 2025 00:15:34 +0800 Subject: [PATCH 3/5] Add latest news section, CloudFerro logo (#485) **Related Issue(s):** - None **Description:** - Add `Latest News` section to readme - Add `CloudFerro` logo to the Sponsors and Supporters section **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- CHANGELOG.md | 3 +++ README.md | 6 ++++++ assets/cloudferro-logo.png | Bin 0 -> 6341 bytes 3 files changed, 9 insertions(+) create mode 100644 assets/cloudferro-logo.png diff --git a/CHANGELOG.md b/CHANGELOG.md index ed5cdeefc..8692132fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +- CloudFerro logo to sponsors and supporters list [#485](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/485) +- Latest news section to README [#485](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/485) + ### Changed - Removed ENV_MAX_LIMIT environment variable; maximum limits are now handled by the default global limit environment variable. [#482](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/482) diff --git a/README.md b/README.md index 5694a2133..839be8c2f 100644 --- a/README.md +++ b/README.md @@ -25,8 +25,13 @@ The following organizations have contributed time and/or funding to support the Healy Hyperspatial Atomic Maps VITO Remote Sensing + CloudFerro

+## Latest News + +- 10/04/2025: The [CloudFerro](https://cloudferro.com/) logo has been added to the sponsors and supporters list above. Their sponsorship of the ongoing collections search extension work has been invaluable. This is in addition to the many other important changes and updates their developers have added to the project. + ## Project Introduction - What is SFEOS? SFEOS (stac-fastapi-elasticsearch-opensearch) is a high-performance, scalable API implementation for serving SpatioTemporal Asset Catalog (STAC) data - an enhanced GeoJSON format designed specifically for geospatial assets like satellite imagery, aerial photography, and other Earth observation data. This project enables organizations to: @@ -67,6 +72,7 @@ This project is built on the following technologies: STAC, stac-fastapi, FastAPI - [stac-fastapi-elasticsearch-opensearch](#stac-fastapi-elasticsearch-opensearch) - [Sponsors & Supporters](#sponsors--supporters) + - [Latest News](#latest-news) - [Project Introduction - What is SFEOS?](#project-introduction---what-is-sfeos) - [Common Deployment Patterns](#common-deployment-patterns) - [Technologies](#technologies) diff --git a/assets/cloudferro-logo.png b/assets/cloudferro-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..464d53e649804239b4430aaa0abbb15938b99174 GIT binary patch literal 6341 zcmcIoc~n!^_P%K-AT}WZ6(s}%QK62lN}v%Tnn)Cff^|Se<%K$@fEtwtDgi78F+qwr z3pBBcGgqNnQ4|fJRjbjeQAAOy#G&E``h0>d*5?U zde}HQ%gNmd!!XvUk+Sg^_K64l@*M49B)?T|#4yIHc~WWED5+Eswwy{~3X!2Hf@!n}TW+ArR8yuo?P;2xj%*QG2dE6Xe2 zlbUh(0Q*3S;0^Qs!-qR8-aYSZ>@?LFe@587u6l8NeUN1CgiW)TXqj7Lwm$lH_=Sg) z$}jgh?dT<$dGbQv=y5-cf4=`<;$y?Hj8}czN`v=KzBuhOM~(6Dg)hFl&$|C;Y*o$i zuRUhF z;&0Qo6^Ckze>)f3_1u#_=OcIdt?khTx9i~D>G!aZPeu0|Zj8d$?R>W?GwE#O=s(7v zooFwt{AAB%LCMPJ{%?j(+xx{=_j-~}e$?}G>&D*9Nhw-2oDm;vnDZ-xc|;O_aqF}N-FG=l z(q`uzW3buDChu7XTW^=_S{&2%y69mO^PKOZ=tN8!3u&=ejF!u=(bz#OEOx+V9gK8{ z8#!e$Wbg+1WeoS-8V`dGs!?M{I6QE2a{HnK$58tk!@8UvB^x#&{??z>cVnxU@GD<5 zH8tFD{PPnAUtisepT6wb>iDt8BdWUH6l{r`_1Tx|@HU3)4=yf;*Tp~JGmKAqt*ySi zBy=CgHP)OK+Q~m^)v~T1go8_+AKl-@MH@Fj$j?x9G-^F@t|Re z2?Y}cWK5$t*o=|dkj4@v>54QaIB39?Ab^~pU3n3NKNuqkBV*(qq0q*ZD{_M5B~c+n zH6l#$vTrAhV8YNR*hKiRjT@OcbT|F@cz-M8=YA#%#SH_Z;5eyeOnZ;FNK@XWCdG z!bn$$l7fjS8SRmttwY2hQEu06uDpFM=CifJK82^t<~Deq z_gUJyBDp4C>$ASHX>CX@ALbjXobBg$ii@9IZ4T@ok=nUJZ%T|*6}nIEz0Z?3P81;& z>2goh-qC1=*gqT)RGhRFACU2{^Mxr-Is?Ddi3xOoos(EML?iAUUuLc}*Ld{YE(GrJw}b$0Yna z*yI{^S+W^bGf4=Qut1CiwK79Bo^0WGXjoQ?L=&(DLljzwX_N&k7d#yG4qF#eC8~ZP zMLAs!dQc-?B7x~6*7M!@t+U-)-0`dK6i_EOepQ$LLB&nSL9q}0k!s>-Z$pq?u&>kT z#1SN+QbWjZ^-8b*h*yM&S{-Btk_9n>6u=So5Mpfwg{(n)GKG7wT#JkquEJcr^U zGhTR^-}|9=k2by>f9Yi3vfk_vLsLW$UiB)yCW6b$tna1`9MVg4w3jHT>$b02D%g10 zK<3cV8}q9CTkHNjk&(E+%$?l-Z#VYIiPN4IP^8PvkX(i-gK@?;#5CyefZ5b&HfE|S*q~_Fidc)7u3P>7?)M($M6Ti445r4Z& zL7AmsEYty7jxvu>7#-yD2dN|zt6vWP8tX4>C8`Pj)QW2vy3kLFjEM5Ln8G(x9w+#V zdvgit&FcAb`3(okq{U>qMzJf9t5n}n1RlHcAwF~d;}-Yk+gDp>xAhPSkm&J2c*&F8 z;F4#n5UIKTqTK4F_s&O}2Hc5ey%}?3yGk{YM(D<$?n*gq36W2xWX!)~!XMBMlT^xi zl=&5nt{s=^6MFzujDqf9lIHj9F z>_o|>w)hUFRMHzR{|^n98Vvg>_&A{nzeoRXFyTK^{}*nyM9?Wyio^p^Ehw=Oayin~ zGC>)5c(PJ*nvy*6TFhD452@qIE9!nuXGJw;>&BpR=1EMZtaM_>_=;^XdZF@VR#@SqjmsU1wp;UkGu1xbFDarJGjZuW|GI_aGbPp=tJ0~c!qrxymA=GD#3kcfkb1xfS2#wLv@1I<*z6kx!2{EJzg=(;N#HD^*M4cDRs9&0rCpTzEMhbjL=})1i3y?wW$p*6A+1oQ}FZ*41n9@N+c<^R3hFyd0Kpk9GC0c=)55 zf>7(UCojiY7n4clfsKb}*I2I8OgTKij=GqP_Q|U?mJK$O&bs|N>txo*IORkdSEDti z7@oG9rrXP>n8+6!{Bd#C=?&B#mMR@}!$1097-Y%{QTFKUv3(8chB3 z#;A65=8dwi)Di=iCn^89SuVfy*`*8?^1{|F1>nL-(C*r{56bJ#;YIRzncF;PyqR`5 zX_p`D*pYz&B6l(m`QR-^*`vG16G|%t&gZ4niYeY=SH40Tw~uQz_Y&Dv?lFD2=nwz; z*tQqhbCxYX<6iXmoZ{Xu%-For6G<;T7YIZV!QunQTVEY4ZEY>9sJwfzSO1msvovWP zrapdx`_GJj`BaxT!nnKWD(all6EUx=) zd-dTtmv(F3ER{7(UA1?{`?{KE1MV)LHM?%?ix)0L4*VXT92s)UTWf@j_3K#)YX=+gv~9k-|s95Qa8E z>r=law(E!Vk$lz>TQ=$~#C~#wJdkbML(km9Lkhx)oOBJ}Cv4NOCGDu;1?7PyZwerM zVwvHrIsT7^-^!PjBhH9Qc}2+G!aO$hL}Wvv3fm6~L=F(W3=D3#GJ3!Y1*;65gf%Ru z3~jd0*K6oUqdo8&wvntMB(LEzEuEUGL}-$#u(@>NG;Aj=QGwvwcQDaS0Xd&q<)BZc zSN)$1bpv9y*1Lghx3V^?i`Qzk)o$Eut|^u^mDSj9Wy6c*m-=mp`hKP}i&gFa)mQ(% z*~yW8YhX}N5ZoEzny!WRgUY#O zYs|A*Xdl5niWxqB#TaZ|_<$$V#i1bR35g0*z!$t8KKn^*wC&JpxI}>O2zm@Yt(9sh z8%&XU$Zw3qv=e>~^-UCbZ@I4SpLlKatdqA&SF~<2=RdF9(!n<)n{6pIclGcH3`Bvr z{~d){Htbl4TwZyo?~q;qO}fKCWDF%shHe>v2V9X*QNS$-&VVd36zH}CCLGytKOv%{ zfbh@+83mevI|qs}Twajj(XgoUZTulv=oqm7iI5Qo*rnhe0nlTh4{6uV3n~TNQ@l{i zLA;QNBrmwp0Kq6;^aKryCX&tS{#U+WL@r23Xp!aCHOXaH-HYz+FsuC(x9ti=KKvqA zo>VH8laS;}lK6*cXbhndQBuLcEgl^uB8fpAVnmcyk~DjQB=jA)>9d7}?@HdOUW3PA z4>slI264G^Z~Z(L&e*;1G@~aEezY35)1zP&T%|$(Lop+QXcY?0OJpU3IwwsAP5gxn zYBoq?4U3$B)ifCdizckFSD+*zS41y3d^0d`(4eag*$(XtkR!wZ$w4wOc-nd_h_OW@ zIuhJQ5(Lzbp*L8A$v^fFmZ1j0ke#tRmG0`ZQ+R)*OeJ>OSq{OdXPL5Q-Mq&RsZ;?= zZ;{$@a_>`4wY9YluL6TTx0cM{B8`aUaJ`FK%0GKlMr3wSYsJf+-QFq`3J-=DYi{4Z gz07D|2p7_I(XpbJbG(Pb1vfTo#5mc3;ZZ670XMyU!vFvP literal 0 HcmV?d00001 From 20fa813217af8cf85629ff656987ca16535d1860 Mon Sep 17 00:00:00 2001 From: Jonathan Healy Date: Wed, 8 Oct 2025 15:56:05 +0800 Subject: [PATCH 4/5] Make latest news section scrollable, adjust logo size (#487) **Related Issue(s):** - None **Description:** - Make latest news section scrollable, adjust logo size **PR Checklist:** - [x] Code is formatted and linted (run `pre-commit run --all-files`) - [x] Tests pass (run `make test`) - [x] Documentation has been updated to reflect changes, if applicable - [x] Changes are added to the changelog --- README.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 839be8c2f..9ebc8208f 100644 --- a/README.md +++ b/README.md @@ -25,12 +25,16 @@ The following organizations have contributed time and/or funding to support the Healy Hyperspatial Atomic Maps VITO Remote Sensing - CloudFerro + CloudFerro

## Latest News -- 10/04/2025: The [CloudFerro](https://cloudferro.com/) logo has been added to the sponsors and supporters list above. Their sponsorship of the ongoing collections search extension work has been invaluable. This is in addition to the many other important changes and updates their developers have added to the project. +
+ +- 10/04/2025: The [CloudFerro](https://cloudferro.com/) logo has been added to the sponsors and supporters list above. Their sponsorship of the ongoing collections search extension work has been invaluable. This is in addition to the many other important changes and updates their developers have added to the project. + +
## Project Introduction - What is SFEOS? From 7d68424bd2c8fb197f058dc520fb74f24eb9dc42 Mon Sep 17 00:00:00 2001 From: Yuri Zmytrakov Date: Sun, 12 Oct 2025 19:37:17 +0200 Subject: [PATCH 5/5] fix: implement recommendations --- stac_fastapi/core/stac_fastapi/core/core.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/core/stac_fastapi/core/core.py b/stac_fastapi/core/stac_fastapi/core/core.py index c7d372041..619712c9e 100644 --- a/stac_fastapi/core/stac_fastapi/core/core.py +++ b/stac_fastapi/core/stac_fastapi/core/core.py @@ -275,8 +275,8 @@ async def all_collections( if os.getenv("STAC_GLOBAL_COLLECTION_MAX_LIMIT") else None ) - default_limit = int(os.getenv("STAC_DEFAULT_COLLECTION_LIMIT", 300)) query_limit = request.query_params.get("limit") + default_limit = int(os.getenv("STAC_DEFAULT_COLLECTION_LIMIT", 300)) body_limit = None try: @@ -742,14 +742,23 @@ async def post_search( if os.getenv("STAC_GLOBAL_ITEM_MAX_LIMIT") else None ) + query_limit = request.query_params.get("limit") default_limit = int(os.getenv("STAC_DEFAULT_ITEM_LIMIT", 10)) - requested_limit = getattr(search_request, "limit", None) + body_limit = None + try: + if request.method == "POST" and request.body(): + body_data = await request.json() + body_limit = body_data.get("limit") + except Exception: + pass - if requested_limit is None: - limit = default_limit + if body_limit is not None: + limit = int(body_limit) + elif query_limit: + limit = int(query_limit) else: - limit = requested_limit + limit = default_limit if global_max_limit: limit = min(limit, global_max_limit)