From 3a4b13e686c9b73a8b75eec5ee8b8c791b337388 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 3 Oct 2023 13:52:39 +0200 Subject: [PATCH 01/53] update stac-fastapi to v2.4.8 --- stac_fastapi/elasticsearch/setup.py | 8 ++--- .../stac_fastapi/elasticsearch/app.py | 35 ++----------------- .../stac_fastapi/elasticsearch/core.py | 5 +++ .../elasticsearch/tests/api/test_api.py | 4 ++- stac_fastapi/elasticsearch/tests/conftest.py | 14 ++++---- 5 files changed, 20 insertions(+), 46 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 7b0edddb..1a7fc7cf 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -8,11 +8,11 @@ install_requires = [ "fastapi", "attrs", - "pydantic[dotenv]", + "pydantic[dotenv]<2", "stac_pydantic==2.0.*", - "stac-fastapi.types==2.4.3", - "stac-fastapi.api==2.4.3", - "stac-fastapi.extensions==2.4.3", + "stac-fastapi.types==2.4.8", + "stac-fastapi.api==2.4.8", + "stac-fastapi.extensions==2.4.8", "elasticsearch[async]==7.17.9", "elasticsearch-dsl==7.4.1", "pystac[validation]", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 84eece90..96d07314 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -29,48 +29,17 @@ session = Session.create_from_settings(settings) -@attr.s -class FixedSortExtension(SortExtension): - """SortExtension class fixed with correct paths, removing extra forward-slash.""" - - conformance_classes: List[str] = attr.ib( - factory=lambda: ["https://api.stacspec.org/v1.0.0-beta.4/item-search#sort"] - ) - - @attr.s class FixedFilterExtension(FilterExtension): - """FilterExtension class fixed with correct paths, removing extra forward-slash.""" - - conformance_classes: List[str] = attr.ib( - default=[ - "https://api.stacspec.org/v1.0.0-rc.1/item-search#filter", - "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/filter", - "http://www.opengis.net/spec/ogcapi-features-3/1.0/conf/features-filter", - "http://www.opengis.net/spec/cql2/1.0/conf/cql2-text", - "http://www.opengis.net/spec/cql2/1.0/conf/cql2-json", - "http://www.opengis.net/spec/cql2/1.0/conf/basic-cql2", - "http://www.opengis.net/spec/cql2/1.0/conf/basic-spatial-operators", - ] - ) client = attr.ib(factory=EsAsyncBaseFiltersClient) -@attr.s -class FixedQueryExtension(QueryExtension): - """Fixed Query Extension string.""" - - conformance_classes: List[str] = attr.ib( - factory=lambda: ["https://api.stacspec.org/v1.0.0-beta.4/item-search#query"] - ) - - extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), BulkTransactionExtension(client=BulkTransactionsClient(session=session)), FieldsExtension(), - FixedQueryExtension(), - FixedSortExtension(), + QueryExtension(), + SortExtension(), TokenPaginationExtension(), ContextExtension(), FixedFilterExtension(), diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index e2af91a6..776ac746 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -286,6 +286,7 @@ async def get_search( token: Optional[str] = None, fields: Optional[List[str]] = None, sortby: Optional[str] = None, + intersects: Optional[str] = None, # filter: Optional[str] = None, # todo: requires fastapi > 2.3 unreleased # filter_lang: Optional[str] = None, # todo: requires fastapi > 2.3 unreleased **kwargs, @@ -302,6 +303,7 @@ async def get_search( token (Optional[str]): Access token to use when searching the catalog. fields (Optional[List[str]]): Fields to include or exclude from the results. sortby (Optional[str]): Sorting options for the results. + intersects (Optional[str]): GeoJSON geometry to search in. kwargs: Additional parameters to be passed to the API. Returns: @@ -322,6 +324,9 @@ async def get_search( if datetime: base_args["datetime"] = datetime + if intersects: + base_args["intersects"] = intersects + if sortby: # https://github.com/radiantearth/stac-spec/tree/master/api-spec/extensions/sort#http-get-or-post-form sort_param = [] diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 7dbf3996..042d9de4 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -187,7 +187,9 @@ async def test_app_query_extension_limit_lt0(app_client): async def test_app_query_extension_limit_gt10000(app_client): - assert (await app_client.post("/search", json={"limit": 10001})).status_code == 400 + resp = await app_client.post("/search", json={"limit": 10001}) + assert resp.status_code == 200 + assert resp.json()['context']['limit'] == 10000 async def test_app_query_extension_limit_10000(app_client): diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index b755425c..0a24b1f8 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -10,11 +10,6 @@ from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model -from stac_fastapi.elasticsearch.app import ( - FixedFilterExtension, - FixedQueryExtension, - FixedSortExtension, -) from stac_fastapi.elasticsearch.config import AsyncElasticsearchSettings from stac_fastapi.elasticsearch.core import ( BulkTransactionsClient, @@ -22,9 +17,12 @@ TransactionsClient, ) from stac_fastapi.elasticsearch.database_logic import create_collection_index +from stac_fastapi.elasticsearch.extensions import QueryExtension from stac_fastapi.extensions.core import ( # FieldsExtension, ContextExtension, FieldsExtension, + FilterExtension, + SortExtension, TokenPaginationExtension, TransactionExtension, ) @@ -160,11 +158,11 @@ async def app(): client=TransactionsClient(session=None), settings=settings ), ContextExtension(), - FixedSortExtension(), + SortExtension(), FieldsExtension(), - FixedQueryExtension(), + QueryExtension(), TokenPaginationExtension(), - FixedFilterExtension(), + FilterExtension(), ] post_request_model = create_post_request_model(extensions) From 3117b8b339d074660b922c44eaf2c96a8ea068e6 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 3 Oct 2023 13:56:25 +0200 Subject: [PATCH 02/53] fix pre-commit checks --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py | 4 ++-- stac_fastapi/elasticsearch/tests/api/test_api.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 96d07314..4c59729d 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -1,6 +1,4 @@ """FastAPI application.""" -from typing import List - import attr from stac_fastapi.api.app import StacApi @@ -31,6 +29,8 @@ @attr.s class FixedFilterExtension(FilterExtension): + """FilterExtension class implementation with EsAsyncBaseFiltersClient.""" + client = attr.ib(factory=EsAsyncBaseFiltersClient) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 042d9de4..c8edca78 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -189,7 +189,7 @@ async def test_app_query_extension_limit_lt0(app_client): async def test_app_query_extension_limit_gt10000(app_client): resp = await app_client.post("/search", json={"limit": 10001}) assert resp.status_code == 200 - assert resp.json()['context']['limit'] == 10000 + assert resp.json()["context"]["limit"] == 10000 async def test_app_query_extension_limit_10000(app_client): From 6e2fc2d8a1de92c286faaaf904a67288f28e4e00 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 3 Oct 2023 15:22:00 +0200 Subject: [PATCH 03/53] pass client to filter extension object instead of creating custom class --- .../elasticsearch/stac_fastapi/elasticsearch/app.py | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py index 4c59729d..e3c4bc64 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/app.py @@ -1,6 +1,4 @@ """FastAPI application.""" -import attr - from stac_fastapi.api.app import StacApi from stac_fastapi.api.models import create_get_request_model, create_post_request_model from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -26,14 +24,6 @@ settings = ElasticsearchSettings() session = Session.create_from_settings(settings) - -@attr.s -class FixedFilterExtension(FilterExtension): - """FilterExtension class implementation with EsAsyncBaseFiltersClient.""" - - client = attr.ib(factory=EsAsyncBaseFiltersClient) - - extensions = [ TransactionExtension(client=TransactionsClient(session=session), settings=settings), BulkTransactionExtension(client=BulkTransactionsClient(session=session)), @@ -42,7 +32,7 @@ class FixedFilterExtension(FilterExtension): SortExtension(), TokenPaginationExtension(), ContextExtension(), - FixedFilterExtension(), + FilterExtension(client=EsAsyncBaseFiltersClient()), ] post_request_model = create_post_request_model(extensions) From f48289dd0c79de01a967fb3ccbd7d39f05ea656f Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Thu, 5 Oct 2023 11:39:09 +0200 Subject: [PATCH 04/53] use Elasticsearch aliases with number suffix in index name --- .../elasticsearch/database_logic.py | 35 +++++++++++-------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6a7a1f27..4c0142c2 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -25,7 +25,7 @@ COLLECTIONS_INDEX = os.getenv("STAC_COLLECTIONS_INDEX", "collections") ITEMS_INDEX_PREFIX = os.getenv("STAC_ITEMS_INDEX_PREFIX", "items_") -DEFAULT_INDICES = f"*,-*kibana*,-{COLLECTIONS_INDEX}" +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*" DEFAULT_SORT = { "properties.datetime": {"order": "desc"}, @@ -150,7 +150,7 @@ def indices(collection_ids: Optional[List[str]]) -> str: A string of comma-separated index names. If `collection_ids` is None, returns the default indices. """ if collection_ids is None: - return DEFAULT_INDICES + return ITEM_INDICES else: return ",".join([f"{ITEMS_INDEX_PREFIX}{c.strip()}" for c in collection_ids]) @@ -164,7 +164,8 @@ async def create_collection_index() -> None: client = AsyncElasticsearchSettings().create_client await client.indices.create( - index=COLLECTIONS_INDEX, + index=f"{COLLECTIONS_INDEX}-000001", + aliases={COLLECTIONS_INDEX: {}}, mappings=ES_COLLECTIONS_MAPPINGS, ignore=400, # ignore 400 already exists code ) @@ -183,9 +184,11 @@ async def create_item_index(collection_id: str): """ client = AsyncElasticsearchSettings().create_client + index_name = index_by_collection_id(collection_id) await client.indices.create( - index=index_by_collection_id(collection_id), + index=f"{index_by_collection_id(collection_id)}-000001", + aliases={index_name: {}}, mappings=ES_ITEMS_MAPPINGS, settings=ES_ITEMS_SETTINGS, ignore=400, # ignore 400 already exists code @@ -201,7 +204,14 @@ async def delete_item_index(collection_id: str): """ client = AsyncElasticsearchSettings().create_client - await client.indices.delete(index=index_by_collection_id(collection_id)) + name = index_by_collection_id(collection_id) + resolved = await client.indices.resolve_index(name=name) + if "aliases" in resolved and resolved["aliases"]: + [alias] = resolved["aliases"] + await client.indices.delete_alias(index=alias["indices"], name=alias["name"]) + await client.indices.delete(index=alias["indices"]) + else: + await client.indices.delete(index=name) await client.close() @@ -759,14 +769,11 @@ async def bulk_async( `mk_actions` function is called to generate a list of actions for the bulk insert. If `refresh` is set to True, the index is refreshed after the bulk insert. The function does not return any value. """ - await asyncio.get_event_loop().run_in_executor( - None, - lambda: helpers.bulk( - self.sync_client, - mk_actions(collection_id, processed_items), - refresh=refresh, - raise_on_error=False, - ), + await helpers.async_bulk( + self.client, + mk_actions(collection_id, processed_items), + refresh=refresh, + raise_on_error=False, ) def bulk_sync( @@ -797,7 +804,7 @@ def bulk_sync( async def delete_items(self) -> None: """Danger. this is only for tests.""" await self.client.delete_by_query( - index=DEFAULT_INDICES, + index=ITEM_INDICES, body={"query": {"match_all": {}}}, wait_for_completion=True, ) From 920f121c4d0e77ff3a721c773ebaa00cc0ec1e17 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Thu, 5 Oct 2023 11:42:51 +0200 Subject: [PATCH 05/53] add changes to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c743ba6..7ab9c443 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) ### Changed +- Use aliases on Elasticsearch indices, add number suffix in index name. ### Fixed - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) From 6b7379a60578055145b777c30e2d3b8b6f2b55fa Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Mon, 9 Oct 2023 11:41:59 +0200 Subject: [PATCH 06/53] do not index proj:geometry and proj:centroid as geo fields --- CHANGELOG.md | 1 + .../stac_fastapi/elasticsearch/database_logic.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c743ba6..6cd2b837 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) - Corrected the automatic converstion of float values to int when building Filter Clauses [#135](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/135) +- Do not index `proj:geometry` and `proj:centroid` fields as geo shapes [#154](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/154) ## [v0.3.0] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6a7a1f27..f80c470c 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -67,13 +67,13 @@ { "proj_centroid": { "match": "proj:centroid", - "mapping": {"type": "geo_point"}, + "mapping": {"type": "object", "enabled": False}, } }, { "proj_geometry": { "match": "proj:geometry", - "mapping": {"type": "geo_shape"}, + "mapping": {"type": "object", "enabled": False}, } }, { From 4904f84c08c64e42cdf5f180d342371e485ecdbf Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 17 Oct 2023 10:46:30 +0200 Subject: [PATCH 07/53] re-add exclusion of collections index and internal indices to pattern of item indices in order to ensure backwards compatibility --- .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 22f40fed..126ab652 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -39,7 +39,7 @@ ":", } -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*" +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}" DEFAULT_SORT = { "properties.datetime": {"order": "desc"}, From e1d6232bcc4485176f610ea778dfafdf6de41145 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 17 Oct 2023 10:48:49 +0200 Subject: [PATCH 08/53] use wildcard in exclusion of collections index to support aliases with numbered indices --- .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 126ab652..702225ce 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -39,7 +39,7 @@ ":", } -ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}" +ITEM_INDICES = f"{ITEMS_INDEX_PREFIX}*,-*kibana*,-{COLLECTIONS_INDEX}*" DEFAULT_SORT = { "properties.datetime": {"order": "desc"}, From 7f1b5e5c17d898cd519bedb2dfcfff715d5c6041 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 21 Oct 2023 13:01:30 +0800 Subject: [PATCH 09/53] fix GET sortby, tests --- .../stac_fastapi/elasticsearch/core.py | 3 +- .../elasticsearch/tests/api/test_api.py | 98 ++++++++++++++++++- stac_fastapi/elasticsearch/tests/conftest.py | 4 +- 3 files changed, 102 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index e2af91a6..84d47a2a 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -329,9 +329,10 @@ async def get_search( sort_param.append( { "field": sort[1:], - "direction": "asc" if sort[0] == "+" else "desc", + "direction": "desc" if sort[0] == "-" else "asc", } ) + print(sort_param) base_args["sortby"] = sort_param # todo: requires fastapi > 2.3 unreleased diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 7dbf3996..14a3b1b3 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -2,6 +2,8 @@ import uuid from datetime import datetime, timedelta +import pytest + from ..conftest import create_collection, create_item ROUTES = { @@ -31,17 +33,20 @@ } +@pytest.mark.asyncio async def test_post_search_content_type(app_client, ctx): params = {"limit": 1} resp = await app_client.post("/search", json=params) assert resp.headers["content-type"] == "application/geo+json" +@pytest.mark.asyncio async def test_get_search_content_type(app_client, ctx): resp = await app_client.get("/search") assert resp.headers["content-type"] == "application/geo+json" +@pytest.mark.asyncio async def test_api_headers(app_client): resp = await app_client.get("/api") assert ( @@ -50,11 +55,13 @@ async def test_api_headers(app_client): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_router(app): api_routes = set([f"{list(route.methods)[0]} {route.path}" for route in app.routes]) assert len(api_routes - ROUTES) == 0 +@pytest.mark.asyncio async def test_app_transaction_extension(app_client, ctx): item = copy.deepcopy(ctx.item) item["id"] = str(uuid.uuid4()) @@ -64,6 +71,7 @@ async def test_app_transaction_extension(app_client, ctx): await app_client.delete(f"/collections/{item['collection']}/items/{item['id']}") +@pytest.mark.asyncio async def test_app_search_response(app_client, ctx): resp = await app_client.get("/search", params={"ids": ["test-item"]}) assert resp.status_code == 200 @@ -75,6 +83,7 @@ async def test_app_search_response(app_client, ctx): assert resp_json.get("stac_extensions") is None +@pytest.mark.asyncio async def test_app_context_extension(app_client, ctx, txn_client): test_item = ctx.item test_item["id"] = "test-item-2" @@ -108,6 +117,7 @@ async def test_app_context_extension(app_client, ctx, txn_client): assert matched == 1 +@pytest.mark.asyncio async def test_app_fields_extension(app_client, ctx, txn_client): resp = await app_client.get("/search", params={"collections": ["test-collection"]}) assert resp.status_code == 200 @@ -115,6 +125,7 @@ async def test_app_fields_extension(app_client, ctx, txn_client): assert list(resp_json["features"][0]["properties"]) == ["datetime"] +@pytest.mark.asyncio async def test_app_fields_extension_query(app_client, ctx, txn_client): resp = await app_client.post( "/search", @@ -128,6 +139,7 @@ async def test_app_fields_extension_query(app_client, ctx, txn_client): assert list(resp_json["features"][0]["properties"]) == ["datetime", "proj:epsg"] +@pytest.mark.asyncio async def test_app_fields_extension_no_properties_get(app_client, ctx, txn_client): resp = await app_client.get( "/search", params={"collections": ["test-collection"], "fields": "-properties"} @@ -137,6 +149,7 @@ async def test_app_fields_extension_no_properties_get(app_client, ctx, txn_clien assert "properties" not in resp_json["features"][0] +@pytest.mark.asyncio async def test_app_fields_extension_no_properties_post(app_client, ctx, txn_client): resp = await app_client.post( "/search", @@ -150,6 +163,7 @@ async def test_app_fields_extension_no_properties_post(app_client, ctx, txn_clie assert "properties" not in resp_json["features"][0] +@pytest.mark.asyncio async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_client): item = ctx.item resp = await app_client.get( @@ -166,6 +180,7 @@ async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_c assert feature["properties"][expected_prop] == expected_value +@pytest.mark.asyncio async def test_app_query_extension_gt(app_client, ctx): params = {"query": {"proj:epsg": {"gt": ctx.item["properties"]["proj:epsg"]}}} resp = await app_client.post("/search", json=params) @@ -174,6 +189,7 @@ async def test_app_query_extension_gt(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_app_query_extension_gte(app_client, ctx): params = {"query": {"proj:epsg": {"gte": ctx.item["properties"]["proj:epsg"]}}} resp = await app_client.post("/search", json=params) @@ -182,21 +198,95 @@ async def test_app_query_extension_gte(app_client, ctx): assert len(resp.json()["features"]) == 1 +@pytest.mark.asyncio async def test_app_query_extension_limit_lt0(app_client): assert (await app_client.post("/search", json={"limit": -1})).status_code == 400 +@pytest.mark.asyncio async def test_app_query_extension_limit_gt10000(app_client): assert (await app_client.post("/search", json={"limit": 10001})).status_code == 400 +@pytest.mark.asyncio async def test_app_query_extension_limit_10000(app_client): params = {"limit": 10000} resp = await app_client.post("/search", json=params) assert resp.status_code == 200 -async def test_app_sort_extension(app_client, txn_client, ctx): +@pytest.mark.asyncio +async def test_app_sort_extension_get_asc(app_client, txn_client, ctx): + first_item = ctx.item + item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) + + second_item = dict(first_item) + second_item["id"] = "another-item" + another_item_date = item_date - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + await create_item(txn_client, second_item) + + resp = await app_client.get("/search?sortby=+properties.datetime") + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][1]["id"] == first_item["id"] + assert resp_json["features"][0]["id"] == second_item["id"] + + +@pytest.mark.asyncio +async def test_app_sort_extension_get_desc(app_client, txn_client, ctx): + first_item = ctx.item + item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) + + second_item = dict(first_item) + second_item["id"] = "another-item" + another_item_date = item_date - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + await create_item(txn_client, second_item) + + resp = await app_client.get("/search?sortby=-properties.datetime") + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][0]["id"] == first_item["id"] + assert resp_json["features"][1]["id"] == second_item["id"] + + +@pytest.mark.asyncio +async def test_app_sort_extension_post_asc(app_client, txn_client, ctx): + first_item = ctx.item + item_date = datetime.strptime( + first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" + ) + + second_item = dict(first_item) + second_item["id"] = "another-item" + another_item_date = item_date - timedelta(days=1) + second_item["properties"]["datetime"] = another_item_date.strftime( + "%Y-%m-%dT%H:%M:%SZ" + ) + await create_item(txn_client, second_item) + + params = { + "collections": [first_item["collection"]], + "sortby": [{"field": "properties.datetime", "direction": "asc"}], + } + resp = await app_client.post("/search", json=params) + assert resp.status_code == 200 + resp_json = resp.json() + assert resp_json["features"][1]["id"] == first_item["id"] + assert resp_json["features"][0]["id"] == second_item["id"] + + +@pytest.mark.asyncio +async def test_app_sort_extension_post_desc(app_client, txn_client, ctx): first_item = ctx.item item_date = datetime.strptime( first_item["properties"]["datetime"], "%Y-%m-%dT%H:%M:%SZ" @@ -221,6 +311,7 @@ async def test_app_sort_extension(app_client, txn_client, ctx): assert resp_json["features"][1]["id"] == second_item["id"] +@pytest.mark.asyncio async def test_search_invalid_date(app_client, ctx): params = { "datetime": "2020-XX-01/2020-10-30", @@ -231,6 +322,7 @@ async def test_search_invalid_date(app_client, ctx): assert resp.status_code == 400 +@pytest.mark.asyncio async def test_search_point_intersects(app_client, ctx): point = [150.04, -33.14] intersects = {"type": "Point", "coordinates": point} @@ -246,6 +338,7 @@ async def test_search_point_intersects(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_point_does_not_intersect(app_client, ctx): point = [15.04, -3.14] intersects = {"type": "Point", "coordinates": point} @@ -261,6 +354,7 @@ async def test_search_point_does_not_intersect(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_datetime_non_interval(app_client, ctx): dt_formats = [ "2020-02-12T12:30:22+00:00", @@ -282,6 +376,7 @@ async def test_datetime_non_interval(app_client, ctx): assert resp_json["features"][0]["properties"]["datetime"][0:19] == dt[0:19] +@pytest.mark.asyncio async def test_bbox_3d(app_client, ctx): australia_bbox = [106.343365, -47.199523, 0.1, 168.218365, -19.437288, 0.1] params = { @@ -294,6 +389,7 @@ async def test_bbox_3d(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_line_string_intersects(app_client, ctx): line = [[150.04, -33.14], [150.22, -33.89]] intersects = {"type": "LineString", "coordinates": line} diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index b755425c..c5de7a69 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -61,7 +61,9 @@ class Config: @pytest.fixture(scope="session") def event_loop(): - return asyncio.get_event_loop() + loop = asyncio.new_event_loop() + yield loop + loop.close() def _load_file(filename: str) -> Dict: From 2573cf1dbbe527eab8c03755459344c1a8febeab Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 21 Oct 2023 13:42:20 +0800 Subject: [PATCH 10/53] update changelog, tests, mark tests --- CHANGELOG.md | 1 + .../tests/clients/test_elasticsearch.py | 15 +++ .../tests/extensions/test_filter.py | 7 ++ .../tests/resources/test_collection.py | 8 ++ .../tests/resources/test_conformance.py | 3 + .../tests/resources/test_item.py | 99 +++++++++---------- .../tests/resources/test_mgmt.py | 4 + 7 files changed, 83 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52d2701b..b2559365 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) - Corrected the automatic converstion of float values to int when building Filter Clauses [#135](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/135) - Remove unsupported characters from Elasticsearch index names [#153](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/153) +- Fixed GET /search sortby requests https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/25 ## [v0.3.0] diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py index e46d4d1f..3da8f86d 100644 --- a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py +++ b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py @@ -11,6 +11,7 @@ from ..conftest import MockRequest, create_item +@pytest.mark.asyncio async def test_create_collection(app_client, ctx, core_client, txn_client): in_coll = deepcopy(ctx.collection) in_coll["id"] = str(uuid.uuid4()) @@ -20,6 +21,7 @@ async def test_create_collection(app_client, ctx, core_client, txn_client): await txn_client.delete_collection(in_coll["id"]) +@pytest.mark.asyncio async def test_create_collection_already_exists(app_client, ctx, txn_client): data = deepcopy(ctx.collection) @@ -32,6 +34,7 @@ async def test_create_collection_already_exists(app_client, ctx, txn_client): await txn_client.delete_collection(data["id"]) +@pytest.mark.asyncio async def test_update_collection( core_client, txn_client, @@ -49,6 +52,7 @@ async def test_update_collection( await txn_client.delete_collection(data["id"]) +@pytest.mark.asyncio async def test_delete_collection( core_client, txn_client, @@ -63,6 +67,7 @@ async def test_delete_collection( await core_client.get_collection(data["id"], request=MockRequest) +@pytest.mark.asyncio async def test_get_collection( core_client, txn_client, @@ -76,6 +81,7 @@ async def test_get_collection( await txn_client.delete_collection(data["id"]) +@pytest.mark.asyncio async def test_get_item(app_client, ctx, core_client): got_item = await core_client.get_item( item_id=ctx.item["id"], @@ -86,6 +92,7 @@ async def test_get_item(app_client, ctx, core_client): assert got_item["collection"] == ctx.item["collection"] +@pytest.mark.asyncio async def test_get_collection_items(app_client, ctx, core_client, txn_client): coll = ctx.collection num_of_items_to_create = 5 @@ -106,6 +113,7 @@ async def test_get_collection_items(app_client, ctx, core_client, txn_client): assert item["collection"] == coll["id"] +@pytest.mark.asyncio async def test_create_item(ctx, core_client, txn_client): resp = await core_client.get_item( ctx.item["id"], ctx.item["collection"], request=MockRequest @@ -115,6 +123,7 @@ async def test_create_item(ctx, core_client, txn_client): ) == Item(**resp).dict(exclude={"links": ..., "properties": {"created", "updated"}}) +@pytest.mark.asyncio async def test_create_item_already_exists(ctx, txn_client): with pytest.raises(ConflictError): await txn_client.create_item( @@ -125,6 +134,7 @@ async def test_create_item_already_exists(ctx, txn_client): ) +@pytest.mark.asyncio async def test_update_item(ctx, core_client, txn_client): ctx.item["properties"]["foo"] = "bar" collection_id = ctx.item["collection"] @@ -139,6 +149,7 @@ async def test_update_item(ctx, core_client, txn_client): assert updated_item["properties"]["foo"] == "bar" +@pytest.mark.asyncio async def test_update_geometry(ctx, core_client, txn_client): new_coordinates = [ [ @@ -163,6 +174,7 @@ async def test_update_geometry(ctx, core_client, txn_client): assert updated_item["geometry"]["coordinates"] == new_coordinates +@pytest.mark.asyncio async def test_delete_item(ctx, core_client, txn_client): await txn_client.delete_item(ctx.item["id"], ctx.item["collection"]) @@ -172,6 +184,7 @@ async def test_delete_item(ctx, core_client, txn_client): ) +@pytest.mark.asyncio async def test_bulk_item_insert(ctx, core_client, txn_client, bulk_txn_client): items = {} for _ in range(10): @@ -193,6 +206,7 @@ async def test_bulk_item_insert(ctx, core_client, txn_client, bulk_txn_client): # ) +@pytest.mark.asyncio async def test_feature_collection_insert( core_client, txn_client, @@ -212,6 +226,7 @@ async def test_feature_collection_insert( assert len(fc["features"]) >= 10 +@pytest.mark.asyncio async def test_landing_page_no_collection_title(ctx, core_client, txn_client, app): ctx.collection["id"] = "new_id" del ctx.collection["title"] diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index 43aadf18..f20ab4ea 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -3,9 +3,12 @@ from os import listdir from os.path import isfile, join +import pytest + THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +@pytest.mark.asyncio async def test_search_filters(app_client, ctx): filters = [] @@ -19,6 +22,7 @@ async def test_search_filters(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_search_filter_extension_eq(app_client, ctx): params = {"filter": {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}} resp = await app_client.post("/search", json=params) @@ -27,6 +31,7 @@ async def test_search_filter_extension_eq(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_filter_extension_gte(app_client, ctx): # there's one item that can match, so one of these queries should match it and the other shouldn't params = { @@ -58,6 +63,7 @@ async def test_search_filter_extension_gte(app_client, ctx): assert len(resp.json()["features"]) == 0 +@pytest.mark.asyncio async def test_search_filter_ext_and(app_client, ctx): params = { "filter": { @@ -80,6 +86,7 @@ async def test_search_filter_ext_and(app_client, ctx): assert len(resp.json()["features"]) == 1 +@pytest.mark.asyncio async def test_search_filter_extension_floats(app_client, ctx): sun_elevation = ctx.item["properties"]["view:sun_elevation"] diff --git a/stac_fastapi/elasticsearch/tests/resources/test_collection.py b/stac_fastapi/elasticsearch/tests/resources/test_collection.py index f37b36b0..22c35221 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_collection.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_collection.py @@ -1,6 +1,8 @@ import pystac +import pytest +@pytest.mark.asyncio async def test_create_and_delete_collection(app_client, load_test_data): """Test creation and deletion of a collection""" test_collection = load_test_data("test_collection.json") @@ -13,6 +15,7 @@ async def test_create_and_delete_collection(app_client, load_test_data): assert resp.status_code == 204 +@pytest.mark.asyncio async def test_create_collection_conflict(app_client, ctx): """Test creation of a collection which already exists""" # This collection ID is created in the fixture, so this should be a conflict @@ -20,12 +23,14 @@ async def test_create_collection_conflict(app_client, ctx): assert resp.status_code == 409 +@pytest.mark.asyncio async def test_delete_missing_collection(app_client): """Test deletion of a collection which does not exist""" resp = await app_client.delete("/collections/missing-collection") assert resp.status_code == 404 +@pytest.mark.asyncio async def test_update_collection_already_exists(ctx, app_client): """Test updating a collection which already exists""" ctx.collection["keywords"].append("test") @@ -38,6 +43,7 @@ async def test_update_collection_already_exists(ctx, app_client): assert "test" in resp_json["keywords"] +@pytest.mark.asyncio async def test_update_new_collection(app_client, load_test_data): """Test updating a collection which does not exist (same as creation)""" test_collection = load_test_data("test_collection.json") @@ -47,12 +53,14 @@ async def test_update_new_collection(app_client, load_test_data): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_collection_not_found(app_client): """Test read a collection which does not exist""" resp = await app_client.get("/collections/does-not-exist") assert resp.status_code == 404 +@pytest.mark.asyncio async def test_returns_valid_collection(ctx, app_client): """Test validates fetched collection with jsonschema""" resp = await app_client.put("/collections", json=ctx.collection) diff --git a/stac_fastapi/elasticsearch/tests/resources/test_conformance.py b/stac_fastapi/elasticsearch/tests/resources/test_conformance.py index ab70a00b..d93d8b81 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_conformance.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_conformance.py @@ -20,6 +20,7 @@ def get_link(landing_page, rel_type): ) +@pytest.mark.asyncio async def test_landing_page_health(response): """Test landing page""" assert response.status_code == 200 @@ -39,6 +40,7 @@ async def test_landing_page_health(response): ] +@pytest.mark.asyncio @pytest.mark.parametrize("rel_type,expected_media_type,expected_path", link_tests) async def test_landing_page_links( response_json, app_client, rel_type, expected_media_type, expected_path @@ -59,6 +61,7 @@ async def test_landing_page_links( # code here seems meaningless since it would be the same as if the endpoint did not exist. Once # https://github.com/stac-utils/stac-fastapi/pull/227 has been merged we can add this to the # parameterized tests above. +@pytest.mark.asyncio async def test_search_link(response_json): search_link = get_link(response_json, "search") diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_fastapi/elasticsearch/tests/resources/test_item.py index 76f38f79..5b382873 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_item.py @@ -23,6 +23,7 @@ def rfc3339_str_to_datetime(s: str) -> datetime: return ciso8601.parse_rfc3339(s) +@pytest.mark.asyncio async def test_create_and_delete_item(app_client, ctx, txn_client): """Test creation and deletion of a single item (transactions extension)""" @@ -46,6 +47,7 @@ async def test_create_and_delete_item(app_client, ctx, txn_client): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_create_item_conflict(app_client, ctx): """Test creation of an item which already exists (transactions extension)""" @@ -57,6 +59,7 @@ async def test_create_item_conflict(app_client, ctx): assert resp.status_code == 409 +@pytest.mark.asyncio async def test_delete_missing_item(app_client, load_test_data): """Test deletion of an item which does not exist (transactions extension)""" test_item = load_test_data("test_item.json") @@ -66,6 +69,7 @@ async def test_delete_missing_item(app_client, load_test_data): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_create_item_missing_collection(app_client, ctx): """Test creation of an item without a parent collection (transactions extension)""" ctx.item["collection"] = "stac_is_cool" @@ -75,6 +79,7 @@ async def test_create_item_missing_collection(app_client, ctx): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_create_uppercase_collection_with_item(app_client, ctx, txn_client): """Test creation of a collection and item with uppercase collection ID (transactions extension)""" collection_id = "UPPERCASE" @@ -87,6 +92,7 @@ async def test_create_uppercase_collection_with_item(app_client, ctx, txn_client assert resp.status_code == 200 +@pytest.mark.asyncio async def test_update_item_already_exists(app_client, ctx): """Test updating an item which already exists (transactions extension)""" @@ -106,6 +112,7 @@ async def test_update_item_already_exists(app_client, ctx): ) +@pytest.mark.asyncio async def test_update_new_item(app_client, ctx): """Test updating an item which does not exist (transactions extension)""" test_item = ctx.item @@ -118,6 +125,7 @@ async def test_update_new_item(app_client, ctx): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_update_item_missing_collection(app_client, ctx): """Test updating an item without a parent collection (transactions extension)""" # Try to update collection of the item @@ -128,6 +136,7 @@ async def test_update_item_missing_collection(app_client, ctx): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_update_item_geometry(app_client, ctx): ctx.item["id"] = "update_test_item_1" @@ -162,6 +171,7 @@ async def test_update_item_geometry(app_client, ctx): assert resp.json()["geometry"]["coordinates"] == new_coordinates +@pytest.mark.asyncio async def test_get_item(app_client, ctx): """Test read an item by id (core)""" get_item = await app_client.get( @@ -170,6 +180,7 @@ async def test_get_item(app_client, ctx): assert get_item.status_code == 200 +@pytest.mark.asyncio async def test_returns_valid_item(app_client, ctx): """Test validates fetched item with jsonschema""" test_item = ctx.item @@ -186,6 +197,7 @@ async def test_returns_valid_item(app_client, ctx): item.validate() +@pytest.mark.asyncio async def test_get_item_collection(app_client, ctx, txn_client): """Test read an item collection (core)""" item_count = randint(1, 4) @@ -202,6 +214,7 @@ async def test_get_item_collection(app_client, ctx, txn_client): assert matched == item_count + 1 +@pytest.mark.asyncio async def test_item_collection_filter_bbox(app_client, ctx): item = ctx.item collection = item["collection"] @@ -223,6 +236,7 @@ async def test_item_collection_filter_bbox(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_item_collection_filter_datetime(app_client, ctx): item = ctx.item collection = item["collection"] @@ -244,6 +258,7 @@ async def test_item_collection_filter_datetime(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio @pytest.mark.skip(reason="Pagination extension not implemented") async def test_pagination(app_client, load_test_data): """Test item collection pagination (paging extension)""" @@ -272,6 +287,7 @@ async def test_pagination(app_client, load_test_data): assert second_page["context"]["returned"] == 3 +@pytest.mark.asyncio async def test_item_timestamps(app_client, ctx): """Test created and updated timestamps (common metadata)""" # start_time = now_to_rfc3339_str() @@ -300,6 +316,7 @@ async def test_item_timestamps(app_client, ctx): ) +@pytest.mark.asyncio async def test_item_search_by_id_post(app_client, ctx, txn_client): """Test POST search by item id (core)""" ids = ["test1", "test2", "test3"] @@ -315,6 +332,7 @@ async def test_item_search_by_id_post(app_client, ctx, txn_client): assert set([feat["id"] for feat in resp_json["features"]]) == set(ids) +@pytest.mark.asyncio async def test_item_search_spatial_query_post(app_client, ctx): """Test POST search with spatial query (core)""" test_item = ctx.item @@ -329,6 +347,7 @@ async def test_item_search_spatial_query_post(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio async def test_item_search_temporal_query_post(app_client, ctx): """Test POST search with single-tailed spatio-temporal query (core)""" @@ -347,6 +366,7 @@ async def test_item_search_temporal_query_post(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio async def test_item_search_temporal_window_post(app_client, ctx): """Test POST search with two-tailed spatio-temporal query (core)""" test_item = ctx.item @@ -365,6 +385,7 @@ async def test_item_search_temporal_window_post(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio @pytest.mark.skip(reason="KeyError: 'features") async def test_item_search_temporal_open_window(app_client, ctx): """Test POST search with open spatio-temporal query (core)""" @@ -379,39 +400,7 @@ async def test_item_search_temporal_open_window(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] -@pytest.mark.skip(reason="sortby date not implemented") -async def test_item_search_sort_post(app_client, load_test_data): - """Test POST search with sorting (sort extension)""" - first_item = load_test_data("test_item.json") - item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"]) - resp = await app_client.post( - f"/collections/{first_item['collection']}/items", json=first_item - ) - assert resp.status_code == 200 - - second_item = load_test_data("test_item.json") - second_item["id"] = "another-item" - another_item_date = item_date - timedelta(days=1) - second_item["properties"]["datetime"] = datetime_to_str(another_item_date) - resp = await app_client.post( - f"/collections/{second_item['collection']}/items", json=second_item - ) - assert resp.status_code == 200 - - params = { - "collections": [first_item["collection"]], - "sortby": [{"field": "datetime", "direction": "desc"}], - } - resp = await app_client.post("/search", json=params) - assert resp.status_code == 200 - resp_json = resp.json() - assert resp_json["features"][0]["id"] == first_item["id"] - assert resp_json["features"][1]["id"] == second_item["id"] - await app_client.delete( - f"/collections/{first_item['collection']}/items/{first_item['id']}" - ) - - +@pytest.mark.asyncio async def test_item_search_by_id_get(app_client, ctx, txn_client): """Test GET search by item id (core)""" ids = ["test1", "test2", "test3"] @@ -427,6 +416,7 @@ async def test_item_search_by_id_get(app_client, ctx, txn_client): assert set([feat["id"] for feat in resp_json["features"]]) == set(ids) +@pytest.mark.asyncio async def test_item_search_bbox_get(app_client, ctx): """Test GET search with spatial query (core)""" params = { @@ -439,6 +429,7 @@ async def test_item_search_bbox_get(app_client, ctx): assert resp_json["features"][0]["id"] == ctx.item["id"] +@pytest.mark.asyncio async def test_item_search_get_without_collections(app_client, ctx): """Test GET search without specifying collections""" @@ -449,6 +440,7 @@ async def test_item_search_get_without_collections(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_item_search_get_with_non_existent_collections(app_client, ctx): """Test GET search with non-existent collections""" @@ -457,6 +449,7 @@ async def test_item_search_get_with_non_existent_collections(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_item_search_temporal_window_get(app_client, ctx): """Test GET search with spatio-temporal query (core)""" test_item = ctx.item @@ -474,27 +467,7 @@ async def test_item_search_temporal_window_get(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] -@pytest.mark.skip(reason="sorting not fully implemented") -async def test_item_search_sort_get(app_client, ctx, txn_client): - """Test GET search with sorting (sort extension)""" - first_item = ctx.item - item_date = rfc3339_str_to_datetime(first_item["properties"]["datetime"]) - await create_item(txn_client, ctx.item) - - second_item = ctx.item.copy() - second_item["id"] = "another-item" - another_item_date = item_date - timedelta(days=1) - second_item.update({"properties": {"datetime": datetime_to_str(another_item_date)}}) - await create_item(txn_client, second_item) - - params = {"collections": [first_item["collection"]], "sortby": "-datetime"} - resp = await app_client.get("/search", params=params) - assert resp.status_code == 200 - resp_json = resp.json() - assert resp_json["features"][0]["id"] == first_item["id"] - assert resp_json["features"][1]["id"] == second_item["id"] - - +@pytest.mark.asyncio async def test_item_search_post_without_collection(app_client, ctx): """Test POST search without specifying a collection""" test_item = ctx.item @@ -505,6 +478,7 @@ async def test_item_search_post_without_collection(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_item_search_properties_es(app_client, ctx): """Test POST search with JSONB query (query extension)""" @@ -517,6 +491,7 @@ async def test_item_search_properties_es(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_item_search_properties_field(app_client): """Test POST search indexed field with query (query extension)""" @@ -528,6 +503,7 @@ async def test_item_search_properties_field(app_client): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_item_search_get_query_extension(app_client, ctx): """Test GET search with JSONB query (query extension)""" @@ -554,12 +530,14 @@ async def test_item_search_get_query_extension(app_client, ctx): ) +@pytest.mark.asyncio async def test_get_missing_item_collection(app_client): """Test reading a collection which does not exist""" resp = await app_client.get("/collections/invalid-collection/items") assert resp.status_code == 404 +@pytest.mark.asyncio async def test_pagination_item_collection(app_client, ctx, txn_client): """Test item collection pagination links (paging extension)""" ids = [ctx.item["id"]] @@ -596,6 +574,7 @@ async def test_pagination_item_collection(app_client, ctx, txn_client): assert not set(item_ids) - set(ids) +@pytest.mark.asyncio async def test_pagination_post(app_client, ctx, txn_client): """Test POST pagination (paging extension)""" ids = [ctx.item["id"]] @@ -631,6 +610,7 @@ async def test_pagination_post(app_client, ctx, txn_client): assert not set(item_ids) - set(ids) +@pytest.mark.asyncio async def test_pagination_token_idempotent(app_client, ctx, txn_client): """Test that pagination tokens are idempotent (paging extension)""" ids = [ctx.item["id"]] @@ -661,6 +641,7 @@ async def test_pagination_token_idempotent(app_client, ctx, txn_client): ] +@pytest.mark.asyncio async def test_field_extension_get_includes(app_client, ctx): """Test GET search with included fields (fields extension)""" test_item = ctx.item @@ -673,6 +654,7 @@ async def test_field_extension_get_includes(app_client, ctx): assert not set(feat_properties) - {"proj:epsg", "gsd", "datetime"} +@pytest.mark.asyncio async def test_field_extension_get_excludes(app_client, ctx): """Test GET search with included fields (fields extension)""" test_item = ctx.item @@ -686,6 +668,7 @@ async def test_field_extension_get_excludes(app_client, ctx): assert "gsd" not in resp_json["features"][0]["properties"].keys() +@pytest.mark.asyncio async def test_field_extension_post(app_client, ctx): """Test POST search with included and excluded fields (fields extension)""" test_item = ctx.item @@ -707,6 +690,7 @@ async def test_field_extension_post(app_client, ctx): } +@pytest.mark.asyncio async def test_field_extension_exclude_and_include(app_client, ctx): """Test POST search including/excluding same field (fields extension)""" test_item = ctx.item @@ -723,6 +707,7 @@ async def test_field_extension_exclude_and_include(app_client, ctx): assert "eo:cloud_cover" not in resp_json["features"][0]["properties"] +@pytest.mark.asyncio async def test_field_extension_exclude_default_includes(app_client, ctx): """Test POST search excluding a forbidden field (fields extension)""" test_item = ctx.item @@ -733,6 +718,7 @@ async def test_field_extension_exclude_default_includes(app_client, ctx): assert "gsd" not in resp_json["features"][0] +@pytest.mark.asyncio async def test_search_intersects_and_bbox(app_client): """Test POST search intersects and bbox are mutually exclusive (core)""" bbox = [-118, 34, -117, 35] @@ -742,6 +728,7 @@ async def test_search_intersects_and_bbox(app_client): assert resp.status_code == 400 +@pytest.mark.asyncio async def test_get_missing_item(app_client, load_test_data): """Test read item which does not exist (transactions extension)""" test_coll = load_test_data("test_collection.json") @@ -749,6 +736,7 @@ async def test_get_missing_item(app_client, load_test_data): assert resp.status_code == 404 +@pytest.mark.asyncio @pytest.mark.skip(reason="invalid queries not implemented") async def test_search_invalid_query_field(app_client): body = {"query": {"gsd": {"lt": 100}, "invalid-field": {"eq": 50}}} @@ -756,6 +744,7 @@ async def test_search_invalid_query_field(app_client): assert resp.status_code == 400 +@pytest.mark.asyncio async def test_search_bbox_errors(app_client): body = {"query": {"bbox": [0]}} resp = await app_client.post("/search", json=body) @@ -770,6 +759,7 @@ async def test_search_bbox_errors(app_client): assert resp.status_code == 400 +@pytest.mark.asyncio async def test_conformance_classes_configurable(): """Test conformance class configurability""" landing = LandingPageMixin() @@ -787,6 +777,7 @@ async def test_conformance_classes_configurable(): assert client.conformance_classes()[0] == "this is a test" +@pytest.mark.asyncio async def test_search_datetime_validation_errors(app_client): bad_datetimes = [ "37-01-01T12:00:27.87Z", diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py index 9d2bc3dc..2b7d9728 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py @@ -1,3 +1,7 @@ +import pytest + + +@pytest.mark.asyncio async def test_ping_no_param(app_client): """ Test ping endpoint with a mocked client. From 9bad4c5f84a9639bd64a0eb217bc2a0ef3fd68d3 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 21 Oct 2023 11:23:28 +0300 Subject: [PATCH 11/53] fix link --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b2559365..3cf6552d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,7 +21,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) - Corrected the automatic converstion of float values to int when building Filter Clauses [#135](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/135) - Remove unsupported characters from Elasticsearch index names [#153](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/153) -- Fixed GET /search sortby requests https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/25 +- Fixed GET /search sortby requests [#25](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/25) ## [v0.3.0] From b983f285667e07a469cf5c5b449656b7d005603d Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 21 Oct 2023 18:58:06 +0300 Subject: [PATCH 12/53] parse intersection string, test --- .../stac_fastapi/elasticsearch/core.py | 4 ++-- stac_fastapi/elasticsearch/tests/api/test_api.py | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 776ac746..031a0235 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -4,7 +4,7 @@ from datetime import datetime as datetime_type from datetime import timezone from typing import Any, Dict, List, Optional, Set, Type, Union -from urllib.parse import urljoin +from urllib.parse import unquote_plus, urljoin import attr import stac_pydantic @@ -325,7 +325,7 @@ async def get_search( base_args["datetime"] = datetime if intersects: - base_args["intersects"] = intersects + base_args["intersects"] = json.loads(unquote_plus(intersects)) if sortby: # https://github.com/radiantearth/stac-spec/tree/master/api-spec/extensions/sort#http-get-or-post-form diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index c8edca78..c45c35ff 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -233,7 +233,18 @@ async def test_search_invalid_date(app_client, ctx): assert resp.status_code == 400 -async def test_search_point_intersects(app_client, ctx): +async def test_search_point_intersects_get(app_client, ctx): + point = [150.04, -33.14] + resp = await app_client.get( + f"/search?intersects={'type':'Point','coordinates':{point}}" + ) + + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 1 + + +async def test_search_point_intersects_post(app_client, ctx): point = [150.04, -33.14] intersects = {"type": "Point", "coordinates": point} From 836bce019ac7a01508cae720ea2f4a2a27c40f26 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 21 Oct 2023 19:15:53 +0300 Subject: [PATCH 13/53] add get polygon test --- .../elasticsearch/tests/api/test_api.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index c45c35ff..3ae65ebd 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -2,6 +2,8 @@ import uuid from datetime import datetime, timedelta +import pytest + from ..conftest import create_collection, create_item ROUTES = { @@ -233,10 +235,21 @@ async def test_search_invalid_date(app_client, ctx): assert resp.status_code == 400 +@pytest.mark.asyncio async def test_search_point_intersects_get(app_client, ctx): - point = [150.04, -33.14] resp = await app_client.get( - f"/search?intersects={'type':'Point','coordinates':{point}}" + '/search?intersects={"type":"Point","coordinates":[150.04,-33.14]}' + ) + + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_polygon_intersects_get(app_client, ctx): + resp = await app_client.get( + '/search?intersects={"type":"Polygon","coordinates":[[[149.04, -34.14],[149.04, -32.14],[151.04, -32.14],[151.04, -34.14],[149.04, -34.14]]]}' ) assert resp.status_code == 200 From b59cf702eba1021daff33601dc74fa02f8e94aea Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 21 Oct 2023 19:24:54 +0300 Subject: [PATCH 14/53] update changelog --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52d2701b..072c7465 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,12 +10,13 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148) - -### Added - - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) +- Added support for GET /search intersection queries [#158](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/158) ### Changed + +- Updated core stac-fastapi libraries to 2.4.8 from 2.4.3 [#151](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/151) + ### Fixed - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) From e9c258b8a1f60a71b21cc0836c593b493f4ae282 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 23 Oct 2023 23:03:26 +0800 Subject: [PATCH 15/53] GET filter extension requests --- stac_fastapi/elasticsearch/setup.py | 1 + .../stac_fastapi/elasticsearch/core.py | 48 +++++--- .../tests/extensions/test_filter.py | 114 +++++++++++++++++- 3 files changed, 138 insertions(+), 25 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 1a7fc7cf..6ff8cd86 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -17,6 +17,7 @@ "elasticsearch-dsl==7.4.1", "pystac[validation]", "uvicorn", + "orjson", "overrides", "starlette", "geojson-pydantic", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 776ac746..f22f209f 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -1,19 +1,21 @@ """Item crud client.""" -import json import logging +import re from datetime import datetime as datetime_type from datetime import timezone from typing import Any, Dict, List, Optional, Set, Type, Union -from urllib.parse import urljoin +from urllib.parse import unquote_plus, urljoin import attr +import orjson import stac_pydantic -from fastapi import HTTPException +from fastapi import HTTPException, Request from overrides import overrides from pydantic import ValidationError +from pygeofilter.backends.cql2_json import to_cql2 +from pygeofilter.parsers.cql2_text import parse as parse_cql2_text from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes -from starlette.requests import Request from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -274,9 +276,9 @@ def _return_date(interval_str): return {"lte": end_date, "gte": start_date} - @overrides async def get_search( self, + request: Request, collections: Optional[List[str]] = None, ids: Optional[List[str]] = None, bbox: Optional[List[NumType]] = None, @@ -287,8 +289,8 @@ async def get_search( fields: Optional[List[str]] = None, sortby: Optional[str] = None, intersects: Optional[str] = None, - # filter: Optional[str] = None, # todo: requires fastapi > 2.3 unreleased - # filter_lang: Optional[str] = None, # todo: requires fastapi > 2.3 unreleased + filter: Optional[str] = None, + filter_lang: Optional[str] = None, **kwargs, ) -> ItemCollection: """Get search results from the database. @@ -318,9 +320,17 @@ async def get_search( "bbox": bbox, "limit": limit, "token": token, - "query": json.loads(query) if query else query, + "query": orjson.loads(query) if query else query, } + # this is borrowed from stac-fastapi-pgstac + # Kludgy fix because using factory does not allow alias for filter-lan + query_params = str(request.query_params) + if filter_lang is None: + match = re.search(r"filter-lang=([a-z0-9-]+)", query_params, re.IGNORECASE) + if match: + filter_lang = match.group(1) + if datetime: base_args["datetime"] = datetime @@ -328,7 +338,6 @@ async def get_search( base_args["intersects"] = intersects if sortby: - # https://github.com/radiantearth/stac-spec/tree/master/api-spec/extensions/sort#http-get-or-post-form sort_param = [] for sort in sortby: sort_param.append( @@ -339,12 +348,13 @@ async def get_search( ) base_args["sortby"] = sort_param - # todo: requires fastapi > 2.3 unreleased - # if filter: - # if filter_lang == "cql2-text": - # base_args["filter-lang"] = "cql2-json" - # base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter))) - # print(f'>>> {base_args["filter"]}') + if filter: + if filter_lang == "cql2-text": + base_args["filter-lang"] = "cql2-json" + base_args["filter"] = orjson.loads(to_cql2(parse_cql2_text(filter))) + else: + base_args["filter-lang"] = "cql2-json" + base_args["filter"] = orjson.loads(unquote_plus(filter)) if fields: includes = set() @@ -363,13 +373,12 @@ async def get_search( search_request = self.post_request_model(**base_args) except ValidationError: raise HTTPException(status_code=400, detail="Invalid parameters provided") - resp = await self.post_search(search_request, request=kwargs["request"]) + resp = await self.post_search(search_request=search_request, request=request) return resp - @overrides async def post_search( - self, search_request: BaseSearchPostRequest, **kwargs + self, search_request: BaseSearchPostRequest, request: Request ) -> ItemCollection: """ Perform a POST search on the catalog. @@ -384,7 +393,6 @@ async def post_search( Raises: HTTPException: If there is an error with the cql2_json filter. """ - request: Request = kwargs["request"] base_url = str(request.base_url) search = self.database.make_search() @@ -471,7 +479,7 @@ async def post_search( filter_kwargs = search_request.fields.filter_fields items = [ - json.loads(stac_pydantic.Item(**feat).json(**filter_kwargs)) + orjson.loads(stac_pydantic.Item(**feat).json(**filter_kwargs)) for feat in items ] diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index 43aadf18..d9db48cd 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -3,10 +3,13 @@ from os import listdir from os.path import isfile, join +import pytest + THIS_DIR = os.path.dirname(os.path.abspath(__file__)) -async def test_search_filters(app_client, ctx): +@pytest.mark.asyncio +async def test_search_filters_post(app_client, ctx): filters = [] pwd = f"{THIS_DIR}/cql2" @@ -19,7 +22,18 @@ async def test_search_filters(app_client, ctx): assert resp.status_code == 200 -async def test_search_filter_extension_eq(app_client, ctx): +@pytest.mark.asyncio +async def test_search_filter_extension_eq_get(app_client, ctx): + resp = await app_client.get( + '/search?filter-lang=cql2-json&filter={"op":"=","args":[{"property":"id"},"test-item"]}' + ) + assert resp.status_code == 200 + resp_json = resp.json() + assert len(resp_json["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_extension_eq_post(app_client, ctx): params = {"filter": {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}} resp = await app_client.post("/search", json=params) assert resp.status_code == 200 @@ -27,7 +41,26 @@ async def test_search_filter_extension_eq(app_client, ctx): assert len(resp_json["features"]) == 1 -async def test_search_filter_extension_gte(app_client, ctx): +@pytest.mark.asyncio +async def test_search_filter_extension_gte_get(app_client, ctx): + # there's one item that can match, so one of these queries should match it and the other shouldn't + resp = await app_client.get( + '/search?filter-lang=cql2-json&filter={"op":"<=","args":[{"property": "properties.proj:epsg"},32756]}' + ) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + resp = await app_client.get( + '/search?filter-lang=cql2-json&filter={"op":">","args":[{"property": "properties.proj:epsg"},32756]}' + ) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 0 + + +@pytest.mark.asyncio +async def test_search_filter_extension_gte_post(app_client, ctx): # there's one item that can match, so one of these queries should match it and the other shouldn't params = { "filter": { @@ -58,7 +91,53 @@ async def test_search_filter_extension_gte(app_client, ctx): assert len(resp.json()["features"]) == 0 -async def test_search_filter_ext_and(app_client, ctx): +@pytest.mark.asyncio +async def test_search_filter_ext_and_get(app_client, ctx): + resp = await app_client.get( + '/search?filter-lang=cql2-json&filter={"op":"and","args":[{"op":"<=","args":[{"property":"properties.proj:epsg"},32756]},{"op":"=","args":[{"property":"id"},"test-item"]}]}' + ) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_ext_and_get_cql2text_id(app_client, ctx): + collection = ctx.item["collection"] + id = ctx.item["id"] + filter = f"id='{id}' AND collection='{collection}'" + resp = await app_client.get(f"/search?filter-lang=cql2-text&filter={filter}") + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_ext_and_get_cql2text_cloud_cover(app_client, ctx): + collection = ctx.item["collection"] + cloud_cover = ctx.item["properties"]["eo:cloud_cover"] + filter = f"cloud_cover={cloud_cover} AND collection='{collection}'" + resp = await app_client.get(f"/search?filter-lang=cql2-text&filter={filter}") + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + +@pytest.mark.asyncio +async def test_search_filter_ext_and_get_cql2text_cloud_cover_no_results( + app_client, ctx +): + collection = ctx.item["collection"] + cloud_cover = ctx.item["properties"]["eo:cloud_cover"] + 1 + filter = f"cloud_cover={cloud_cover} AND collection='{collection}'" + resp = await app_client.get(f"/search?filter-lang=cql2-text&filter={filter}") + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 0 + + +@pytest.mark.asyncio +async def test_search_filter_ext_and_post(app_client, ctx): params = { "filter": { "op": "and", @@ -80,7 +159,32 @@ async def test_search_filter_ext_and(app_client, ctx): assert len(resp.json()["features"]) == 1 -async def test_search_filter_extension_floats(app_client, ctx): +@pytest.mark.asyncio +async def test_search_filter_extension_floats_get(app_client, ctx): + resp = await app_client.get( + """/search?filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30891534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30691534"]}]}""" + ) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 1 + + resp = await app_client.get( + """/search?filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item-7"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30891534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30691534"]}]}""" + ) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 0 + + resp = await app_client.get( + """/search?filter={"op":"and","args":[{"op":"=","args":[{"property":"id"},"test-item"]},{"op":">","args":[{"property":"properties.view:sun_elevation"},"-37.30591534"]},{"op":"<","args":[{"property":"properties.view:sun_elevation"},"-37.30491534"]}]}""" + ) + + assert resp.status_code == 200 + assert len(resp.json()["features"]) == 0 + + +@pytest.mark.asyncio +async def test_search_filter_extension_floats_post(app_client, ctx): sun_elevation = ctx.item["properties"]["view:sun_elevation"] params = { From e67c8a57da2bcdad8e6270f3b39b69bc0a833a8f Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Mon, 23 Oct 2023 23:11:47 +0800 Subject: [PATCH 16/53] update changelog --- CHANGELOG.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 52d2701b..438a3af2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,10 +10,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148) - -### Added - - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) +- GET /search filter extension queries [#163](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/163) ### Changed ### Fixed From 1f26818876ada89a637f3e1a3b5dd884967bb523 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 24 Oct 2023 10:14:00 +0200 Subject: [PATCH 17/53] restore proj:centroid mapping as geo_point as it is supposed to be in WGS84 --- .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 0c27abe4..e59f4041 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -81,7 +81,7 @@ { "proj_centroid": { "match": "proj:centroid", - "mapping": {"type": "object", "enabled": False}, + "mapping": {"type": "geo_point"}, } }, { From 8912e814185e520d391062b5c065c337244307f1 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 24 Oct 2023 16:56:03 +0200 Subject: [PATCH 18/53] Remove reference to proj;centroid in CHANGELOG Co-authored-by: Phil Varner --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ca44270a..28088184 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) - Corrected the automatic converstion of float values to int when building Filter Clauses [#135](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/135) -- Do not index `proj:geometry` and `proj:centroid` fields as geo shapes [#154](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/154) +- Do not index `proj:geometry` field as geo_shape [#154](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/154) - Remove unsupported characters from Elasticsearch index names [#153](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/153) ## [v0.3.0] From 15e4b9c8d223d63f9a91534e3404a8fe66711ad2 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 4 Nov 2023 19:38:12 +0800 Subject: [PATCH 19/53] first pass --- .../stac_fastapi/elasticsearch/core.py | 69 +++++++++++++------ .../elasticsearch/database_logic.py | 25 +++++-- 2 files changed, 67 insertions(+), 27 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 776ac746..78034406 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -4,16 +4,17 @@ from datetime import datetime as datetime_type from datetime import timezone from typing import Any, Dict, List, Optional, Set, Type, Union -from urllib.parse import urljoin +from base64 import urlsafe_b64encode import attr import stac_pydantic from fastapi import HTTPException from overrides import overrides from pydantic import ValidationError +from starlette.requests import Request from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes -from starlette.requests import Request +from urllib.parse import urljoin from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -70,7 +71,12 @@ class CoreClient(AsyncBaseCoreClient): database = DatabaseLogic() @overrides - async def all_collections(self, **kwargs) -> Collections: + async def all_collections( + self, + limit: Optional[int] = 10, + token: Optional[str] = None, + **kwargs + ) -> Collections: """Read all collections from the database. Returns: @@ -80,30 +86,49 @@ async def all_collections(self, **kwargs) -> Collections: Raises: Exception: If any error occurs while reading the collections from the database. """ + request: Request = kwargs["request"] base_url = str(kwargs["request"].base_url) + hits = self.database.get_all_collections(limit=limit, token=token) + + next_search_after = None + next_link = None + if len(hits) == limit: + last_hit = hits[-1] + next_search_after = last_hit['sort'] + next_token = urlsafe_b64encode(','.join(map(str, next_search_after)).encode()).decode() + paging_links = PagingLinks( + next=next_token, request=request + ) + next_link = paging_links.link_next() + + links=[ + { + "rel": Relations.root.value, + "type": MimeTypes.json, + "href": base_url, + }, + { + "rel": Relations.parent.value, + "type": MimeTypes.json, + "href": base_url, + }, + { + "rel": Relations.self.value, + "type": MimeTypes.json, + "href": urljoin(base_url, "collections"), + } + ] + + if next_link: + links.append(next_link) + return Collections( collections=[ - self.collection_serializer.db_to_stac(c, base_url=base_url) - for c in await self.database.get_all_collections() - ], - links=[ - { - "rel": Relations.root.value, - "type": MimeTypes.json, - "href": base_url, - }, - { - "rel": Relations.parent.value, - "type": MimeTypes.json, - "href": base_url, - }, - { - "rel": Relations.self.value, - "type": MimeTypes.json, - "href": urljoin(base_url, "collections"), - }, + self.collection_serializer.db_to_stac(c["_source"], base_url=base_url) + for c in hits ], + links=links ) @overrides diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index e59f4041..5ab149cf 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -18,6 +18,7 @@ from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item + logger = logging.getLogger(__name__) NumType = Union[float, int] @@ -295,9 +296,17 @@ class DatabaseLogic: """CORE LOGIC""" - async def get_all_collections(self) -> Iterable[Dict[str, Any]]: + async def get_all_collections( + self, + token: Optional[str], + limit: int + ) -> Iterable[Dict[str, Any]]: """Retrieve a list of all collections from the database. + Args: + token (Optional[str]): The token used to return the next set of results. + limit (int): Number of results to return + Returns: collections (Iterable[Dict[str, Any]]): A list of dictionaries containing the source data for each collection. @@ -306,10 +315,16 @@ async def get_all_collections(self) -> Iterable[Dict[str, Any]]: with the `COLLECTIONS_INDEX` as the target index and `size=1000` to retrieve up to 1000 records. The result is a generator of dictionaries containing the source data for each collection. """ - # https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/65 - # collections should be paginated, but at least return more than the default 10 for now - collections = await self.client.search(index=COLLECTIONS_INDEX, size=1000) - return (c["_source"] for c in collections["hits"]["hits"]) + search_after = None + if token: + search_after = urlsafe_b64decode(token.encode()).decode().split(",") + collections = await self.client.search( + index=COLLECTIONS_INDEX, + search_after=search_after, + size=limit + ) + hits = collections["hits"]["hits"] + return hits async def get_one_item(self, collection_id: str, item_id: str) -> Dict: """Retrieve a single item from the database. From 4e69ba0bfb02e94ea06bfe6a56c39902cd8055fb Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 02:04:36 +0800 Subject: [PATCH 20/53] update es --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d17e3af2..1cad4dee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: elasticsearch: container_name: es-container - image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.1.3} + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.10.4} environment: ES_JAVA_OPTS: -Xms512m -Xmx1g volumes: From 158f256e73654b3830fc85f5822db989f05b5800 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 02:08:42 +0800 Subject: [PATCH 21/53] paginate, sort collections --- .../stac_fastapi/elasticsearch/core.py | 36 +++++++++---------- .../elasticsearch/database_logic.py | 13 +++---- 2 files changed, 21 insertions(+), 28 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 78034406..7490f942 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -1,20 +1,20 @@ """Item crud client.""" import json import logging +from base64 import urlsafe_b64encode from datetime import datetime as datetime_type from datetime import timezone from typing import Any, Dict, List, Optional, Set, Type, Union -from base64 import urlsafe_b64encode +from urllib.parse import urljoin import attr import stac_pydantic from fastapi import HTTPException from overrides import overrides from pydantic import ValidationError -from starlette.requests import Request from stac_pydantic.links import Relations from stac_pydantic.shared import MimeTypes -from urllib.parse import urljoin +from starlette.requests import Request from stac_fastapi.elasticsearch import serializers from stac_fastapi.elasticsearch.config import ElasticsearchSettings @@ -71,12 +71,7 @@ class CoreClient(AsyncBaseCoreClient): database = DatabaseLogic() @overrides - async def all_collections( - self, - limit: Optional[int] = 10, - token: Optional[str] = None, - **kwargs - ) -> Collections: + async def all_collections(self, **kwargs) -> Collections: """Read all collections from the database. Returns: @@ -89,20 +84,23 @@ async def all_collections( request: Request = kwargs["request"] base_url = str(kwargs["request"].base_url) - hits = self.database.get_all_collections(limit=limit, token=token) + limit = int(request.query_params["limit"]) if "limit" in request.query_params else 10 + token = request.query_params["token"] if "token" in request.query_params else None + + hits = await self.database.get_all_collections(limit=limit, token=token) next_search_after = None next_link = None if len(hits) == limit: last_hit = hits[-1] - next_search_after = last_hit['sort'] - next_token = urlsafe_b64encode(','.join(map(str, next_search_after)).encode()).decode() - paging_links = PagingLinks( - next=next_token, request=request - ) + next_search_after = last_hit["sort"] + next_token = urlsafe_b64encode( + ",".join(map(str, next_search_after)).encode() + ).decode() + paging_links = PagingLinks(next=next_token, request=request) next_link = paging_links.link_next() - links=[ + links = [ { "rel": Relations.root.value, "type": MimeTypes.json, @@ -117,18 +115,18 @@ async def all_collections( "rel": Relations.self.value, "type": MimeTypes.json, "href": urljoin(base_url, "collections"), - } + }, ] if next_link: links.append(next_link) - + return Collections( collections=[ self.collection_serializer.db_to_stac(c["_source"], base_url=base_url) for c in hits ], - links=links + links=links, ) @overrides diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 5ab149cf..e5bc9ca6 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -18,7 +18,6 @@ from stac_fastapi.types.errors import ConflictError, NotFoundError from stac_fastapi.types.stac import Collection, Item - logger = logging.getLogger(__name__) NumType = Union[float, int] @@ -297,31 +296,27 @@ class DatabaseLogic: """CORE LOGIC""" async def get_all_collections( - self, - token: Optional[str], - limit: int + self, token: Optional[str], limit: int ) -> Iterable[Dict[str, Any]]: """Retrieve a list of all collections from the database. Args: token (Optional[str]): The token used to return the next set of results. - limit (int): Number of results to return + limit (int): Number of results to return Returns: collections (Iterable[Dict[str, Any]]): A list of dictionaries containing the source data for each collection. Notes: The collections are retrieved from the Elasticsearch database using the `client.search` method, - with the `COLLECTIONS_INDEX` as the target index and `size=1000` to retrieve up to 1000 records. + with the `COLLECTIONS_INDEX` as the target index and `size=limit` to retrieve records. The result is a generator of dictionaries containing the source data for each collection. """ search_after = None if token: search_after = urlsafe_b64decode(token.encode()).decode().split(",") collections = await self.client.search( - index=COLLECTIONS_INDEX, - search_after=search_after, - size=limit + index=COLLECTIONS_INDEX, search_after=search_after, size=limit, sort={"id": {"order": "asc"}} ) hits = collections["hits"]["hits"] return hits From e562d97bc949e12c3265e701f8a76b016fc90505 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 02:10:54 +0800 Subject: [PATCH 22/53] pre-commit --- .../elasticsearch/stac_fastapi/elasticsearch/core.py | 10 ++++++++-- .../stac_fastapi/elasticsearch/database_logic.py | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 7490f942..8ab2dc25 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -84,8 +84,14 @@ async def all_collections(self, **kwargs) -> Collections: request: Request = kwargs["request"] base_url = str(kwargs["request"].base_url) - limit = int(request.query_params["limit"]) if "limit" in request.query_params else 10 - token = request.query_params["token"] if "token" in request.query_params else None + limit = ( + int(request.query_params["limit"]) + if "limit" in request.query_params + else 10 + ) + token = ( + request.query_params["token"] if "token" in request.query_params else None + ) hits = await self.database.get_all_collections(limit=limit, token=token) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index e5bc9ca6..df64cc3b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -316,7 +316,10 @@ async def get_all_collections( if token: search_after = urlsafe_b64decode(token.encode()).decode().split(",") collections = await self.client.search( - index=COLLECTIONS_INDEX, search_after=search_after, size=limit, sort={"id": {"order": "asc"}} + index=COLLECTIONS_INDEX, + search_after=search_after, + size=limit, + sort={"id": {"order": "asc"}}, ) hits = collections["hits"]["hits"] return hits From 650c0cfe5e1d90b01c40e2fd7daa03dd50dcdbc7 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 11:28:14 +0800 Subject: [PATCH 23/53] add query params to mock --- .../elasticsearch/tests/clients/test_elasticsearch.py | 1 + stac_fastapi/elasticsearch/tests/conftest.py | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py index e46d4d1f..33738335 100644 --- a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py +++ b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py @@ -212,6 +212,7 @@ async def test_feature_collection_insert( assert len(fc["features"]) >= 10 +@pytest.mark.asyncio async def test_landing_page_no_collection_title(ctx, core_client, txn_client, app): ctx.collection["id"] = "new_id" del ctx.collection["title"] diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index 0a24b1f8..4ef6ba0c 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -41,11 +41,16 @@ class MockRequest: base_url = "http://test-server" def __init__( - self, method: str = "GET", url: str = "XXXX", app: Optional[Any] = None + self, + method: str = "GET", + url: str = "XXXX", + app: Optional[Any] = None, + query_params: dict[str, Any] = {"limit": "10"}, ): self.method = method self.url = url self.app = app + self.query_params = query_params or {} class TestSettings(AsyncElasticsearchSettings): From 4842e69cea1108f1849df2a520b8285b78dbd02c Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 11:44:57 +0800 Subject: [PATCH 24/53] use Dict --- stac_fastapi/elasticsearch/tests/conftest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index 4ef6ba0c..51f14534 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -45,7 +45,7 @@ def __init__( method: str = "GET", url: str = "XXXX", app: Optional[Any] = None, - query_params: dict[str, Any] = {"limit": "10"}, + query_params: Dict[str, Any] = {"limit": "10"}, ): self.method = method self.url = url From a4db2cf477da485e726d47aebfae85afa90220c8 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 11:51:20 +0800 Subject: [PATCH 25/53] update es in cicd --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index 3bf7a999..caeae818 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -16,7 +16,7 @@ jobs: services: elasticsearch_8_svc: - image: docker.elastic.co/elasticsearch/elasticsearch:8.1.3 + image: docker.elastic.co/elasticsearch/elasticsearch:8.10.4 env: cluster.name: stac-cluster node.name: es01 From 90cccc99d1b779900d03d1b2dcb86ea7f4f7ed65 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 12:43:16 +0800 Subject: [PATCH 26/53] add test --- .../tests/resources/test_collection.py | 40 ++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/tests/resources/test_collection.py b/stac_fastapi/elasticsearch/tests/resources/test_collection.py index f37b36b0..72911db8 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_collection.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_collection.py @@ -1,5 +1,7 @@ import pystac - +import pytest +import uuid +from ..conftest import create_collection async def test_create_and_delete_collection(app_client, load_test_data): """Test creation and deletion of a collection""" @@ -70,3 +72,39 @@ async def test_returns_valid_collection(ctx, app_client): resp_json, root=mock_root, preserve_dict=False ) collection.validate() + +@pytest.mark.asyncio +async def test_pagination_collection(app_client, ctx, txn_client): + """Test collection pagination links""" + ids = [ctx.collection["id"]] + + # Ingest 5 collections + for _ in range(5): + ctx.collection["id"] = str(uuid.uuid4()) + await create_collection(txn_client, collection=ctx.collection) + ids.append(ctx.collection["id"]) + + # Paginate through all 6 collections with a limit of 1 (expecting 7 requests) + page = await app_client.get( + f"/collections", params={"limit": 1} + ) + + collection_ids = [] + idx = 0 + for idx in range(100): + page_data = page.json() + next_link = list(filter(lambda link: link["rel"] == "next", page_data["links"])) + if not next_link: + assert not page_data["collections"] + break + + assert len(page_data["collections"]) == 1 + collection_ids.append(page_data["collections"][0]["id"]) + + href = next_link[0]["href"][len("http://test-server") :] + page = await app_client.get(href) + + assert idx == len(ids) + + # Confirm we have paginated through all collections + assert not set(collection_ids) - set(ids) From be248a0daafa1e69a63556d82cedbb6ebea3fbea Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 12:45:33 +0800 Subject: [PATCH 27/53] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28088184..22753ca4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added - Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148) +- Pagination for /collections - GET all collections - route [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164) ### Added From 1e034ba54f07480a65a18eea1167ee7d378c3123 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 12:48:16 +0800 Subject: [PATCH 28/53] pre-commit --- CHANGELOG.md | 3 +++ .../elasticsearch/tests/resources/test_collection.py | 10 ++++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 22753ca4..8561c4e1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) ### Changed + +- Update elasticsearch version from 8.1.3 to 8.10.4 in cicd, gh actions [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164) + ### Fixed - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) diff --git a/stac_fastapi/elasticsearch/tests/resources/test_collection.py b/stac_fastapi/elasticsearch/tests/resources/test_collection.py index 72911db8..902eb613 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_collection.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_collection.py @@ -1,8 +1,11 @@ +import uuid + import pystac import pytest -import uuid + from ..conftest import create_collection + async def test_create_and_delete_collection(app_client, load_test_data): """Test creation and deletion of a collection""" test_collection = load_test_data("test_collection.json") @@ -73,6 +76,7 @@ async def test_returns_valid_collection(ctx, app_client): ) collection.validate() + @pytest.mark.asyncio async def test_pagination_collection(app_client, ctx, txn_client): """Test collection pagination links""" @@ -85,9 +89,7 @@ async def test_pagination_collection(app_client, ctx, txn_client): ids.append(ctx.collection["id"]) # Paginate through all 6 collections with a limit of 1 (expecting 7 requests) - page = await app_client.get( - f"/collections", params={"limit": 1} - ) + page = await app_client.get("/collections", params={"limit": 1}) collection_ids = [] idx = 0 From caaf55bb1f9303c77ee5d77cf69131a985681a32 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 14:06:13 +0800 Subject: [PATCH 29/53] update test(s) --- .../elasticsearch/tests/api/test_api.py | 26 ++++++++++ .../tests/clients/test_elasticsearch.py | 14 +++++ .../tests/extensions/test_filter.py | 7 +++ .../tests/resources/test_collection.py | 51 +++++++++++-------- .../tests/resources/test_conformance.py | 3 ++ .../tests/resources/test_item.py | 35 +++++++++++++ .../tests/resources/test_mgmt.py | 4 ++ 7 files changed, 120 insertions(+), 20 deletions(-) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index c8edca78..feee7997 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -2,6 +2,8 @@ import uuid from datetime import datetime, timedelta +import pytest + from ..conftest import create_collection, create_item ROUTES = { @@ -31,17 +33,20 @@ } +@pytest.mark.asyncio async def test_post_search_content_type(app_client, ctx): params = {"limit": 1} resp = await app_client.post("/search", json=params) assert resp.headers["content-type"] == "application/geo+json" +@pytest.mark.asyncio async def test_get_search_content_type(app_client, ctx): resp = await app_client.get("/search") assert resp.headers["content-type"] == "application/geo+json" +@pytest.mark.asyncio async def test_api_headers(app_client): resp = await app_client.get("/api") assert ( @@ -50,11 +55,13 @@ async def test_api_headers(app_client): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_router(app): api_routes = set([f"{list(route.methods)[0]} {route.path}" for route in app.routes]) assert len(api_routes - ROUTES) == 0 +@pytest.mark.asyncio async def test_app_transaction_extension(app_client, ctx): item = copy.deepcopy(ctx.item) item["id"] = str(uuid.uuid4()) @@ -64,6 +71,7 @@ async def test_app_transaction_extension(app_client, ctx): await app_client.delete(f"/collections/{item['collection']}/items/{item['id']}") +@pytest.mark.asyncio async def test_app_search_response(app_client, ctx): resp = await app_client.get("/search", params={"ids": ["test-item"]}) assert resp.status_code == 200 @@ -75,6 +83,7 @@ async def test_app_search_response(app_client, ctx): assert resp_json.get("stac_extensions") is None +@pytest.mark.asyncio async def test_app_context_extension(app_client, ctx, txn_client): test_item = ctx.item test_item["id"] = "test-item-2" @@ -108,6 +117,7 @@ async def test_app_context_extension(app_client, ctx, txn_client): assert matched == 1 +@pytest.mark.asyncio async def test_app_fields_extension(app_client, ctx, txn_client): resp = await app_client.get("/search", params={"collections": ["test-collection"]}) assert resp.status_code == 200 @@ -115,6 +125,7 @@ async def test_app_fields_extension(app_client, ctx, txn_client): assert list(resp_json["features"][0]["properties"]) == ["datetime"] +@pytest.mark.asyncio async def test_app_fields_extension_query(app_client, ctx, txn_client): resp = await app_client.post( "/search", @@ -128,6 +139,7 @@ async def test_app_fields_extension_query(app_client, ctx, txn_client): assert list(resp_json["features"][0]["properties"]) == ["datetime", "proj:epsg"] +@pytest.mark.asyncio async def test_app_fields_extension_no_properties_get(app_client, ctx, txn_client): resp = await app_client.get( "/search", params={"collections": ["test-collection"], "fields": "-properties"} @@ -137,6 +149,7 @@ async def test_app_fields_extension_no_properties_get(app_client, ctx, txn_clien assert "properties" not in resp_json["features"][0] +@pytest.mark.asyncio async def test_app_fields_extension_no_properties_post(app_client, ctx, txn_client): resp = await app_client.post( "/search", @@ -150,6 +163,7 @@ async def test_app_fields_extension_no_properties_post(app_client, ctx, txn_clie assert "properties" not in resp_json["features"][0] +@pytest.mark.asyncio async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_client): item = ctx.item resp = await app_client.get( @@ -166,6 +180,7 @@ async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_c assert feature["properties"][expected_prop] == expected_value +@pytest.mark.asyncio async def test_app_query_extension_gt(app_client, ctx): params = {"query": {"proj:epsg": {"gt": ctx.item["properties"]["proj:epsg"]}}} resp = await app_client.post("/search", json=params) @@ -174,6 +189,7 @@ async def test_app_query_extension_gt(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_app_query_extension_gte(app_client, ctx): params = {"query": {"proj:epsg": {"gte": ctx.item["properties"]["proj:epsg"]}}} resp = await app_client.post("/search", json=params) @@ -182,22 +198,26 @@ async def test_app_query_extension_gte(app_client, ctx): assert len(resp.json()["features"]) == 1 +@pytest.mark.asyncio async def test_app_query_extension_limit_lt0(app_client): assert (await app_client.post("/search", json={"limit": -1})).status_code == 400 +@pytest.mark.asyncio async def test_app_query_extension_limit_gt10000(app_client): resp = await app_client.post("/search", json={"limit": 10001}) assert resp.status_code == 200 assert resp.json()["context"]["limit"] == 10000 +@pytest.mark.asyncio async def test_app_query_extension_limit_10000(app_client): params = {"limit": 10000} resp = await app_client.post("/search", json=params) assert resp.status_code == 200 +@pytest.mark.asyncio async def test_app_sort_extension(app_client, txn_client, ctx): first_item = ctx.item item_date = datetime.strptime( @@ -223,6 +243,7 @@ async def test_app_sort_extension(app_client, txn_client, ctx): assert resp_json["features"][1]["id"] == second_item["id"] +@pytest.mark.asyncio async def test_search_invalid_date(app_client, ctx): params = { "datetime": "2020-XX-01/2020-10-30", @@ -233,6 +254,7 @@ async def test_search_invalid_date(app_client, ctx): assert resp.status_code == 400 +@pytest.mark.asyncio async def test_search_point_intersects(app_client, ctx): point = [150.04, -33.14] intersects = {"type": "Point", "coordinates": point} @@ -248,6 +270,7 @@ async def test_search_point_intersects(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_point_does_not_intersect(app_client, ctx): point = [15.04, -3.14] intersects = {"type": "Point", "coordinates": point} @@ -263,6 +286,7 @@ async def test_search_point_does_not_intersect(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_datetime_non_interval(app_client, ctx): dt_formats = [ "2020-02-12T12:30:22+00:00", @@ -284,6 +308,7 @@ async def test_datetime_non_interval(app_client, ctx): assert resp_json["features"][0]["properties"]["datetime"][0:19] == dt[0:19] +@pytest.mark.asyncio async def test_bbox_3d(app_client, ctx): australia_bbox = [106.343365, -47.199523, 0.1, 168.218365, -19.437288, 0.1] params = { @@ -296,6 +321,7 @@ async def test_bbox_3d(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_line_string_intersects(app_client, ctx): line = [[150.04, -33.14], [150.22, -33.89]] intersects = {"type": "LineString", "coordinates": line} diff --git a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py index 33738335..3da8f86d 100644 --- a/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py +++ b/stac_fastapi/elasticsearch/tests/clients/test_elasticsearch.py @@ -11,6 +11,7 @@ from ..conftest import MockRequest, create_item +@pytest.mark.asyncio async def test_create_collection(app_client, ctx, core_client, txn_client): in_coll = deepcopy(ctx.collection) in_coll["id"] = str(uuid.uuid4()) @@ -20,6 +21,7 @@ async def test_create_collection(app_client, ctx, core_client, txn_client): await txn_client.delete_collection(in_coll["id"]) +@pytest.mark.asyncio async def test_create_collection_already_exists(app_client, ctx, txn_client): data = deepcopy(ctx.collection) @@ -32,6 +34,7 @@ async def test_create_collection_already_exists(app_client, ctx, txn_client): await txn_client.delete_collection(data["id"]) +@pytest.mark.asyncio async def test_update_collection( core_client, txn_client, @@ -49,6 +52,7 @@ async def test_update_collection( await txn_client.delete_collection(data["id"]) +@pytest.mark.asyncio async def test_delete_collection( core_client, txn_client, @@ -63,6 +67,7 @@ async def test_delete_collection( await core_client.get_collection(data["id"], request=MockRequest) +@pytest.mark.asyncio async def test_get_collection( core_client, txn_client, @@ -76,6 +81,7 @@ async def test_get_collection( await txn_client.delete_collection(data["id"]) +@pytest.mark.asyncio async def test_get_item(app_client, ctx, core_client): got_item = await core_client.get_item( item_id=ctx.item["id"], @@ -86,6 +92,7 @@ async def test_get_item(app_client, ctx, core_client): assert got_item["collection"] == ctx.item["collection"] +@pytest.mark.asyncio async def test_get_collection_items(app_client, ctx, core_client, txn_client): coll = ctx.collection num_of_items_to_create = 5 @@ -106,6 +113,7 @@ async def test_get_collection_items(app_client, ctx, core_client, txn_client): assert item["collection"] == coll["id"] +@pytest.mark.asyncio async def test_create_item(ctx, core_client, txn_client): resp = await core_client.get_item( ctx.item["id"], ctx.item["collection"], request=MockRequest @@ -115,6 +123,7 @@ async def test_create_item(ctx, core_client, txn_client): ) == Item(**resp).dict(exclude={"links": ..., "properties": {"created", "updated"}}) +@pytest.mark.asyncio async def test_create_item_already_exists(ctx, txn_client): with pytest.raises(ConflictError): await txn_client.create_item( @@ -125,6 +134,7 @@ async def test_create_item_already_exists(ctx, txn_client): ) +@pytest.mark.asyncio async def test_update_item(ctx, core_client, txn_client): ctx.item["properties"]["foo"] = "bar" collection_id = ctx.item["collection"] @@ -139,6 +149,7 @@ async def test_update_item(ctx, core_client, txn_client): assert updated_item["properties"]["foo"] == "bar" +@pytest.mark.asyncio async def test_update_geometry(ctx, core_client, txn_client): new_coordinates = [ [ @@ -163,6 +174,7 @@ async def test_update_geometry(ctx, core_client, txn_client): assert updated_item["geometry"]["coordinates"] == new_coordinates +@pytest.mark.asyncio async def test_delete_item(ctx, core_client, txn_client): await txn_client.delete_item(ctx.item["id"], ctx.item["collection"]) @@ -172,6 +184,7 @@ async def test_delete_item(ctx, core_client, txn_client): ) +@pytest.mark.asyncio async def test_bulk_item_insert(ctx, core_client, txn_client, bulk_txn_client): items = {} for _ in range(10): @@ -193,6 +206,7 @@ async def test_bulk_item_insert(ctx, core_client, txn_client, bulk_txn_client): # ) +@pytest.mark.asyncio async def test_feature_collection_insert( core_client, txn_client, diff --git a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py index 43aadf18..f20ab4ea 100644 --- a/stac_fastapi/elasticsearch/tests/extensions/test_filter.py +++ b/stac_fastapi/elasticsearch/tests/extensions/test_filter.py @@ -3,9 +3,12 @@ from os import listdir from os.path import isfile, join +import pytest + THIS_DIR = os.path.dirname(os.path.abspath(__file__)) +@pytest.mark.asyncio async def test_search_filters(app_client, ctx): filters = [] @@ -19,6 +22,7 @@ async def test_search_filters(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_search_filter_extension_eq(app_client, ctx): params = {"filter": {"op": "=", "args": [{"property": "id"}, ctx.item["id"]]}} resp = await app_client.post("/search", json=params) @@ -27,6 +31,7 @@ async def test_search_filter_extension_eq(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_filter_extension_gte(app_client, ctx): # there's one item that can match, so one of these queries should match it and the other shouldn't params = { @@ -58,6 +63,7 @@ async def test_search_filter_extension_gte(app_client, ctx): assert len(resp.json()["features"]) == 0 +@pytest.mark.asyncio async def test_search_filter_ext_and(app_client, ctx): params = { "filter": { @@ -80,6 +86,7 @@ async def test_search_filter_ext_and(app_client, ctx): assert len(resp.json()["features"]) == 1 +@pytest.mark.asyncio async def test_search_filter_extension_floats(app_client, ctx): sun_elevation = ctx.item["properties"]["view:sun_elevation"] diff --git a/stac_fastapi/elasticsearch/tests/resources/test_collection.py b/stac_fastapi/elasticsearch/tests/resources/test_collection.py index 902eb613..9061ac1e 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_collection.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_collection.py @@ -3,9 +3,10 @@ import pystac import pytest -from ..conftest import create_collection +from ..conftest import create_collection, delete_collections_and_items, refresh_indices +@pytest.mark.asyncio async def test_create_and_delete_collection(app_client, load_test_data): """Test creation and deletion of a collection""" test_collection = load_test_data("test_collection.json") @@ -18,6 +19,7 @@ async def test_create_and_delete_collection(app_client, load_test_data): assert resp.status_code == 204 +@pytest.mark.asyncio async def test_create_collection_conflict(app_client, ctx): """Test creation of a collection which already exists""" # This collection ID is created in the fixture, so this should be a conflict @@ -25,12 +27,14 @@ async def test_create_collection_conflict(app_client, ctx): assert resp.status_code == 409 +@pytest.mark.asyncio async def test_delete_missing_collection(app_client): """Test deletion of a collection which does not exist""" resp = await app_client.delete("/collections/missing-collection") assert resp.status_code == 404 +@pytest.mark.asyncio async def test_update_collection_already_exists(ctx, app_client): """Test updating a collection which already exists""" ctx.collection["keywords"].append("test") @@ -43,6 +47,7 @@ async def test_update_collection_already_exists(ctx, app_client): assert "test" in resp_json["keywords"] +@pytest.mark.asyncio async def test_update_new_collection(app_client, load_test_data): """Test updating a collection which does not exist (same as creation)""" test_collection = load_test_data("test_collection.json") @@ -52,12 +57,14 @@ async def test_update_new_collection(app_client, load_test_data): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_collection_not_found(app_client): """Test read a collection which does not exist""" resp = await app_client.get("/collections/does-not-exist") assert resp.status_code == 404 +@pytest.mark.asyncio async def test_returns_valid_collection(ctx, app_client): """Test validates fetched collection with jsonschema""" resp = await app_client.put("/collections", json=ctx.collection) @@ -80,33 +87,37 @@ async def test_returns_valid_collection(ctx, app_client): @pytest.mark.asyncio async def test_pagination_collection(app_client, ctx, txn_client): """Test collection pagination links""" - ids = [ctx.collection["id"]] - # Ingest 5 collections - for _ in range(5): + # Clear existing collections if necessary + await delete_collections_and_items(txn_client) + + # Ingest 6 collections + ids = set() + for _ in range(6): ctx.collection["id"] = str(uuid.uuid4()) await create_collection(txn_client, collection=ctx.collection) - ids.append(ctx.collection["id"]) + ids.add(ctx.collection["id"]) - # Paginate through all 6 collections with a limit of 1 (expecting 7 requests) - page = await app_client.get("/collections", params={"limit": 1}) + await refresh_indices(txn_client) - collection_ids = [] - idx = 0 - for idx in range(100): + # Paginate through all 6 collections with a limit of 1 + collection_ids = set() + page = await app_client.get("/collections", params={"limit": 1}) + while True: page_data = page.json() - next_link = list(filter(lambda link: link["rel"] == "next", page_data["links"])) + assert ( + len(page_data["collections"]) <= 1 + ) # Each page should have 1 or 0 collections + collection_ids.update(coll["id"] for coll in page_data["collections"]) + + next_link = next( + (link for link in page_data["links"] if link["rel"] == "next"), None + ) if not next_link: - assert not page_data["collections"] - break - - assert len(page_data["collections"]) == 1 - collection_ids.append(page_data["collections"][0]["id"]) + break # No more pages - href = next_link[0]["href"][len("http://test-server") :] + href = next_link["href"][len("http://test-server") :] page = await app_client.get(href) - assert idx == len(ids) - # Confirm we have paginated through all collections - assert not set(collection_ids) - set(ids) + assert collection_ids == ids diff --git a/stac_fastapi/elasticsearch/tests/resources/test_conformance.py b/stac_fastapi/elasticsearch/tests/resources/test_conformance.py index ab70a00b..d93d8b81 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_conformance.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_conformance.py @@ -20,6 +20,7 @@ def get_link(landing_page, rel_type): ) +@pytest.mark.asyncio async def test_landing_page_health(response): """Test landing page""" assert response.status_code == 200 @@ -39,6 +40,7 @@ async def test_landing_page_health(response): ] +@pytest.mark.asyncio @pytest.mark.parametrize("rel_type,expected_media_type,expected_path", link_tests) async def test_landing_page_links( response_json, app_client, rel_type, expected_media_type, expected_path @@ -59,6 +61,7 @@ async def test_landing_page_links( # code here seems meaningless since it would be the same as if the endpoint did not exist. Once # https://github.com/stac-utils/stac-fastapi/pull/227 has been merged we can add this to the # parameterized tests above. +@pytest.mark.asyncio async def test_search_link(response_json): search_link = get_link(response_json, "search") diff --git a/stac_fastapi/elasticsearch/tests/resources/test_item.py b/stac_fastapi/elasticsearch/tests/resources/test_item.py index 76f38f79..2cd442f0 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_item.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_item.py @@ -23,6 +23,7 @@ def rfc3339_str_to_datetime(s: str) -> datetime: return ciso8601.parse_rfc3339(s) +@pytest.mark.asyncio async def test_create_and_delete_item(app_client, ctx, txn_client): """Test creation and deletion of a single item (transactions extension)""" @@ -46,6 +47,7 @@ async def test_create_and_delete_item(app_client, ctx, txn_client): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_create_item_conflict(app_client, ctx): """Test creation of an item which already exists (transactions extension)""" @@ -57,6 +59,7 @@ async def test_create_item_conflict(app_client, ctx): assert resp.status_code == 409 +@pytest.mark.asyncio async def test_delete_missing_item(app_client, load_test_data): """Test deletion of an item which does not exist (transactions extension)""" test_item = load_test_data("test_item.json") @@ -66,6 +69,7 @@ async def test_delete_missing_item(app_client, load_test_data): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_create_item_missing_collection(app_client, ctx): """Test creation of an item without a parent collection (transactions extension)""" ctx.item["collection"] = "stac_is_cool" @@ -75,6 +79,7 @@ async def test_create_item_missing_collection(app_client, ctx): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_create_uppercase_collection_with_item(app_client, ctx, txn_client): """Test creation of a collection and item with uppercase collection ID (transactions extension)""" collection_id = "UPPERCASE" @@ -87,6 +92,7 @@ async def test_create_uppercase_collection_with_item(app_client, ctx, txn_client assert resp.status_code == 200 +@pytest.mark.asyncio async def test_update_item_already_exists(app_client, ctx): """Test updating an item which already exists (transactions extension)""" @@ -106,6 +112,7 @@ async def test_update_item_already_exists(app_client, ctx): ) +@pytest.mark.asyncio async def test_update_new_item(app_client, ctx): """Test updating an item which does not exist (transactions extension)""" test_item = ctx.item @@ -118,6 +125,7 @@ async def test_update_new_item(app_client, ctx): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_update_item_missing_collection(app_client, ctx): """Test updating an item without a parent collection (transactions extension)""" # Try to update collection of the item @@ -128,6 +136,7 @@ async def test_update_item_missing_collection(app_client, ctx): assert resp.status_code == 404 +@pytest.mark.asyncio async def test_update_item_geometry(app_client, ctx): ctx.item["id"] = "update_test_item_1" @@ -162,6 +171,7 @@ async def test_update_item_geometry(app_client, ctx): assert resp.json()["geometry"]["coordinates"] == new_coordinates +@pytest.mark.asyncio async def test_get_item(app_client, ctx): """Test read an item by id (core)""" get_item = await app_client.get( @@ -170,6 +180,7 @@ async def test_get_item(app_client, ctx): assert get_item.status_code == 200 +@pytest.mark.asyncio async def test_returns_valid_item(app_client, ctx): """Test validates fetched item with jsonschema""" test_item = ctx.item @@ -186,6 +197,7 @@ async def test_returns_valid_item(app_client, ctx): item.validate() +@pytest.mark.asyncio async def test_get_item_collection(app_client, ctx, txn_client): """Test read an item collection (core)""" item_count = randint(1, 4) @@ -202,6 +214,7 @@ async def test_get_item_collection(app_client, ctx, txn_client): assert matched == item_count + 1 +@pytest.mark.asyncio async def test_item_collection_filter_bbox(app_client, ctx): item = ctx.item collection = item["collection"] @@ -223,6 +236,7 @@ async def test_item_collection_filter_bbox(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_item_collection_filter_datetime(app_client, ctx): item = ctx.item collection = item["collection"] @@ -272,6 +286,7 @@ async def test_pagination(app_client, load_test_data): assert second_page["context"]["returned"] == 3 +@pytest.mark.asyncio async def test_item_timestamps(app_client, ctx): """Test created and updated timestamps (common metadata)""" # start_time = now_to_rfc3339_str() @@ -300,6 +315,7 @@ async def test_item_timestamps(app_client, ctx): ) +@pytest.mark.asyncio async def test_item_search_by_id_post(app_client, ctx, txn_client): """Test POST search by item id (core)""" ids = ["test1", "test2", "test3"] @@ -315,6 +331,7 @@ async def test_item_search_by_id_post(app_client, ctx, txn_client): assert set([feat["id"] for feat in resp_json["features"]]) == set(ids) +@pytest.mark.asyncio async def test_item_search_spatial_query_post(app_client, ctx): """Test POST search with spatial query (core)""" test_item = ctx.item @@ -329,6 +346,7 @@ async def test_item_search_spatial_query_post(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio async def test_item_search_temporal_query_post(app_client, ctx): """Test POST search with single-tailed spatio-temporal query (core)""" @@ -347,6 +365,7 @@ async def test_item_search_temporal_query_post(app_client, ctx): assert resp_json["features"][0]["id"] == test_item["id"] +@pytest.mark.asyncio async def test_item_search_temporal_window_post(app_client, ctx): """Test POST search with two-tailed spatio-temporal query (core)""" test_item = ctx.item @@ -412,6 +431,7 @@ async def test_item_search_sort_post(app_client, load_test_data): ) +@pytest.mark.asyncio async def test_item_search_by_id_get(app_client, ctx, txn_client): """Test GET search by item id (core)""" ids = ["test1", "test2", "test3"] @@ -427,6 +447,7 @@ async def test_item_search_by_id_get(app_client, ctx, txn_client): assert set([feat["id"] for feat in resp_json["features"]]) == set(ids) +@pytest.mark.asyncio async def test_item_search_bbox_get(app_client, ctx): """Test GET search with spatial query (core)""" params = { @@ -439,6 +460,7 @@ async def test_item_search_bbox_get(app_client, ctx): assert resp_json["features"][0]["id"] == ctx.item["id"] +@pytest.mark.asyncio async def test_item_search_get_without_collections(app_client, ctx): """Test GET search without specifying collections""" @@ -449,6 +471,7 @@ async def test_item_search_get_without_collections(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_item_search_get_with_non_existent_collections(app_client, ctx): """Test GET search with non-existent collections""" @@ -457,6 +480,7 @@ async def test_item_search_get_with_non_existent_collections(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_item_search_temporal_window_get(app_client, ctx): """Test GET search with spatio-temporal query (core)""" test_item = ctx.item @@ -495,6 +519,7 @@ async def test_item_search_sort_get(app_client, ctx, txn_client): assert resp_json["features"][1]["id"] == second_item["id"] +@pytest.mark.asyncio async def test_item_search_post_without_collection(app_client, ctx): """Test POST search without specifying a collection""" test_item = ctx.item @@ -505,6 +530,7 @@ async def test_item_search_post_without_collection(app_client, ctx): assert resp.status_code == 200 +@pytest.mark.asyncio async def test_item_search_properties_es(app_client, ctx): """Test POST search with JSONB query (query extension)""" @@ -517,6 +543,7 @@ async def test_item_search_properties_es(app_client, ctx): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_item_search_properties_field(app_client): """Test POST search indexed field with query (query extension)""" @@ -528,6 +555,7 @@ async def test_item_search_properties_field(app_client): assert len(resp_json["features"]) == 0 +@pytest.mark.asyncio async def test_item_search_get_query_extension(app_client, ctx): """Test GET search with JSONB query (query extension)""" @@ -554,12 +582,14 @@ async def test_item_search_get_query_extension(app_client, ctx): ) +@pytest.mark.asyncio async def test_get_missing_item_collection(app_client): """Test reading a collection which does not exist""" resp = await app_client.get("/collections/invalid-collection/items") assert resp.status_code == 404 +@pytest.mark.asyncio async def test_pagination_item_collection(app_client, ctx, txn_client): """Test item collection pagination links (paging extension)""" ids = [ctx.item["id"]] @@ -596,6 +626,7 @@ async def test_pagination_item_collection(app_client, ctx, txn_client): assert not set(item_ids) - set(ids) +@pytest.mark.asyncio async def test_pagination_post(app_client, ctx, txn_client): """Test POST pagination (paging extension)""" ids = [ctx.item["id"]] @@ -631,6 +662,7 @@ async def test_pagination_post(app_client, ctx, txn_client): assert not set(item_ids) - set(ids) +@pytest.mark.asyncio async def test_pagination_token_idempotent(app_client, ctx, txn_client): """Test that pagination tokens are idempotent (paging extension)""" ids = [ctx.item["id"]] @@ -661,6 +693,7 @@ async def test_pagination_token_idempotent(app_client, ctx, txn_client): ] +@pytest.mark.asyncio async def test_field_extension_get_includes(app_client, ctx): """Test GET search with included fields (fields extension)""" test_item = ctx.item @@ -673,6 +706,7 @@ async def test_field_extension_get_includes(app_client, ctx): assert not set(feat_properties) - {"proj:epsg", "gsd", "datetime"} +@pytest.mark.asyncio async def test_field_extension_get_excludes(app_client, ctx): """Test GET search with included fields (fields extension)""" test_item = ctx.item @@ -686,6 +720,7 @@ async def test_field_extension_get_excludes(app_client, ctx): assert "gsd" not in resp_json["features"][0]["properties"].keys() +@pytest.mark.asyncio async def test_field_extension_post(app_client, ctx): """Test POST search with included and excluded fields (fields extension)""" test_item = ctx.item diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py index 9d2bc3dc..2b7d9728 100644 --- a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py +++ b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py @@ -1,3 +1,7 @@ +import pytest + + +@pytest.mark.asyncio async def test_ping_no_param(app_client): """ Test ping endpoint with a mocked client. From 81bb67beb1af7d5848baf4ffe1d5be9448ca9fce Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 23:01:53 +0800 Subject: [PATCH 30/53] add notes to readme --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index a38ce14a..6c7ae665 100644 --- a/README.md +++ b/README.md @@ -61,7 +61,19 @@ curl -X "POST" "http://localhost:8080/collections" \ ``` Note: this "Collections Transaction" behavior is not part of the STAC API, but may be soon. + + +## Collection pagination + +The collections route handles optional `limit` and `page` parameters. The `links` field that is +returned from the `/collections` route contains a `next` link that can be used to get the next page +of results. +```shell +curl -X "GET" "http://localhost:8080/collections?limit=1&page=2" \ + -H 'Content-Type: application/json; charset=utf-8' +}' +``` ## Testing From edcff23cfa8d8f8cb8379cebfde84171fcc23cd9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sun, 5 Nov 2023 23:08:07 +0800 Subject: [PATCH 31/53] fix readme --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 6c7ae665..5b4b7685 100644 --- a/README.md +++ b/README.md @@ -65,14 +65,12 @@ Note: this "Collections Transaction" behavior is not part of the STAC API, but m ## Collection pagination -The collections route handles optional `limit` and `page` parameters. The `links` field that is -returned from the `/collections` route contains a `next` link that can be used to get the next page -of results. +The collections route handles optional `limit` and `token` parameters. The `links` field that is +returned from the `/collections` route contains a `next` link with the token that can be used to +get the next page of results. ```shell -curl -X "GET" "http://localhost:8080/collections?limit=1&page=2" \ - -H 'Content-Type: application/json; charset=utf-8' -}' +curl -X "GET" "http://localhost:8080/collections?limit=1&token=example_token" ``` ## Testing From eade0b4176aec064b3b116003359e9255785025c Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 7 Nov 2023 14:36:38 +0100 Subject: [PATCH 32/53] add link to PR in CHANGELOG --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a4a7b84..c99c10f4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,7 +16,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) ### Changed -- Use aliases on Elasticsearch indices, add number suffix in index name. + +- Use aliases on Elasticsearch indices, add number suffix in index name. [#152](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/152) + ### Fixed - Corrected the closing of client connections in ES index management functions [#132](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/132) From 89fb57d94246441e9ac2a4e4ce15753e8e136a00 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 7 Nov 2023 19:01:41 +0300 Subject: [PATCH 33/53] ues orjson --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index f850c193..e4aa5846 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -335,7 +335,7 @@ async def get_search( base_args["datetime"] = datetime if intersects: - base_args["intersects"] = json.loads(unquote_plus(intersects)) + base_args["intersects"] = orjson.loads(unquote_plus(intersects)) if sortby: sort_param = [] From 97814f41986b9dc101f2f02d6c6de14615fa46c9 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 7 Nov 2023 19:26:43 +0300 Subject: [PATCH 34/53] pre-commit --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 537f20c1..058be6e3 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -1,7 +1,7 @@ """Item crud client.""" import logging -from base64 import urlsafe_b64encode import re +from base64 import urlsafe_b64encode from datetime import datetime as datetime_type from datetime import timezone from typing import Any, Dict, List, Optional, Set, Type, Union From 093b5d01a0e1c6e694188fca83c551e2422e61ad Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 7 Nov 2023 20:11:01 +0300 Subject: [PATCH 35/53] create v1.0.0 --- CHANGELOG.md | 12 +++++++++++- VERSION | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b11ee3c..387542ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,15 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Added +### Changed + +### Fixed + + +## [v1.0.0] + +### Added + - Collection-level Assets to the CollectionSerializer [#148](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/148) - Pagination for /collections - GET all collections - route [#164](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/164) - Examples folder with example docker setup for running sfes from pip [#147](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/147) @@ -79,7 +88,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. -[Unreleased]: +[Unreleased]: +[v1.0.0]: [v0.3.0]: [v0.2.0]: [v0.1.0]: diff --git a/VERSION b/VERSION index 9325c3cc..afaf360d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.0 \ No newline at end of file +1.0.0 \ No newline at end of file From ba02eb8a6af12fb1809f7add95d4e2eeb39f0de0 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 8 Nov 2023 16:57:17 +0800 Subject: [PATCH 36/53] update version --- .../elasticsearch/stac_fastapi/elasticsearch/version.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py index 68999e11..1eeef171 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/version.py @@ -1,2 +1,2 @@ """library version.""" -__version__ = "0.3.0" +__version__ = "1.0.0" From 7b3c87e609db657b20fa4cb53a3e07c453da48d7 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 8 Nov 2023 17:01:28 +0800 Subject: [PATCH 37/53] change spacing --- CHANGELOG.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 387542ad..7f9226fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Remove unsupported characters from Elasticsearch index names [#153](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/153) - Fixed GET /search sortby requests [#25](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/25) + ## [v0.3.0] ### Added @@ -50,11 +51,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Fields Extension [#129](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/129) - Support for Python 3.11 [#131](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/131) - ### Changed - Updated core stac-fastapi libraries to 2.4.3 from 2.3.0 [#127](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/127) + ## [v0.2.0] ### Added @@ -77,6 +78,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - When using bulk ingest, items will continue to be ingested if any of them fail. Previously, the call would fail immediately if any items failed. + ## [v0.1.0] ### Changed @@ -88,6 +90,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Use genexp in execute_search and get_all_collections to return results. - Added db_to_stac serializer to item_collection method in core.py. + [Unreleased]: [v1.0.0]: [v0.3.0]: From 54186c6a7446b709c67dc8a3add4a8d54489714b Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Mon, 20 Nov 2023 16:01:14 +0100 Subject: [PATCH 38/53] #166: exclude unset fields in search response --- CHANGELOG.md | 1 + .../stac_fastapi/elasticsearch/core.py | 4 +++- .../elasticsearch/tests/api/test_api.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f9226fa..d50d6ff9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed +- Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166) ## [v1.0.0] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 65e9a458..76bb04cd 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -509,7 +509,9 @@ async def post_search( filter_kwargs = search_request.fields.filter_fields items = [ - orjson.loads(stac_pydantic.Item(**feat).json(**filter_kwargs)) + orjson.loads( + stac_pydantic.Item(**feat).json(**filter_kwargs, exclude_unset=True) + ) for feat in items ] diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index 932edc08..c50256da 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -163,6 +163,23 @@ async def test_app_fields_extension_no_properties_post(app_client, ctx, txn_clie assert "properties" not in resp_json["features"][0] +@pytest.mark.asyncio +async def test_app_fields_extension_no_null_fields(app_client, ctx, txn_client): + resp = await app_client.get("/search", params={"collections": ["test-collection"]}) + assert resp.status_code == 200 + resp_json = resp.json() + # check if no null fields: https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166 + for feature in resp_json["features"]: + # assert "bbox" not in feature["geometry"] + for link in feature["links"]: + assert all(a not in link or link[a] is not None for a in ("title", "asset")) + for asset in feature["assets"]: + assert all( + a not in asset or asset[a] is not None + for a in ("start_datetime", "created") + ) + + @pytest.mark.asyncio async def test_app_fields_extension_return_all_properties(app_client, ctx, txn_client): item = ctx.item From f94624733feb608a449f762083e999a045b526ce Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 21 Nov 2023 12:48:00 +0800 Subject: [PATCH 39/53] update es drivers to 8.11.0 --- stac_fastapi/elasticsearch/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 6ff8cd86..c0f941f0 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -13,8 +13,8 @@ "stac-fastapi.types==2.4.8", "stac-fastapi.api==2.4.8", "stac-fastapi.extensions==2.4.8", - "elasticsearch[async]==7.17.9", - "elasticsearch-dsl==7.4.1", + "elasticsearch[async]==8.11.0", + "elasticsearch-dsl==8.11.0", "pystac[validation]", "uvicorn", "orjson", From 9e4fa7a83432b9416b44bd524f13a05cca1a90e6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Tue, 21 Nov 2023 12:48:09 +0800 Subject: [PATCH 40/53] update changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d50d6ff9..66dd8d67 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Changed +- Elasticsearch drivers from 7.17.9 to 8.11.0 [#169](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/169) + ### Fixed - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166) From 05423b55179d93d6ae8b5697a4a39c91833ad2e1 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 12:48:30 +0800 Subject: [PATCH 41/53] update config --- .../stac_fastapi/elasticsearch/config.py | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index e1630d67..f6b04551 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -1,5 +1,6 @@ """API configuration.""" import os +import ssl from typing import Any, Dict, Set from elasticsearch import AsyncElasticsearch, Elasticsearch # type: ignore @@ -7,28 +8,37 @@ def _es_config() -> Dict[str, Any]: + # Determine the scheme (http or https) + use_ssl = os.getenv("ES_USE_SSL", "true").lower() == "true" + scheme = "https" if use_ssl else "http" + + # Configure the hosts parameter with the correct scheme + hosts = [f"{scheme}://{os.getenv('ES_HOST')}:{os.getenv('ES_PORT')}"] + + # Initialize the configuration dictionary config = { - "hosts": [{"host": os.getenv("ES_HOST"), "port": os.getenv("ES_PORT")}], - "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"}, - "use_ssl": True, - "verify_certs": True, + "hosts": hosts, + "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"} } - if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): - config["http_auth"] = (u, p) + # Explicitly exclude SSL settings when not using SSL + if not use_ssl: + return config - if (v := os.getenv("ES_USE_SSL")) and v == "false": - config["use_ssl"] = False + # Include SSL settings if using https + config["ssl_version"] = ssl.TLSVersion.TLSv1_2 + config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" - if (v := os.getenv("ES_VERIFY_CERTS")) and v == "false": - config["verify_certs"] = False + # Include CA Certificates if verifying certs + if config["verify_certs"]: + config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") - if v := os.getenv("CURL_CA_BUNDLE"): - config["ca_certs"] = v + # Handle authentication + if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): + config["http_auth"] = (u, p) return config - _forbidden_fields: Set[str] = {"type"} From eb586c6b37ecb54fa9be2e65d9daaf503a59e307 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 12:48:41 +0800 Subject: [PATCH 42/53] ues options --- .../stac_fastapi/elasticsearch/database_logic.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6b2cd433..da16b7bf 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -170,18 +170,12 @@ def indices(collection_ids: Optional[List[str]]) -> str: async def create_collection_index() -> None: - """Create the index for Collections in Elasticsearch. - - This function creates the Elasticsearch index for the `Collections` with the predefined mapping. - If the index already exists, the function ignores the error and continues execution. - """ client = AsyncElasticsearchSettings().create_client - await client.indices.create( + await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", aliases={COLLECTIONS_INDEX: {}}, - mappings=ES_COLLECTIONS_MAPPINGS, - ignore=400, # ignore 400 already exists code + mappings=ES_COLLECTIONS_MAPPINGS ) await client.close() @@ -200,12 +194,11 @@ async def create_item_index(collection_id: str): client = AsyncElasticsearchSettings().create_client index_name = index_by_collection_id(collection_id) - await client.indices.create( + await client.options(ignore_status=400).indices.create( index=f"{index_by_collection_id(collection_id)}-000001", aliases={index_name: {}}, mappings=ES_ITEMS_MAPPINGS, settings=ES_ITEMS_SETTINGS, - ignore=400, # ignore 400 already exists code ) await client.close() From 7a23ed7a4e8308b5d113cf0372eaefacdc916b90 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 12:48:48 +0800 Subject: [PATCH 43/53] update test --- stac_fastapi/elasticsearch/tests/api/test_api.py | 1 + 1 file changed, 1 insertion(+) diff --git a/stac_fastapi/elasticsearch/tests/api/test_api.py b/stac_fastapi/elasticsearch/tests/api/test_api.py index c50256da..74f0bb55 100644 --- a/stac_fastapi/elasticsearch/tests/api/test_api.py +++ b/stac_fastapi/elasticsearch/tests/api/test_api.py @@ -363,6 +363,7 @@ async def test_search_polygon_intersects_get(app_client, ctx): assert len(resp_json["features"]) == 1 +@pytest.mark.asyncio async def test_search_point_intersects_post(app_client, ctx): point = [150.04, -33.14] intersects = {"type": "Point", "coordinates": point} From 96904fa48ace47cad2dab50ad48f526547c5c630 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 13:01:28 +0800 Subject: [PATCH 44/53] pre-commit --- .../stac_fastapi/elasticsearch/config.py | 11 +++++++---- .../stac_fastapi/elasticsearch/database_logic.py | 9 ++++++++- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index f6b04551..ecfa8e4b 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -18,7 +18,7 @@ def _es_config() -> Dict[str, Any]: # Initialize the configuration dictionary config = { "hosts": hosts, - "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"} + "headers": {"accept": "application/vnd.elasticsearch+json; compatible-with=7"}, } # Explicitly exclude SSL settings when not using SSL @@ -26,12 +26,14 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_2 - config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" + config["ssl_version"] = ssl.TLSVersion.TLSv1_2 # type: ignore + config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore # Include CA Certificates if verifying certs if config["verify_certs"]: - config["ca_certs"] = os.getenv("CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt") + config["ca_certs"] = os.getenv( + "CURL_CA_BUNDLE", "/etc/ssl/certs/ca-certificates.crt" + ) # Handle authentication if (u := os.getenv("ES_USER")) and (p := os.getenv("ES_PASS")): @@ -39,6 +41,7 @@ def _es_config() -> Dict[str, Any]: return config + _forbidden_fields: Set[str] = {"type"} diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index da16b7bf..4554e74c 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -170,12 +170,19 @@ def indices(collection_ids: Optional[List[str]]) -> str: async def create_collection_index() -> None: + """ + Create the index for a Collection. + + Returns: + None + + """ client = AsyncElasticsearchSettings().create_client await client.options(ignore_status=400).indices.create( index=f"{COLLECTIONS_INDEX}-000001", aliases={COLLECTIONS_INDEX: {}}, - mappings=ES_COLLECTIONS_MAPPINGS + mappings=ES_COLLECTIONS_MAPPINGS, ) await client.close() From e7fd47de46f61941c097e8a55c366d23412565c7 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 13:02:02 +0800 Subject: [PATCH 45/53] es containers to 8.11.0 --- .github/workflows/cicd.yml | 2 +- docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index caeae818..e63b5097 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -16,7 +16,7 @@ jobs: services: elasticsearch_8_svc: - image: docker.elastic.co/elasticsearch/elasticsearch:8.10.4 + image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0 env: cluster.name: stac-cluster node.name: es01 diff --git a/docker-compose.yml b/docker-compose.yml index 1cad4dee..db3352fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,7 +31,7 @@ services: elasticsearch: container_name: es-container - image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.10.4} + image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTICSEARCH_VERSION:-8.11.0} environment: ES_JAVA_OPTS: -Xms512m -Xmx1g volumes: From 2efdd41470214e116c3c46512eb563c4b890ef85 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Wed, 22 Nov 2023 13:07:34 +0800 Subject: [PATCH 46/53] use TLSv1_3 --- stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py index ecfa8e4b..8634d3b9 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/config.py @@ -26,7 +26,7 @@ def _es_config() -> Dict[str, Any]: return config # Include SSL settings if using https - config["ssl_version"] = ssl.TLSVersion.TLSv1_2 # type: ignore + config["ssl_version"] = ssl.TLSVersion.TLSv1_3 # type: ignore config["verify_certs"] = os.getenv("ES_VERIFY_CERTS", "true").lower() != "false" # type: ignore # Include CA Certificates if verifying certs From 02b896a9812f831c4f61fd6d5dd1fb088da9e5c4 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 5 Dec 2023 09:53:17 +0100 Subject: [PATCH 47/53] upgrade stac-fastapi to v2.4.9, implement support for BulkTransaction method parameter --- stac_fastapi/elasticsearch/setup.py | 6 +++--- .../elasticsearch/stac_fastapi/elasticsearch/core.py | 12 +++++++----- .../stac_fastapi/elasticsearch/database_logic.py | 10 ++++++---- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/stac_fastapi/elasticsearch/setup.py b/stac_fastapi/elasticsearch/setup.py index 6ff8cd86..fa39a276 100644 --- a/stac_fastapi/elasticsearch/setup.py +++ b/stac_fastapi/elasticsearch/setup.py @@ -10,9 +10,9 @@ "attrs", "pydantic[dotenv]<2", "stac_pydantic==2.0.*", - "stac-fastapi.types==2.4.8", - "stac-fastapi.api==2.4.8", - "stac-fastapi.extensions==2.4.8", + "stac-fastapi.types==2.4.9", + "stac-fastapi.api==2.4.9", + "stac-fastapi.extensions==2.4.9", "elasticsearch[async]==7.17.9", "elasticsearch-dsl==7.4.1", "pystac[validation]", diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 76bb04cd..53940aa5 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -26,7 +26,7 @@ from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, - Items, + Items, BulkTransactionMethod, ) from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings @@ -568,7 +568,7 @@ async def create_item( if item["type"] == "FeatureCollection": bulk_client = BulkTransactionsClient() processed_items = [ - bulk_client.preprocess_item(item, base_url) for item in item["features"] # type: ignore + bulk_client.preprocess_item(item, base_url, BulkTransactionMethod.INSERT) for item in item["features"] # type: ignore ] await self.database.bulk_async( @@ -718,17 +718,19 @@ def __attrs_post_init__(self): settings = ElasticsearchSettings() self.client = settings.create_client - def preprocess_item(self, item: stac_types.Item, base_url) -> stac_types.Item: + def preprocess_item(self, item: stac_types.Item, base_url, method: BulkTransactionMethod) -> stac_types.Item: """Preprocess an item to match the data model. Args: item: The item to preprocess. base_url: The base URL of the request. + method: The bulk transaction method. Returns: The preprocessed item. """ - return self.database.sync_prep_create_item(item=item, base_url=base_url) + exist_ok = (method == BulkTransactionMethod.UPSERT) + return self.database.sync_prep_create_item(item=item, base_url=base_url, exist_ok=exist_ok) @overrides def bulk_item_insert( @@ -751,7 +753,7 @@ def bulk_item_insert( base_url = "" processed_items = [ - self.preprocess_item(item, base_url) for item in items.items.values() + self.preprocess_item(item, base_url, items.method) for item in items.items.values() ] # not a great way to get the collection_id-- should be part of the method signature diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 6b2cd433..d9cafa54 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -591,13 +591,14 @@ async def check_collection_exists(self, collection_id: str): if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): raise NotFoundError(f"Collection {collection_id} does not exist") - async def prep_create_item(self, item: Item, base_url: str) -> Item: + async def prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: """ Preps an item for insertion into the database. Args: item (Item): The item to be prepped for insertion. base_url (str): The base URL used to create the item's self URL. + exist_ok (bool): Indicates whether the item can exist already. Returns: Item: The prepped item. @@ -608,7 +609,7 @@ async def prep_create_item(self, item: Item, base_url: str) -> Item: """ await self.check_collection_exists(collection_id=item["collection"]) - if await self.client.exists( + if not exist_ok and await self.client.exists( index=index_by_collection_id(item["collection"]), id=mk_item_id(item["id"], item["collection"]), ): @@ -618,7 +619,7 @@ async def prep_create_item(self, item: Item, base_url: str) -> Item: return self.item_serializer.stac_to_db(item, base_url) - def sync_prep_create_item(self, item: Item, base_url: str) -> Item: + def sync_prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: """ Prepare an item for insertion into the database. @@ -629,6 +630,7 @@ def sync_prep_create_item(self, item: Item, base_url: str) -> Item: Args: item (Item): The item to be inserted into the database. base_url (str): The base URL used for constructing URLs for the item. + exist_ok (bool): Indicates whether the item can exist already. Returns: Item: The item after preparation is done. @@ -642,7 +644,7 @@ def sync_prep_create_item(self, item: Item, base_url: str) -> Item: if not self.sync_client.exists(index=COLLECTIONS_INDEX, id=collection_id): raise NotFoundError(f"Collection {collection_id} does not exist") - if self.sync_client.exists( + if not exist_ok and self.sync_client.exists( index=index_by_collection_id(collection_id), id=mk_item_id(item_id, collection_id), ): From 1281805bc86d04a761948a194a79870d840706d0 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 5 Dec 2023 09:55:28 +0100 Subject: [PATCH 48/53] fix formatting --- .../stac_fastapi/elasticsearch/core.py | 16 +++++++++++----- .../stac_fastapi/elasticsearch/database_logic.py | 8 ++++++-- 2 files changed, 17 insertions(+), 7 deletions(-) diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py index 53940aa5..4fb9f174 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/core.py @@ -26,7 +26,8 @@ from stac_fastapi.elasticsearch.session import Session from stac_fastapi.extensions.third_party.bulk_transactions import ( BaseBulkTransactionsClient, - Items, BulkTransactionMethod, + BulkTransactionMethod, + Items, ) from stac_fastapi.types import stac as stac_types from stac_fastapi.types.config import Settings @@ -718,7 +719,9 @@ def __attrs_post_init__(self): settings = ElasticsearchSettings() self.client = settings.create_client - def preprocess_item(self, item: stac_types.Item, base_url, method: BulkTransactionMethod) -> stac_types.Item: + def preprocess_item( + self, item: stac_types.Item, base_url, method: BulkTransactionMethod + ) -> stac_types.Item: """Preprocess an item to match the data model. Args: @@ -729,8 +732,10 @@ def preprocess_item(self, item: stac_types.Item, base_url, method: BulkTransacti Returns: The preprocessed item. """ - exist_ok = (method == BulkTransactionMethod.UPSERT) - return self.database.sync_prep_create_item(item=item, base_url=base_url, exist_ok=exist_ok) + exist_ok = method == BulkTransactionMethod.UPSERT + return self.database.sync_prep_create_item( + item=item, base_url=base_url, exist_ok=exist_ok + ) @overrides def bulk_item_insert( @@ -753,7 +758,8 @@ def bulk_item_insert( base_url = "" processed_items = [ - self.preprocess_item(item, base_url, items.method) for item in items.items.values() + self.preprocess_item(item, base_url, items.method) + for item in items.items.values() ] # not a great way to get the collection_id-- should be part of the method signature diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index d9cafa54..20d5c901 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -591,7 +591,9 @@ async def check_collection_exists(self, collection_id: str): if not await self.client.exists(index=COLLECTIONS_INDEX, id=collection_id): raise NotFoundError(f"Collection {collection_id} does not exist") - async def prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: + async def prep_create_item( + self, item: Item, base_url: str, exist_ok: bool = False + ) -> Item: """ Preps an item for insertion into the database. @@ -619,7 +621,9 @@ async def prep_create_item(self, item: Item, base_url: str, exist_ok: bool = Fal return self.item_serializer.stac_to_db(item, base_url) - def sync_prep_create_item(self, item: Item, base_url: str, exist_ok: bool = False) -> Item: + def sync_prep_create_item( + self, item: Item, base_url: str, exist_ok: bool = False + ) -> Item: """ Prepare an item for insertion into the database. From ea2294b147011518c1863c834b9279ed5db676e5 Mon Sep 17 00:00:00 2001 From: Stijn Caerts Date: Tue, 5 Dec 2023 10:01:03 +0100 Subject: [PATCH 49/53] update changelog --- CHANGELOG.md | 1 + .../elasticsearch/stac_fastapi/elasticsearch/database_logic.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d50d6ff9..2cd268b7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. ### Fixed - Exclude unset fields in search response [#166](https://github.com/stac-utils/stac-fastapi-elasticsearch/issues/166) +- Upgrade stac-fastapi to v2.4.9 [#172](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/172) ## [v1.0.0] diff --git a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py index 20d5c901..1e8c5333 100644 --- a/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py +++ b/stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py @@ -629,7 +629,7 @@ def sync_prep_create_item( This method performs pre-insertion preparation on the given `item`, such as checking if the collection the item belongs to exists, - and verifying that an item with the same ID does not already exist in the database. + and optionally verifying that an item with the same ID does not already exist in the database. Args: item (Item): The item to be inserted into the database. From 8ea51406d8f805ea95ec6b17c7abfc2b53bf5f08 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 6 Dec 2023 12:00:53 +0000 Subject: [PATCH 50/53] Bump actions/setup-python from 4 to 5 Bumps [actions/setup-python](https://github.com/actions/setup-python) from 4 to 5. - [Release notes](https://github.com/actions/setup-python/releases) - [Commits](https://github.com/actions/setup-python/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/setup-python dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/cicd.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cicd.yml b/.github/workflows/cicd.yml index caeae818..a7ce5f8b 100644 --- a/.github/workflows/cicd.yml +++ b/.github/workflows/cicd.yml @@ -58,7 +58,7 @@ jobs: # Setup Python (faster than using Python container) - name: Setup Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Lint code From 0e027411171bc47e21f3d0c7572d48a05fa9ff34 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 9 Dec 2023 11:37:40 +0800 Subject: [PATCH 51/53] remove test --- .../elasticsearch/tests/resources/test_mgmt.py | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 stac_fastapi/elasticsearch/tests/resources/test_mgmt.py diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py deleted file mode 100644 index 2b7d9728..00000000 --- a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py +++ /dev/null @@ -1,13 +0,0 @@ -import pytest - - -@pytest.mark.asyncio -async def test_ping_no_param(app_client): - """ - Test ping endpoint with a mocked client. - Args: - app_client (TestClient): mocked client fixture - """ - res = await app_client.get("/_mgmt/ping") - assert res.status_code == 200 - assert res.json() == {"message": "PONG"} From eef2b3b7d3e7a6671d571dbad4daec95cdf4bc65 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 9 Dec 2023 11:55:15 +0800 Subject: [PATCH 52/53] event loop --- stac_fastapi/elasticsearch/tests/conftest.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index f4b49928..9ddbf691 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -64,7 +64,14 @@ class Config: @pytest.fixture(scope="session") def event_loop(): + # asyncio.new_event_loop().run_until_complete() + + # loop = asyncio.new_event_loop() + # yield loop + # loop.close() + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) yield loop loop.close() From 4e3835c3457239497e0cd1a6f8cf6c14749507c6 Mon Sep 17 00:00:00 2001 From: jonhealy1 Date: Sat, 9 Dec 2023 12:01:46 +0800 Subject: [PATCH 53/53] add test back --- stac_fastapi/elasticsearch/tests/conftest.py | 6 ------ .../elasticsearch/tests/resources/test_mgmt.py | 13 +++++++++++++ 2 files changed, 13 insertions(+), 6 deletions(-) create mode 100644 stac_fastapi/elasticsearch/tests/resources/test_mgmt.py diff --git a/stac_fastapi/elasticsearch/tests/conftest.py b/stac_fastapi/elasticsearch/tests/conftest.py index 9ddbf691..fa093af2 100644 --- a/stac_fastapi/elasticsearch/tests/conftest.py +++ b/stac_fastapi/elasticsearch/tests/conftest.py @@ -64,12 +64,6 @@ class Config: @pytest.fixture(scope="session") def event_loop(): - # asyncio.new_event_loop().run_until_complete() - - # loop = asyncio.new_event_loop() - # yield loop - # loop.close() - loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) yield loop diff --git a/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py new file mode 100644 index 00000000..2b7d9728 --- /dev/null +++ b/stac_fastapi/elasticsearch/tests/resources/test_mgmt.py @@ -0,0 +1,13 @@ +import pytest + + +@pytest.mark.asyncio +async def test_ping_no_param(app_client): + """ + Test ping endpoint with a mocked client. + Args: + app_client (TestClient): mocked client fixture + """ + res = await app_client.get("/_mgmt/ping") + assert res.status_code == 200 + assert res.json() == {"message": "PONG"}