From e347e0f862f8b804f21f7e30dbf2af5fdb4a57e3 Mon Sep 17 00:00:00 2001 From: Moritz-Langer Date: Tue, 30 Jul 2024 14:48:45 +0200 Subject: [PATCH 1/7] Added changes for extra_params to itemtypes, oracle provider and the according tests. --- pygeoapi/api/itemtypes.py | 16 ++++++++++++---- pygeoapi/provider/oracle.py | 7 +++++++ tests/test_oracle_provider.py | 12 ++++++++++++ 3 files changed, 31 insertions(+), 4 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 1fa70c235..9f6409181 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -226,6 +226,8 @@ def get_collection_items( 'properties', 'skipGeometry', 'q', 'filter', 'filter-lang', 'filter-crs'] + extra_params = {} + collections = filter_dict_by_key_value(api.config['resources'], 'type', 'collection') @@ -386,9 +388,13 @@ def get_collection_items( LOGGER.debug('processing property parameters') for k, v in request.params.items(): - if k not in reserved_fieldnames and k in list(p.fields.keys()): - LOGGER.debug(f'Adding property filter {k}={v}') - properties.append((k, v)) + if k not in reserved_fieldnames: + if k in list(p.fields.keys()): + LOGGER.debug(f'Adding property filter {k}={v}') + properties.append((k, v)) + else: + LOGGER.debug(f'Adding extra filter {k}={v}') + extra_params[str(k).lower()] = v LOGGER.debug('processing sort parameter') val = request.params.get('sortby') @@ -479,6 +485,7 @@ def get_collection_items( LOGGER.debug(f'datetime: {datetime_}') LOGGER.debug(f'properties: {properties}') LOGGER.debug(f'select properties: {select_properties}') + LOGGER.debug(f'extra_params: {extra_params}') LOGGER.debug(f'skipGeometry: {skip_geometry}') LOGGER.debug(f'language: {prv_locale}') LOGGER.debug(f'q: {q}') @@ -494,7 +501,8 @@ def get_collection_items( sortby=sortby, skip_geometry=skip_geometry, select_properties=select_properties, crs_transform_spec=crs_transform_spec, - q=q, language=prv_locale, filterq=filter_) + q=q, language=prv_locale, filterq=filter_, + extra_params=extra_params) except ProviderGenericError as err: return api.get_exception( err.http_status_code, headers, request.format, diff --git a/pygeoapi/provider/oracle.py b/pygeoapi/provider/oracle.py index 4bedde773..b4c9dbb27 100644 --- a/pygeoapi/provider/oracle.py +++ b/pygeoapi/provider/oracle.py @@ -614,6 +614,7 @@ def query( q=None, language=None, filterq=None, + extra_params={}, **kwargs, ): """ @@ -630,9 +631,14 @@ def query( :param skip_geometry: bool of whether to skip geometry (default False) :param q: full-text search term(s) :param filterq: CQL query as text string + :param extra_params: Additional parameters added to the + query which are not in reserved + fieldnames or fields in the oracle table :returns: GeoJSON FeaturesCollection """ + LOGGER.debug(f"properties contains: {properties}") + LOGGER.debug(f"Extra Params contains: {extra_params}") # Check mandatory filter properties property_dict = dict(properties) @@ -790,6 +796,7 @@ def query( q, language, filterq, + extra_params=extra_params ) # Clean up placeholders that aren't used by the diff --git a/tests/test_oracle_provider.py b/tests/test_oracle_provider.py index d84b298e7..30d2e00f7 100644 --- a/tests/test_oracle_provider.py +++ b/tests/test_oracle_provider.py @@ -62,8 +62,11 @@ def process_query( q, language, filterq, + extra_params, ): sql = "ID = 10 AND :foo != :bar" + if extra_params.get("custom-auth") == "forbidden": + sql += " AND 'auth' = 'you arent allowed'" if sql_query.find(" WHERE ") == -1: sql_query = sql_query.replace("#WHERE#", f" WHERE {sql}") @@ -632,6 +635,15 @@ def test_query_mandatory_properties_must_be_specified(config): p.query(properties=[("id", "123")]) +def test_extra_params_are_passed_to_sql_manipulator(config_manipulator): + extra_params = {"custom-auth": "forbidden"} + + p = OracleProvider(config_manipulator) + response = p.query(extra_params=extra_params) + + assert not response['features'] + + @pytest.fixture() def database_connection_pool(config_db_conn): os.environ["ORACLE_POOL_MIN"] = "2" # noqa: F841 From 9f2e209b67be9732d30161610a2268099c1b5ec7 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 19 Aug 2024 12:52:22 +0200 Subject: [PATCH 2/7] Add extra params to properties Oracle provider still needs to be adapted to this change --- pygeoapi/api/itemtypes.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 9f6409181..c97ce108e 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -226,8 +226,6 @@ def get_collection_items( 'properties', 'skipGeometry', 'q', 'filter', 'filter-lang', 'filter-crs'] - extra_params = {} - collections = filter_dict_by_key_value(api.config['resources'], 'type', 'collection') @@ -390,11 +388,11 @@ def get_collection_items( for k, v in request.params.items(): if k not in reserved_fieldnames: if k in list(p.fields.keys()): - LOGGER.debug(f'Adding property filter {k}={v}') - properties.append((k, v)) + LOGGER.debug(f'Adding custom property filter {k}={v}') else: - LOGGER.debug(f'Adding extra filter {k}={v}') - extra_params[str(k).lower()] = v + LOGGER.debug('Adding property filter {k}={v}') + + properties.append((k, v)) LOGGER.debug('processing sort parameter') val = request.params.get('sortby') @@ -485,7 +483,6 @@ def get_collection_items( LOGGER.debug(f'datetime: {datetime_}') LOGGER.debug(f'properties: {properties}') LOGGER.debug(f'select properties: {select_properties}') - LOGGER.debug(f'extra_params: {extra_params}') LOGGER.debug(f'skipGeometry: {skip_geometry}') LOGGER.debug(f'language: {prv_locale}') LOGGER.debug(f'q: {q}') @@ -501,8 +498,7 @@ def get_collection_items( sortby=sortby, skip_geometry=skip_geometry, select_properties=select_properties, crs_transform_spec=crs_transform_spec, - q=q, language=prv_locale, filterq=filter_, - extra_params=extra_params) + q=q, language=prv_locale, filterq=filter_) except ProviderGenericError as err: return api.get_exception( err.http_status_code, headers, request.format, From 9685b3725c9f9d54643168e39653c6d6ccde2c5d Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 19 Aug 2024 13:16:50 +0200 Subject: [PATCH 3/7] Adapt oracle provider for new extra params behavior --- pygeoapi/provider/oracle.py | 17 ++++++++++++----- tests/test_oracle_provider.py | 4 ++-- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/pygeoapi/provider/oracle.py b/pygeoapi/provider/oracle.py index b4c9dbb27..87cd5fa1d 100644 --- a/pygeoapi/provider/oracle.py +++ b/pygeoapi/provider/oracle.py @@ -614,7 +614,6 @@ def query( q=None, language=None, filterq=None, - extra_params={}, **kwargs, ): """ @@ -631,14 +630,22 @@ def query( :param skip_geometry: bool of whether to skip geometry (default False) :param q: full-text search term(s) :param filterq: CQL query as text string - :param extra_params: Additional parameters added to the - query which are not in reserved - fieldnames or fields in the oracle table :returns: GeoJSON FeaturesCollection """ LOGGER.debug(f"properties contains: {properties}") - LOGGER.debug(f"Extra Params contains: {extra_params}") + + # NOTE: properties contains field keys plus extra params + # need to split them up here + filtered_properties = [] + extra_params = {} + for (key, value) in properties: + if key in self.fields.keys(): + filtered_properties.append((key, value)) + else: + extra_params[key] = value + + properties = filtered_properties # Check mandatory filter properties property_dict = dict(properties) diff --git a/tests/test_oracle_provider.py b/tests/test_oracle_provider.py index 30d2e00f7..bb24d0209 100644 --- a/tests/test_oracle_provider.py +++ b/tests/test_oracle_provider.py @@ -636,10 +636,10 @@ def test_query_mandatory_properties_must_be_specified(config): def test_extra_params_are_passed_to_sql_manipulator(config_manipulator): - extra_params = {"custom-auth": "forbidden"} + extra_params = [("custom-auth", "forbidden")] p = OracleProvider(config_manipulator) - response = p.query(extra_params=extra_params) + response = p.query(properties=extra_params) assert not response['features'] From 4e1745ede586b3c4f11e2568e7f2bcbebb5d7ccc Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 19 Aug 2024 13:48:58 +0200 Subject: [PATCH 4/7] Fix logging calls for additional properties --- pygeoapi/api/itemtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygeoapi/api/itemtypes.py b/pygeoapi/api/itemtypes.py index 6e5c22b42..661f5cd80 100644 --- a/pygeoapi/api/itemtypes.py +++ b/pygeoapi/api/itemtypes.py @@ -389,9 +389,9 @@ def get_collection_items( for k, v in request.params.items(): if k not in reserved_fieldnames: if k in list(p.fields.keys()): - LOGGER.debug(f'Adding custom property filter {k}={v}') + LOGGER.debug(f'Adding property filter {k}={v}') else: - LOGGER.debug('Adding property filter {k}={v}') + LOGGER.debug(f'Adding additional property filter {k}={v}') properties.append((k, v)) From b742a69ff62173717cf5c051fb5ce34e328f571d Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 19 Aug 2024 13:59:59 +0200 Subject: [PATCH 5/7] Remove trailing comma --- tests/test_oracle_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oracle_provider.py b/tests/test_oracle_provider.py index bb24d0209..5c03cf8b6 100644 --- a/tests/test_oracle_provider.py +++ b/tests/test_oracle_provider.py @@ -62,7 +62,7 @@ def process_query( q, language, filterq, - extra_params, + extra_params ): sql = "ID = 10 AND :foo != :bar" if extra_params.get("custom-auth") == "forbidden": From 79372f98b41cdb18afd140eb8aebc508c65c37a1 Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 19 Aug 2024 14:05:23 +0200 Subject: [PATCH 6/7] Fix grammar in test message --- tests/test_oracle_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oracle_provider.py b/tests/test_oracle_provider.py index 5c03cf8b6..40ff11d39 100644 --- a/tests/test_oracle_provider.py +++ b/tests/test_oracle_provider.py @@ -66,7 +66,7 @@ def process_query( ): sql = "ID = 10 AND :foo != :bar" if extra_params.get("custom-auth") == "forbidden": - sql += " AND 'auth' = 'you arent allowed'" + sql += " AND 'auth' = 'you are not allowed'" if sql_query.find(" WHERE ") == -1: sql_query = sql_query.replace("#WHERE#", f" WHERE {sql}") From 12dd8a98565e1e3d9e2cbb17715e8741bcafbf1f Mon Sep 17 00:00:00 2001 From: Bernhard Mallinger Date: Mon, 19 Aug 2024 14:09:06 +0200 Subject: [PATCH 7/7] Use f-string instead of plus for string manipulation --- tests/test_oracle_provider.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_oracle_provider.py b/tests/test_oracle_provider.py index 40ff11d39..64620ee0f 100644 --- a/tests/test_oracle_provider.py +++ b/tests/test_oracle_provider.py @@ -66,7 +66,7 @@ def process_query( ): sql = "ID = 10 AND :foo != :bar" if extra_params.get("custom-auth") == "forbidden": - sql += " AND 'auth' = 'you are not allowed'" + sql = f"{sql} AND 'auth' = 'you are not allowed'" if sql_query.find(" WHERE ") == -1: sql_query = sql_query.replace("#WHERE#", f" WHERE {sql}")