From 4935c30dbfabfc978f6f33ae52b0f73cef120133 Mon Sep 17 00:00:00 2001 From: Maaike Date: Fri, 22 Nov 2024 14:19:39 +0100 Subject: [PATCH] reduce preventable 404 on the wis2box-api --- .../wis2box/api/config/pygeoapi.py | 21 ++++----- .../wis2box/metadata/discovery.py | 16 ++++--- wis2box-management/wis2box/pubsub/message.py | 43 +++++++++---------- 3 files changed, 43 insertions(+), 37 deletions(-) diff --git a/wis2box-management/wis2box/api/config/pygeoapi.py b/wis2box-management/wis2box/api/config/pygeoapi.py index dcb89c4e..432d8867 100644 --- a/wis2box-management/wis2box/api/config/pygeoapi.py +++ b/wis2box-management/wis2box/api/config/pygeoapi.py @@ -81,10 +81,13 @@ def get_collection(self, name: str) -> dict: :returns: `dict` of collection configuration """ - r = self.http.get(f'{self.url}/{name}') - r.raise_for_status() - - return r.json() + LOGGER.debug(f'get_collection name={name}') + for key in self.list_collections(): + if key == name: + r = self.http.get(f'{self.url}/{name}') + r.raise_for_status() + return r.json() + LOGGER.debug(f'Collection {name} not found') def get_collection_data(self, name: str) -> dict: """ @@ -111,8 +114,10 @@ def add_collection(self, name: str, collection: dict) -> bool: """ if self.has_collection(name): + LOGGER.debug(f'Collection {name} already exists, updating') r = self.http.put(f'{self.url}/{name}', json=collection) else: + LOGGER.debug(f'Collection {name} does not exist, creating') content = {name: collection} r = self.http.post(self.url, json=content) @@ -139,12 +144,8 @@ def has_collection(self, name: str) -> bool: :returns: `bool` of collection result """ - - try: - r = self.http.get(f'{self.url}/{name}') - except Exception: - return False - return r.ok + LOGGER.debug(f'Check if collection {name} is in {[key for key in self.list_collections()]}') # noqa + return name in self.list_collections() def prepare_collection(self, meta: dict) -> bool: """ diff --git a/wis2box-management/wis2box/metadata/discovery.py b/wis2box-management/wis2box/metadata/discovery.py index 1e40085e..332275c7 100644 --- a/wis2box-management/wis2box/metadata/discovery.py +++ b/wis2box-management/wis2box/metadata/discovery.py @@ -235,7 +235,6 @@ def publish_discovery_metadata(metadata: Union[dict, str]): LOGGER.debug('Publishing discovery metadata') try: new_links = [] - if isinstance(metadata, dict): LOGGER.info('Adding WCMP2 record from dictionary') record = metadata @@ -266,14 +265,21 @@ def publish_discovery_metadata(metadata: Union[dict, str]): LOGGER.error(msg) raise RuntimeError(msg) + LOGGER.info('Checking if record / auth enabled') oar = Records(DOCKER_API_URL) try: - LOGGER.debug('Checking if record / auth enabled') - r = oar.collection_item('discovery-metadata', record['id']).json() - if r['wis2box'].get('has_auth', False): + records = oar.collection_items('discovery-metadata') + # find record in existing records + r = next((r for r in records['features'] if r['id'] == record['id']), None) # noqa + if r is None: + LOGGER.debug('Record not found in existing records') + elif r['wis2box'].get('has_auth', False): + LOGGER.debug('Auth enabled, adding to record') record['wis2box']['has_auth'] = True + else: + LOGGER.debug('No auth defined') except Exception: - LOGGER.debug('No auth defined') + LOGGER.error('Failed to check for auth') LOGGER.debug('Publishing to API') upsert_collection_item('discovery-metadata', record) diff --git a/wis2box-management/wis2box/pubsub/message.py b/wis2box-management/wis2box/pubsub/message.py index 1c8ac392..7a076c88 100644 --- a/wis2box-management/wis2box/pubsub/message.py +++ b/wis2box-management/wis2box/pubsub/message.py @@ -212,29 +212,28 @@ def __init__(self, identifier: str, metadata_id: str, filepath: str, } self.message['links'].append(link) - LOGGER.debug(f'Checking for access control (metadata id: {metadata_id})') # noqa - try: - oar = Records(DOCKER_API_URL) - record = oar.collection_item('discovery-metadata', metadata_id) - - if record['wis2box'].get('has_auth'): - LOGGER.debug('Updating message with access control') - - for link in self.message['links']: - if link['href'] == public_file_url: - LOGGER.debug('Adding security object to link') - link['security'] = { - 'default': { - 'type': 'http', - 'scheme': 'bearer', - 'description': 'Please contact the data provider for access' # noqa + # check if metadata record exists and has access control + if metadata_id is not None: + LOGGER.debug(f'Find metadata record with id={metadata_id}') + try: + oar = Records(DOCKER_API_URL) + record = oar.collection_item('discovery-metadata', metadata_id) + if record and record['wis2box'].get('has_auth'): + LOGGER.debug('Updating message with access control') + for link in self.message['links']: + if link['href'] == public_file_url: + LOGGER.debug('Adding security object to link') + link['security'] = { + 'default': { + 'type': 'http', + 'scheme': 'bearer', + 'description': 'Please contact the data provider for access' # noqa + } } - } - - LOGGER.debug('Removing inline content') - self.message['properties'].pop('content', None) - except Exception as err: - LOGGER.debug(f'Cannot locate metadata record: {err}') + LOGGER.debug('Removing inline content') + self.message['properties'].pop('content', None) + except Exception as err: + LOGGER.warning(f'No item with metadata_id={metadata_id}: {err}') # noqa def gcm() -> dict: