Skip to content

Commit

Permalink
Remove assert statements and replace with RuntimeError
Browse files Browse the repository at this point in the history
  • Loading branch information
@jochenklar
jochenklar committed Jan 3, 2024
1 parent 2c8a13f commit 008ae02
Show file tree
Hide file tree
Showing 4 changed files with 157 additions and 129 deletions.
31 changes: 20 additions & 11 deletions isimip_publisher/config.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,31 +28,35 @@ def setup(self, args):
self.ISIMIP_DATA_URL = self.ISIMIP_DATA_URL.rstrip('/')

if self.RIGHTS not in RIGHTS_CHOICES:
raise AssertionError('Incorrect rights "%s": choose from %s', self.RIGHTS, RIGHTS_CHOICES)
raise RuntimeError('Incorrect rights "%s": choose from %s', self.RIGHTS, RIGHTS_CHOICES)

try:
datetime.strptime(self.VERSION, '%Y%m%d')
except ValueError as e:
raise AssertionError("Incorrect version format, should be YYYYMMDD") from e
raise RuntimeError('Incorrect version format, should be YYYYMMDD') from e

@cached_property
def REMOTE_PATH(self):
assert self.REMOTE_DIR is not None, 'REMOTE_DIR is not set'
if self.REMOTE_DIR is None:
raise RuntimeError('REMOTE_DIR is not set')
return Path(self.REMOTE_DIR).expanduser()

@cached_property
def LOCAL_PATH(self):
assert self.LOCAL_DIR is not None, 'LOCAL_DIR is not set'
if self.LOCAL_DIR is None:
raise RuntimeError('LOCAL_DIR is not set')
return Path(self.LOCAL_DIR).expanduser()

@cached_property
def PUBLIC_PATH(self):
assert self.PUBLIC_DIR is not None, 'PUBLIC_DIR is not set'
if self.PUBLIC_DIR is None:
raise RuntimeError('PUBLIC_DIR is not set')
return Path(self.PUBLIC_DIR).expanduser()

@cached_property
def ARCHIVE_PATH(self):
assert self.ARCHIVE_DIR is not None, 'ARCHIVE_DIR is not set'
if self.ARCHIVE_DIR is None:
raise RuntimeError('ARCHIVE_DIR is not set')
return Path(self.ARCHIVE_DIR).expanduser()

@cached_property
Expand All @@ -77,27 +81,32 @@ def TARGET_INCLUDE(self):

@cached_property
def RESOURCE(self):
assert self.RESOURCE_LOCATION is not None, 'RESOURCE_LOCATION is not set'
if self.RESOURCE_LOCATION is None:
raise RuntimeError('RESOURCE_LOCATION is not set')
return fetch_resource(self.RESOURCE_LOCATION)

@cached_property
def DEFINITIONS(self):
assert self.PROTOCOL_LOCATIONS is not None, 'PROTOCOL_LOCATIONS is not set'
if self.PROTOCOL_LOCATIONS is None:
raise RuntimeError('PROTOCOL_LOCATIONS is not set')
return fetch_definitions(self.PROTOCOL_LOCATIONS.split(), self.PATH)

@cached_property
def PATTERN(self):
assert self.PROTOCOL_LOCATIONS is not None, 'PROTOCOL_LOCATIONS is not set'
if self.PROTOCOL_LOCATIONS is None:
raise RuntimeError('PROTOCOL_LOCATIONS is not set')
return fetch_pattern(self.PROTOCOL_LOCATIONS.split(), self.PATH)

@cached_property
def SCHEMA(self):
assert self.PROTOCOL_LOCATIONS is not None, 'PROTOCOL_LOCATIONS is not set'
if self.PROTOCOL_LOCATIONS is None:
raise RuntimeError('PROTOCOL_LOCATIONS is not set')
return fetch_schema(self.PROTOCOL_LOCATIONS.split(), self.PATH)

@cached_property
def TREE(self):
assert self.PROTOCOL_LOCATIONS is not None, 'PROTOCOL_LOCATIONS is not set'
if self.PROTOCOL_LOCATIONS is None:
raise RuntimeError('PROTOCOL_LOCATIONS is not set')
return fetch_tree(self.PROTOCOL_LOCATIONS.split(), self.PATH)


Expand Down
2 changes: 1 addition & 1 deletion isimip_publisher/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ def main():
if hasattr(settings, 'FUNC'):
try:
settings.FUNC()
except AssertionError as e:
except RuntimeError as e:
parser.error(e)
else:
parser.print_help()
Expand Down
183 changes: 95 additions & 88 deletions isimip_publisher/utils/database.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,14 +209,14 @@ def insert_dataset(session, version, rights, restricted, name, path, size, speci

if dataset:
logger.debug('skip dataset %s', path)
assert dataset.target is None, \
f'Dataset {path} is already stored, but with a target'
assert dataset.rights == rights, \
f'Dataset {path} is already stored, but with different rights'
assert dataset.name == name, \
f'Dataset {path} is already stored, but with different name'
assert dataset.specifiers == specifiers, \
f'Dataset {path} is already stored, but with different specifiers'
if dataset.target is not None:
raise RuntimeError(f'Dataset {path} is already stored, but with a target')
if dataset.rights != rights:
raise RuntimeError(f'Dataset {path} is already stored, but with different rights')
if dataset.name != name:
raise RuntimeError(f'Dataset {path} is already stored, but with different name')
if dataset.specifiers != specifiers:
raise RuntimeError(f'Dataset {path} is already stored, but with different specifiers')
else:
# insert a new row for this dataset
logger.debug('insert dataset %s', path)
Expand All @@ -242,17 +242,17 @@ def publish_dataset(session, version, path):
Dataset.public == True # noqa: E712
).one_or_none()

assert public_dataset is None or public_dataset.version == version, \
f'A public dataset with the path {path} and the version {public_dataset.version} was found'
if public_dataset is not None and public_dataset.version != version:
raise RuntimeError(f'A public dataset with the path {path} and the version {public_dataset.version} was found')

# get the dataset
dataset = session.query(Dataset).filter(
Dataset.path == path,
Dataset.version == version
).one_or_none()

assert dataset is not None, \
f'No dataset with the name {path} and the version {public_dataset.version} found'
if dataset is None:
raise RuntimeError(f'No dataset with the name {path} and the version {public_dataset.version} found')

# mark this dataset public
logger.debug('publish dataset %s', path)
Expand All @@ -267,12 +267,12 @@ def update_dataset(session, rights, restricted, path, specifiers):
Dataset.public == True # noqa: E712
).one_or_none()

assert dataset is not None, \
f'No public dataset with the path {path} found.'
if dataset is None:
raise RuntimeError(f'No public dataset with the path {path} found.')

if dataset.target:
assert dataset.target.rights == rights,\
f'Target dataset {dataset.target.path} was found, but with different rights'
if dataset.target.rights != rights:
raise RuntimeError(f'Target dataset {dataset.target.path} was found, but with different rights')

# update the dataset
logger.debug('update dataset %s', path)
Expand Down Expand Up @@ -300,14 +300,14 @@ def insert_dataset_link(session, version, rights, restricted, target_dataset_pat
Dataset.version == version
).one_or_none()

assert target_dataset is not None, \
f'No target dataset for the path {target_dataset_path} with version {version} found'
assert target_dataset.rights == rights, \
f'Target dataset {target_dataset_path}#{version} was found, but with different rights'
assert target_dataset.name == name, \
f'Target dataset {target_dataset_path}#{version} was found, but with a different name'
assert target_dataset.size == size, \
f'Target dataset {target_dataset_path}#{version} was found, but with a different size'
if target_dataset is None:
raise RuntimeError(f'No target dataset for the path {target_dataset_path} with version {version} found')
if target_dataset.rights != rights:
raise RuntimeError(f'Target dataset {target_dataset_path}#{version} was found, but with different rights')
if target_dataset.name != name:
raise RuntimeError(f'Target dataset {target_dataset_path}#{version} was found, but with a different name')
if target_dataset.size != size:
raise RuntimeError(f'Target dataset {target_dataset_path}#{version} was found, but with a different size')

# check if the dataset with this version is already in the database
dataset = session.query(Dataset).filter(
Expand All @@ -317,14 +317,14 @@ def insert_dataset_link(session, version, rights, restricted, target_dataset_pat

if dataset:
logger.debug('skip dataset link %s', path)
assert dataset.target == target_dataset, \
f'Dataset link {path} is already stored, but with a different target'
assert dataset.rights == rights, \
f'Dataset link {path} is already stored, but with different rights'
assert dataset.name == name, \
f'Dataset link {path} is already stored, but with a different name'
assert dataset.specifiers == specifiers, \
f'Dataset link {path} is already stored, but with different specifiers'
if dataset.target != target_dataset:
raise RuntimeError(f'Dataset link {path} is already stored, but with a different target')
if dataset.rights != rights:
raise RuntimeError(f'Dataset link {path} is already stored, but with different rights')
if dataset.name != name:
raise RuntimeError(f'Dataset link {path} is already stored, but with a different name')
if dataset.specifiers != specifiers:
raise RuntimeError(f'Dataset link {path} is already stored, but with different specifiers')
else:
# insert a new row for this dataset
logger.debug('insert dataset %s', path)
Expand Down Expand Up @@ -395,8 +395,8 @@ def insert_file(session, version, dataset_path, uuid, name, path, size,
Dataset.version == version
).one_or_none()

assert dataset is not None, \
f'No dataset with the path {dataset_path} found'
if dataset is None:
raise RuntimeError(f'No dataset with the path {dataset_path} found')

# check if the file is already in the database
file = session.query(File).filter(
Expand All @@ -406,20 +406,20 @@ def insert_file(session, version, dataset_path, uuid, name, path, size,

if file:
logger.debug('skip file %s', path)
assert uuid is None or file.id == uuid, \
f'File {path} is already stored with the same version, but a different id'
assert file.name == name, \
f'File {path} is already stored with the same version, but a different name'
assert file.size == size, \
f'File {path} is already stored with the same version, but a different size'
assert file.checksum == checksum, \
f'File {path} is already stored with the same version, but a different checksum'
assert file.checksum_type == checksum_type, \
f'File {path} is already stored with the same version, but a different checksum_type'
assert file.netcdf_header == netcdf_header, \
f'File {path} is already stored with the same version, but a different netcdf_header'
assert file.specifiers == specifiers, \
f'File {path} is already stored with the same version, but different specifiers'
if uuid is not None and file.id != uuid:
raise RuntimeError(f'File {path} is already stored with the same version, but a different id')
if file.name != name:
raise RuntimeError(f'File {path} is already stored with the same version, but a different name')
if file.size != size:
raise RuntimeError(f'File {path} is already stored with the same version, but a different size')
if file.checksum != checksum:
raise RuntimeError(f'File {path} is already stored with the same version, but a different checksum')
if file.checksum_type != checksum_type:
raise RuntimeError(f'File {path} is already stored with the same version, but a different checksum_type')
if file.netcdf_header != netcdf_header:
raise RuntimeError(f'File {path} is already stored with the same version, but a different netcdf_header')
if file.specifiers != specifiers:
raise RuntimeError(f'File {path} is already stored with the same version, but different specifiers')
else:
# insert a new row for this file
logger.debug('insert file %s', path)
Expand Down Expand Up @@ -449,8 +449,8 @@ def update_file(session, dataset_path, path, specifiers):
Dataset.public == True # noqa: E712
).one_or_none()

assert dataset is not None, \
f'No public dataset with the path {dataset_path} found'
if dataset is None:
raise RuntimeError(f'No public dataset with the path {dataset_path} found')

# check if the file is already in the database
file = session.query(File).filter(
Expand All @@ -464,7 +464,7 @@ def update_file(session, dataset_path, path, specifiers):
file.identifiers = list(specifiers.keys())
file.updated = datetime.utcnow()
else:
raise AssertionError(f'No file with the path {path} found in dataset {dataset_path}')
raise RuntimeError(f'No file with the path {path} found in dataset {dataset_path}')


def insert_file_link(session, version, target_file_path, dataset_path,
Expand All @@ -475,27 +475,27 @@ def insert_file_link(session, version, target_file_path, dataset_path,
File.version == version
).one_or_none()

assert target_file is not None, \
f'No target file for the path {target_file_path} with version {version} found'
assert target_file.name == name, \
f'Target file {target_file_path}#{version} was found, but with a different name'
assert target_file.size == size, \
f'Target file {target_file_path}#{version} was found, but with a different size'
assert target_file.checksum == checksum, \
f'Target file {target_file_path}#{version} was found, but with a different checksum'
assert target_file.checksum_type == checksum_type, \
f'Target file {target_file_path}#{version} was found, but with a different checksum_type'
if target_file is None:
raise RuntimeError(f'No target file for the path {target_file_path} with version {version} found')
if target_file.name != name:
raise RuntimeError(f'Target file {target_file_path}#{version} was found, but with a different name')
if target_file.size != size:
raise RuntimeError(f'Target file {target_file_path}#{version} was found, but with a different size')
if target_file.checksum != checksum:
raise RuntimeError(f'Target file {target_file_path}#{version} was found, but with a different checksum')
if target_file.checksum_type != checksum_type:
raise RuntimeError(f'Target file {target_file_path}#{version} was found, but with a different checksum_type')

# get the linked dataset for this file from the database
dataset = session.query(Dataset).filter(
Dataset.path == dataset_path,
Dataset.version == version
).one_or_none()

assert dataset is not None, \
f'No dataset with the path {dataset_path} found'
assert dataset.target == target_file.dataset, \
f'Dataset for file link does not match dataset for {path}'
if dataset is None:
raise RuntimeError(f'No dataset with the path {dataset_path} found')
if dataset.target != target_file.dataset:
raise RuntimeError(f'Dataset for file link does not match dataset for {path}')

# check if the file is already in the database
file = session.query(File).filter(
Expand All @@ -505,18 +505,20 @@ def insert_file_link(session, version, target_file_path, dataset_path,

if file:
logger.debug('skip file %s', path)
assert file.name == name, \
f'File link {path} is already stored with the same version, but a different name'
assert file.size == size, \
f'File link {path} is already stored with the same version, but a different size'
assert file.checksum == checksum, \
f'File link {path} is already stored with the same version, but a different checksum'
assert file.checksum_type == checksum_type, \
f'File link {path} is already stored with the same version, but a different checksum_type'
assert file.netcdf_header == netcdf_header, \
f'File link {path} is already stored with the same version, but a different netcdf_header'
assert file.specifiers == specifiers, \
f'File link {path} is already stored with the same version, but different specifiers'
if file.name != name:
raise RuntimeError(f'File link {path} is already stored with the same version, but a different name')
if file.size != size:
raise RuntimeError(f'File link {path} is already stored with the same version, but a different size')
if file.checksum != checksum:
raise RuntimeError(f'File link {path} is already stored with the same version, but a different checksum')
if file.checksum_type != checksum_type:
raise RuntimeError(f'File link {path} is already stored with the same version,'
' but a different checksum_type')
if file.netcdf_header != netcdf_header:
raise RuntimeError(f'File link {path} is already stored with the same version,'
' but a different netcdf_header')
if file.specifiers != specifiers:
raise RuntimeError(f'File link {path} is already stored with the same version, but different specifiers')
else:
# insert a new row for this file
logger.debug('insert file %s', path)
Expand All @@ -542,19 +544,22 @@ def insert_resource(session, datacite, paths, datacite_prefix):
title = get_title(datacite)
version = datacite.get('version')

assert doi is not None, 'No DOI was provided.'
assert title is not None, 'No title was provided.'
if doi is None:
raise RuntimeError('No DOI was provided.')
if title is None:
raise RuntimeError('No title was provided.')

# look for the resource in the database
resource = session.query(Resource).filter(
Resource.doi == doi
).one_or_none()

assert resource is None, \
f'A resource with doi={doi} is already in the database.'
if resource is not None:
raise RuntimeError(f'A resource with doi={doi} is already in the database.')

# get the path
assert paths, f'No paths were provided for {doi}.'
if not paths:
raise RuntimeError(f'No paths were provided for {doi}.')

# gather datasets
datasets = []
Expand Down Expand Up @@ -595,16 +600,18 @@ def update_resource(session, datacite):
title = get_title(datacite)
version = datacite.get('version')

assert doi is not None, 'No DOI was provided.'
assert title is not None, 'No title was provided.'
if doi is None:
raise RuntimeError('No DOI was provided.')
if title is None:
raise RuntimeError('No title was provided.')

# look for the resource in the database
resource = session.query(Resource).filter(
Resource.doi == doi
).one_or_none()

assert resource is not None, \
f'A resource with doi={doi} was not found.'
if resource is None:
raise RuntimeError(f'A resource with doi={doi} was not found.')

# update the datecite metadata
resource.title = title
Expand All @@ -621,8 +628,8 @@ def fetch_resource(session, doi):
Resource.doi == doi
).one_or_none()

assert resource is not None, \
f'A resource with doi={doi} was not found.'
if resource is None:
raise RuntimeError(f'A resource with doi={doi} was not found.')

return resource

Expand Down
Loading

0 comments on commit 008ae02

Please sign in to comment.