From 14c889fcb56a1982be4f06357f3837229d1f9993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 15 Nov 2024 08:28:42 +0100 Subject: [PATCH 1/6] Reintroduce #3472 after framework changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 45 ++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 7 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index b8abecf8fc..5f098f83d7 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -31,6 +31,7 @@ @author: Pieter De Baets (Ghent University) @author: Jens Timmerman (Ghent University) @author: Jasper Grimm (University of York) +@author: Jan Andre Reuter (Juelich Supercomputing Centre) """ import copy import os @@ -71,8 +72,8 @@ def __init__(self, *args, **kwargs): self.altroot = None self.altversion = None - # list of EasyConfig instances for components - self.comp_cfgs = [] + # list of EasyConfig instances and their EasyBlocks for components + self.comp_instances = [] # list of EasyConfig instances of components for which to run sanity checks self.comp_cfgs_sanity_check = [] @@ -198,7 +199,7 @@ def __init__(self, *args, **kwargs): if comp_cfg['patches']: self.cfg.update('patches', comp_cfg['patches']) - self.comp_cfgs.append(comp_cfg) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg))) self.cfg.update('checksums', checksums_patches) @@ -217,7 +218,7 @@ def check_checksums(self): """ checksum_issues = super(Bundle, self).check_checksums() - for comp in self.comp_cfgs: + for comp, _ in self.comp_instances: checksum_issues.extend(self.check_checksums_for(comp, sub="of component %s" % comp['name'])) return checksum_issues @@ -247,14 +248,12 @@ def build_step(self): def install_step(self): """Install components, if specified.""" comp_cnt = len(self.cfg['components']) - for idx, cfg in enumerate(self.comp_cfgs): + for idx, (cfg, comp) in enumerate(self.comp_instances): print_msg("installing bundle component %s v%s (%d/%d)..." % (cfg['name'], cfg['version'], idx + 1, comp_cnt)) self.log.info("Installing component %s v%s using easyblock %s", cfg['name'], cfg['version'], cfg.easyblock) - comp = cfg.easyblock(cfg) - # correct build/install dirs comp.builddir = self.builddir comp.install_subdir, comp.installdir = self.install_subdir, self.installdir @@ -324,6 +323,38 @@ def install_step(self): # close log for this component comp.close_log() + def make_module_req_guess(self): + """ + Set module requirements from all comppnents, e.g. $PATH, etc. + During the install step, we only set these requirements temporarily. + Later on when building the module, those paths are not considered. + Therefore, iterate through all the components again and gather + the requirements. + + Do not remove duplicates or check for existance of folders, + as this is done in the generic EasyBlock while creating + the modulefile already. + """ + # Start with the paths from the generic EasyBlock. + # If not added here, they might be missing entirely and fail sanity checks. + final_reqs = super(Bundle, self).make_module_req_guess() + + for cfg, comp in self.comp_instances: + self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) + reqs = comp.make_module_req_guess() + + try: + for key, value in sorted(reqs.items()): + if isinstance(reqs, string_type): + value = [value] + final_reqs.setdefault(key, []) + final_reqs[key] += value + except AttributeError: + raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", + cfg['name'], cfg['version']) + + return final_reqs + def make_module_extra(self, *args, **kwargs): """Set extra stuff in module file, e.g. $EBROOT*, $EBVERSION*, etc.""" if not self.altroot and not self.altversion: From fd3cd657cdfc8b209f6660c324a456c575d90080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 15 Nov 2024 08:33:16 +0100 Subject: [PATCH 2/6] Update clang_aomp for bundle changes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/c/clang_aomp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/c/clang_aomp.py b/easybuild/easyblocks/c/clang_aomp.py index 6f046cdb28..7e8181faba 100644 --- a/easybuild/easyblocks/c/clang_aomp.py +++ b/easybuild/easyblocks/c/clang_aomp.py @@ -145,7 +145,7 @@ def configure_step(self): raise EasyBuildError("Could not find 'ROCm-Device-Libs' source directory in %s", self.builddir) num_comps = len(self.cfg['components']) - for idx, comp in enumerate(self.comp_cfgs): + for idx, (comp, _) in enumerate(self.comp_instances): name = comp['name'] msg = "configuring bundle component %s %s (%d/%d)..." % (name, comp['version'], idx + 1, num_comps) print_msg(msg) From 43ec3552236795f8cd92742e8736dea2d75161f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Fri, 15 Nov 2024 08:33:28 +0100 Subject: [PATCH 3/6] Transfer logfile from bundle to components MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This fixes the increased memory usage when running the EasyConfig test suite as log files would stay open. Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 5f098f83d7..dd17b82a83 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -199,7 +199,7 @@ def __init__(self, *args, **kwargs): if comp_cfg['patches']: self.cfg.update('patches', comp_cfg['patches']) - self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg))) + self.comp_instances.append((comp_cfg, comp_cfg.easyblock(comp_cfg, logfile=self.logfile))) self.cfg.update('checksums', checksums_patches) @@ -320,9 +320,6 @@ def install_step(self): new_val = path env.setvar(envvar, new_val) - # close log for this component - comp.close_log() - def make_module_req_guess(self): """ Set module requirements from all comppnents, e.g. $PATH, etc. From bce45f57918edf9d839fc4bb3c6090d8c39ae7cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 4 Dec 2024 10:19:29 +0100 Subject: [PATCH 4/6] Bundle: Address review comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index dd17b82a83..2571df7f32 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -322,15 +322,15 @@ def install_step(self): def make_module_req_guess(self): """ - Set module requirements from all comppnents, e.g. $PATH, etc. + Set module requirements from all components, e.g. $PATH, etc. During the install step, we only set these requirements temporarily. Later on when building the module, those paths are not considered. Therefore, iterate through all the components again and gather the requirements. - Do not remove duplicates or check for existance of folders, + Do not remove duplicates or check for existence of folders, as this is done in the generic EasyBlock while creating - the modulefile already. + the module file already. """ # Start with the paths from the generic EasyBlock. # If not added here, they might be missing entirely and fail sanity checks. @@ -342,10 +342,10 @@ def make_module_req_guess(self): try: for key, value in sorted(reqs.items()): - if isinstance(reqs, string_type): + if isinstance(value, string_type): value = [value] final_reqs.setdefault(key, []) - final_reqs[key] += value + final_reqs[key].append(value) except AttributeError: raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", cfg['name'], cfg['version']) From 4ebf4a36a1ba66e032cec324b9c00c587ae0839a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 4 Dec 2024 15:32:17 +0100 Subject: [PATCH 5/6] Bundle: Add comment to try-except block in make_module_req_guess() MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index 2571df7f32..caaad68cd2 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -340,6 +340,9 @@ def make_module_req_guess(self): self.log.info("Gathering module paths for component %s v%s", cfg['name'], cfg['version']) reqs = comp.make_module_req_guess() + # Try-except block to fail with an easily understandable error message. + # This should only trigger when an EasyBlock returns non-dict module requirements + # for make_module_req_guess() which should then be fixed in the components EasyBlock. try: for key, value in sorted(reqs.items()): if isinstance(value, string_type): From 4b0522acf511f0e67edad4635b473d054ca40155 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Andr=C3=A9=20Reuter?= Date: Wed, 4 Dec 2024 20:41:02 +0100 Subject: [PATCH 6/6] Bundle: Replace append with extend when copying reqs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jan André Reuter --- easybuild/easyblocks/generic/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/easybuild/easyblocks/generic/bundle.py b/easybuild/easyblocks/generic/bundle.py index caaad68cd2..62228fd5a0 100644 --- a/easybuild/easyblocks/generic/bundle.py +++ b/easybuild/easyblocks/generic/bundle.py @@ -348,7 +348,7 @@ def make_module_req_guess(self): if isinstance(value, string_type): value = [value] final_reqs.setdefault(key, []) - final_reqs[key].append(value) + final_reqs[key].extend(value) except AttributeError: raise EasyBuildError("Cannot process module requirements of bundle component %s v%s", cfg['name'], cfg['version'])