Merge pull request #4650 from easybuilders/4.9.x

release EasyBuild v4.9.4

RELEASE_NOTES

@@ -4,6 +4,18 @@ For more detailed information, please see the git log.
 These release notes can also be consulted at https://easybuild.readthedocs.io/en/latest/Release_notes.html.
 
 
+v4.9.4 (22 september 2024)
+--------------------------
+
+update/bugfix release
+
+- various enhancements, including:
+  - set $LMOD_TERSE_DECORATIONS to 'no' to avoid additional info in output produced by 'ml --terse avail' (#4648)
+- various bug fixes, including:
+  - implement workaround for permission error when copying read-only files that have extended attributes set and using Python 3.6 (#4642)
+  - take into account alternate sysroot for /bin/bash used by run_cmd (#4646)
+
+
 v4.9.3 (14 September 2024)
 --------------------------
 

easybuild/tools/filetools.py

@@ -42,11 +42,13 @@
 """
 import datetime
 import difflib
+import filecmp
 import glob
 import hashlib
 import inspect
 import itertools
 import os
+import platform
 import re
 import shutil
 import signal
@@ -59,7 +61,7 @@
 from functools import partial
 
 from easybuild.base import fancylogger
-from easybuild.tools import run
+from easybuild.tools import LooseVersion, run
 # import build_log must stay, to use of EasyBuildLog
 from easybuild.tools.build_log import EasyBuildError, dry_run_msg, print_msg, print_warning
 from easybuild.tools.config import DEFAULT_WAIT_ON_LOCK_INTERVAL, ERROR, GENERIC_EASYBLOCK_PKG, IGNORE, WARN
@@ -2435,8 +2437,42 @@ def copy_file(path, target_path, force_in_dry_run=False):
             else:
                 mkdir(os.path.dirname(target_path), parents=True)
                 if path_exists:
-                    shutil.copy2(path, target_path)
-                    _log.info("%s copied to %s", path, target_path)
+                    try:
+                        # on filesystems that support extended file attributes, copying read-only files with
+                        # shutil.copy2() will give a PermissionError, when using Python < 3.7
+                        # see https://bugs.python.org/issue24538
+                        shutil.copy2(path, target_path)
+                        _log.info("%s copied to %s", path, target_path)
+                    # catch the more general OSError instead of PermissionError,
+                    # since Python 2.7 doesn't support PermissionError
+                    except OSError as err:
+                        # if file is writable (not read-only), then we give up since it's not a simple permission error
+                        if os.path.exists(target_path) and os.stat(target_path).st_mode & stat.S_IWUSR:
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        pyver = LooseVersion(platform.python_version())
+                        if pyver >= LooseVersion('3.7'):
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+                        elif LooseVersion('3.7') > pyver >= LooseVersion('3'):
+                            if not isinstance(err, PermissionError):
+                                raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        # double-check whether the copy actually succeeded
+                        if not os.path.exists(target_path) or not filecmp.cmp(path, target_path, shallow=False):
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        try:
+                            # re-enable user write permissions in target, copy xattrs, then remove write perms again
+                            adjust_permissions(target_path, stat.S_IWUSR)
+                            shutil._copyxattr(path, target_path)
+                            adjust_permissions(target_path, stat.S_IWUSR, add=False)
+                        except OSError as err:
+                            raise EasyBuildError("Failed to copy file %s to %s: %s", path, target_path, err)
+
+                        msg = ("Failed to copy extended attributes from file %s to %s, due to a bug in shutil (see "
+                               "https://bugs.python.org/issue24538). Copy successful with workaround.")
+                        _log.info(msg, path, target_path)
+
                 elif os.path.islink(path):
                     if os.path.isdir(target_path):
                         target_path = os.path.join(target_path, os.path.basename(path))

easybuild/tools/modules.py

@@ -1448,6 +1448,9 @@ def __init__(self, *args, **kwargs):
         setvar('LMOD_REDIRECT', 'no', verbose=False)
         # disable extended defaults within Lmod (introduced and set as default in Lmod 8.0.7)
         setvar('LMOD_EXTENDED_DEFAULT', 'no', verbose=False)
+        # disabled decorations in "ml --terse avail" output
+        # (introduced in Lmod 8.8, see also https://github.com/TACC/Lmod/issues/690)
+        setvar('LMOD_TERSE_DECORATIONS', 'no', verbose=False)
 
         super(Lmod, self).__init__(*args, **kwargs)
         version = StrictVersion(self.version)

easybuild/tools/options.py

@@ -1969,7 +1969,7 @@ def set_tmpdir(tmpdir=None, raise_error=False):
             os.close(fd)
             os.chmod(tmptest_file, 0o700)
             if not run_cmd(tmptest_file, simple=True, log_ok=False, regexp=False, force_in_dry_run=True, trace=False,
-                           stream_output=False, with_hooks=False):
+                           stream_output=False, with_hooks=False, with_sysroot=False):
                 msg = "The temporary directory (%s) does not allow to execute files. " % tempfile.gettempdir()
                 msg += "This can cause problems in the build process, consider using --tmpdir."
                 if raise_error:

easybuild/tools/run.py

@@ -134,7 +134,7 @@ def get_output_from_process(proc, read_size=None, asynchronous=False):
 @run_cmd_cache
 def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True, log_output=False, path=None,
             force_in_dry_run=False, verbose=True, shell=None, trace=True, stream_output=None, asynchronous=False,
-            with_hooks=True):
+            with_hooks=True, with_sysroot=True):
     """
     Run specified command (in a subshell)
     :param cmd: command to run
@@ -152,6 +152,7 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
     :param stream_output: enable streaming command output to stdout
     :param asynchronous: run command asynchronously (returns subprocess.Popen instance if set to True)
     :param with_hooks: trigger pre/post run_shell_cmd hooks (if defined)
+    :param with_sysroot: prepend sysroot to exec_cmd (if defined)
     """
     cwd = os.getcwd()
 
@@ -228,6 +229,16 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
 
     exec_cmd = "/bin/bash"
 
+    # if EasyBuild is configured to use an alternate sysroot,
+    # we should also run shell commands using the bash shell provided in there,
+    # since /bin/bash may not be compatible with the alternate sysroot
+    if with_sysroot:
+        sysroot = build_option('sysroot')
+        if sysroot:
+            sysroot_bin_bash = os.path.join(sysroot, 'bin', 'bash')
+            if os.path.exists(sysroot_bin_bash):
+                exec_cmd = sysroot_bin_bash
+
     if not shell:
         if isinstance(cmd, list):
             exec_cmd = None
@@ -237,6 +248,8 @@ def run_cmd(cmd, log_ok=True, log_all=False, simple=False, inp=None, regexp=True
         else:
             raise EasyBuildError("Don't know how to prefix with /usr/bin/env for commands of type %s", type(cmd))
 
+    _log.info("Using %s as shell for running cmd: %s", exec_cmd, cmd)
+
     if with_hooks:
         hooks = load_hooks(build_option('hooks'))
         hook_res = run_hook(RUN_SHELL_CMD, hooks, pre_step_hook=True, args=[cmd], kwargs={'work_dir': os.getcwd()})

easybuild/tools/systemtools.py

@@ -275,7 +275,8 @@ def get_avail_core_count():
         core_cnt = int(sum(sched_getaffinity()))
     else:
         # BSD-type systems
-        out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+        out, _ = run_cmd('sysctl -n hw.ncpu', force_in_dry_run=True, trace=False, stream_output=False,
+                         with_hooks=False, with_sysroot=False)
         try:
             if int(out) > 0:
                 core_cnt = int(out)
@@ -312,7 +313,8 @@ def get_total_memory():
     elif os_type == DARWIN:
         cmd = "sysctl -n hw.memsize"
         _log.debug("Trying to determine total memory size on Darwin via cmd '%s'", cmd)
-        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False,
+                          with_sysroot=False)
         if ec == 0:
             memtotal = int(out.strip()) // (1024**2)
 
@@ -394,15 +396,16 @@ def get_cpu_vendor():
 
     elif os_type == DARWIN:
         cmd = "sysctl -n machdep.cpu.vendor"
-        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False, with_hooks=False)
+        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
+                          with_hooks=False, with_sysroot=False)
         out = out.strip()
         if ec == 0 and out in VENDOR_IDS:
             vendor = VENDOR_IDS[out]
             _log.debug("Determined CPU vendor on DARWIN as being '%s' via cmd '%s" % (vendor, cmd))
         else:
             cmd = "sysctl -n machdep.cpu.brand_string"
             out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
-                              with_hooks=False)
+                              with_hooks=False, with_sysroot=False)
             out = out.strip().split(' ')[0]
             if ec == 0 and out in CPU_VENDORS:
                 vendor = out
@@ -505,7 +508,8 @@ def get_cpu_model():
 
     elif os_type == DARWIN:
         cmd = "sysctl -n machdep.cpu.brand_string"
-        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False,
+                          with_sysroot=False)
         if ec == 0:
             model = out.strip()
             _log.debug("Determined CPU model on Darwin using cmd '%s': %s" % (cmd, model))
@@ -550,7 +554,8 @@ def get_cpu_speed():
     elif os_type == DARWIN:
         cmd = "sysctl -n hw.cpufrequency_max"
         _log.debug("Trying to determine CPU frequency on Darwin via cmd '%s'" % cmd)
-        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+        out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False,
+                          with_sysroot=False)
         out = out.strip()
         cpu_freq = None
         if ec == 0 and out:
@@ -599,7 +604,7 @@ def get_cpu_features():
             cmd = "sysctl -n machdep.cpu.%s" % feature_set
             _log.debug("Trying to determine CPU features on Darwin via cmd '%s'", cmd)
             out, ec = run_cmd(cmd, force_in_dry_run=True, trace=False, stream_output=False, log_ok=False,
-                              with_hooks=False)
+                              with_hooks=False, with_sysroot=False)
             if ec == 0:
                 cpu_feat.extend(out.strip().lower().split())
 
@@ -626,8 +631,8 @@ def get_gpu_info():
         try:
             cmd = "nvidia-smi --query-gpu=gpu_name,driver_version --format=csv,noheader"
             _log.debug("Trying to determine NVIDIA GPU info on Linux via cmd '%s'", cmd)
-            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False,
-                              force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True,
+                              trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
             if ec == 0:
                 for line in out.strip().split('\n'):
                     nvidia_gpu_info = gpu_info.setdefault('NVIDIA', {})
@@ -645,15 +650,15 @@ def get_gpu_info():
         try:
             cmd = "rocm-smi --showdriverversion --csv"
             _log.debug("Trying to determine AMD GPU driver on Linux via cmd '%s'", cmd)
-            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False,
-                              force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True,
+                              trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
             if ec == 0:
                 amd_driver = out.strip().split('\n')[1].split(',')[1]
 
             cmd = "rocm-smi --showproductname --csv"
             _log.debug("Trying to determine AMD GPU info on Linux via cmd '%s'", cmd)
-            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False,
-                              force_in_dry_run=True, trace=False, stream_output=False, with_hooks=False)
+            out, ec = run_cmd(cmd, simple=False, log_ok=False, log_all=False, force_in_dry_run=True,
+                              trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
             if ec == 0:
                 for line in out.strip().split('\n')[1:]:
                     amd_card_series = line.split(',')[1]
@@ -900,7 +905,7 @@ def get_tool_version(tool, version_option='--version', ignore_ec=False):
     Output is returned as a single-line string (newlines are replaced by '; ').
     """
     out, ec = run_cmd(' '.join([tool, version_option]), simple=False, log_ok=False, force_in_dry_run=True,
-                      trace=False, stream_output=False, with_hooks=False)
+                      trace=False, stream_output=False, with_hooks=False, with_sysroot=False)
     if not ignore_ec and ec:
         _log.warning("Failed to determine version of %s using '%s %s': %s" % (tool, tool, version_option, out))
         return UNKNOWN

easybuild/tools/version.py

@@ -45,7 +45,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.9.3')
+VERSION = LooseVersion('4.9.4')
 UNKNOWN = 'UNKNOWN'
 UNKNOWN_EASYBLOCKS_VERSION = '0.0.UNKNOWN.EASYBLOCKS'
 

test/framework/filetools.py

@@ -32,6 +32,7 @@
 @author: Maxime Boissonneault (Compute Canada, Universite Laval)
 """
 import datetime
+import filecmp
 import glob
 import logging
 import os
@@ -51,6 +52,8 @@
 from easybuild.tools.config import IGNORE, ERROR, build_option, update_build_option
 from easybuild.tools.multidiff import multidiff
 from easybuild.tools.py2vs3 import StringIO, std_urllib
+from easybuild.tools.run import run_cmd
+from easybuild.tools.systemtools import LINUX, get_os_type
 
 
 class FileToolsTest(EnhancedTestCase):
@@ -1912,6 +1915,47 @@ def test_copy_file(self):
         # However, if we add 'force_in_dry_run=True' it should throw an exception
         self.assertErrorRegex(EasyBuildError, "Could not copy *", ft.copy_file, src, target, force_in_dry_run=True)
 
+    def test_copy_file_xattr(self):
+        """Test copying a file with extended attributes using copy_file."""
+        # test copying a read-only files with extended attributes set
+        # first, create a special file with extended attributes
+        special_file = os.path.join(self.test_prefix, 'special.txt')
+        ft.write_file(special_file, 'special')
+        # make read-only, and set extended attributes
+        attr = ft.which('attr')
+        xattr = ft.which('xattr')
+        # try to attr (Linux) or xattr (macOS) to set extended attributes foo=bar
+        cmd = None
+        if attr:
+            cmd = "attr -s foo -V bar %s" % special_file
+        elif xattr:
+            cmd = "xattr -w foo bar %s" % special_file
+
+        if cmd:
+            (_, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False)
+
+            # need to make file read-only after setting extended attribute
+            ft.adjust_permissions(special_file, stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False)
+
+            # only proceed if setting extended attribute worked
+            if ec == 0:
+                target = os.path.join(self.test_prefix, 'copy.txt')
+                ft.copy_file(special_file, target)
+                self.assertTrue(os.path.exists(target))
+                self.assertTrue(filecmp.cmp(special_file, target, shallow=False))
+
+                # only verify wheter extended attributes were also copied on Linux,
+                # since shutil.copy2 doesn't copy them on macOS;
+                # see warning at https://docs.python.org/3/library/shutil.html
+                if get_os_type() == LINUX:
+                    if attr:
+                        cmd = "attr -g foo %s" % target
+                    else:
+                        cmd = "xattr -l %s" % target
+                    (out, ec) = run_cmd(cmd, simple=False, log_all=False, log_ok=False)
+                    self.assertEqual(ec, 0)
+                    self.assertTrue(out.endswith('\nbar\n'))
+
     def test_copy_files(self):
         """Test copy_files function."""
         test_ecs = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'easyconfigs', 'test_ecs')

test/framework/run.py

@@ -796,6 +796,31 @@ def post_run_shell_cmd_hook(cmd, *args, **kwargs):
         ])
         self.assertEqual(stdout, expected_stdout)
 
+    def test_run_cmd_sysroot(self):
+        """Test with_sysroot option of run_cmd function."""
+
+        # put fake /bin/bash in place that will be picked up when using run_cmd with with_sysroot=True
+        bin_bash = os.path.join(self.test_prefix, 'bin', 'bash')
+        bin_bash_txt = '\n'.join([
+            "#!/bin/bash",
+            "echo 'Hi there I am a fake /bin/bash in %s'" % self.test_prefix,
+            '/bin/bash "$@"',
+        ])
+        write_file(bin_bash, bin_bash_txt)
+        adjust_permissions(bin_bash, stat.S_IXUSR)
+
+        update_build_option('sysroot', self.test_prefix)
+
+        (out, ec) = run_cmd("echo hello")
+        self.assertEqual(ec, 0)
+        self.assertTrue(out.startswith("Hi there I am a fake /bin/bash in"))
+        self.assertTrue(out.endswith("\nhello\n"))
+
+        # picking up on alternate sysroot is enabled by default, but can be disabled via with_sysroot=False
+        (out, ec) = run_cmd("echo hello", with_sysroot=False)
+        self.assertEqual(ec, 0)
+        self.assertEqual(out, "hello\n")
+
 
 def suite():
     """ returns all the testcases in this module """