Skip to content

Commit

Permalink
Merge branch 'develop' into 5.0.x
Browse files Browse the repository at this point in the history
  • Loading branch information
boegel committed Oct 2, 2023
2 parents 7e899c3 + d450a80 commit 9c31fd5
Show file tree
Hide file tree
Showing 11 changed files with 167 additions and 59 deletions.
79 changes: 32 additions & 47 deletions .github/workflows/unit_tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,44 +34,23 @@ jobs:
- ${{needs.setup.outputs.modulesTcl}}
- ${{needs.setup.outputs.modules3}}
- ${{needs.setup.outputs.modules4}}
module_syntax: [Lua, Tcl]
lc_all: [""]
# don't test with Lua module syntax (only supported in Lmod)
exclude:
- modules_tool: ${{needs.setup.outputs.modulesTcl}}
module_syntax: Lua
- modules_tool: ${{needs.setup.outputs.modules3}}
module_syntax: Lua
- modules_tool: ${{needs.setup.outputs.modules4}}
module_syntax: Lua
include:
# Test different Python 3 versions with Lmod 8.x (with both Lua and Tcl module syntax)
- python: 3.7
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
- python: 3.8
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
- python: 3.8
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Tcl
- python: 3.9
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
- python: '3.10'
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
- python: '3.11'
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
- python: '3.11'
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Tcl
# There may be encoding errors in Python 3 which are hidden when an UTF-8 encoding is set
# Hence run the tests (again) with LC_ALL=C and Python 3.6 (or any < 3.7)
- python: 3.6
modules_tool: ${{needs.setup.outputs.lmod8}}
module_syntax: Lua
lc_all: C
fail-fast: false
steps:
Expand Down Expand Up @@ -122,16 +101,11 @@ jobs:
# and are only run after the PR gets merged
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
run: |
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools,
# and only when testing with Lua as module syntax,
# to avoid hitting GitHub rate limit;
# don't install GitHub token when testing with Lmod 7.x or non-Lmod module tools, to avoid hitting GitHub rate limit;
# tests that require a GitHub token are skipped automatically when no GitHub token is available
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]] && [[ "${{matrix.module_syntax}}" == 'Lua' ]]; then
if [[ ! "${{matrix.modules_tool}}" =~ 'Lmod-7' ]] && [[ ! "${{matrix.modules_tool}}" =~ 'modules-' ]]; then
if [ ! -z $GITHUB_TOKEN ]; then
if [ "x${{matrix.python}}" == 'x2.6' ];
then SET_KEYRING="keyring.set_keyring(keyring.backends.file.PlaintextKeyring())";
else SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
fi;
SET_KEYRING="import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring())";
python -c "import keyring; $SET_KEYRING; keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
fi
echo "GitHub token installed!"
Expand Down Expand Up @@ -169,8 +143,6 @@ jobs:
- name: run test suite
env:
EB_VERBOSE: 1
EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
TEST_EASYBUILD_MODULE_SYNTAX: ${{matrix.module_syntax}}
LC_ALL: ${{matrix.lc_all}}
run: |
# run tests *outside* of checked out easybuild-framework directory,
Expand All @@ -195,19 +167,32 @@ jobs:
else
export EASYBUILD_MODULES_TOOL=Lmod
fi
export TEST_EASYBUILD_MODULES_TOOL=$EASYBUILD_MODULES_TOOL
eb --show-config
# gather some useful info on test system
eb --show-system-info
# check GitHub configuration
eb --check-github --github-user=easybuild_test
# create file owned by root but writable by anyone (used by test_copy_file)
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
# run test suite
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
# try and make sure output of running tests is clean (no printed messages/warnings)
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
# '|| true' is needed to avoid that Travis stops the job on non-zero exit of grep (i.e. when there are no matches)
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
export TEST_EASYBUILD_MODULES_TOOL=${EASYBUILD_MODULES_TOOL}
# Run tests with LUA and Tcl module syntax (where supported)
for module_syntax in Lua Tcl; do
# Only Lmod supports Lua
if [[ "${module_syntax}" == "Lua" ]] && [[ "${EASYBUILD_MODULES_TOOL}" != "Lmod" ]]; then
echo "Not testing with '${module_syntax}' as module syntax with '${EASYBUILD_MODULES_TOOL}' as modules tool"
continue
fi
printf '\n\n=====================> Using $module_syntax module syntax <=====================\n\n'
export EASYBUILD_MODULE_SYNTAX="${module_syntax}"
export TEST_EASYBUILD_MODULE_SYNTAX="${EASYBUILD_MODULE_SYNTAX}"
eb --show-config
# gather some useful info on test system
eb --show-system-info
# check GitHub configuration
eb --check-github --github-user=easybuild_test
# create file owned by root but writable by anyone (used by test_copy_file)
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
# run test suite
python -O -m test.framework.suite 2>&1 | tee test_framework_suite.log
# try and make sure output of running tests is clean (no printed messages/warnings)
IGNORE_PATTERNS="no GitHub token available|skipping SvnRepository test|requires Lmod as modules tool|stty: 'standard input': Inappropriate ioctl for device|CryptographyDeprecationWarning: Python 3.[56]|from cryptography.* import |CryptographyDeprecationWarning: Python 2|Blowfish|GC3Pie not available, skipping test"
# '|| true' is needed to avoid that GitHub Actions stops the job on non-zero exit of grep (i.e. when there are no matches)
PRINTED_MSG=$(egrep -v "${IGNORE_PATTERNS}" test_framework_suite.log | grep '\.\n*[A-Za-z]' || true)
test "x$PRINTED_MSG" = "x" || (echo "ERROR: Found printed messages in output of test suite" && echo "${PRINTED_MSG}" && exit 1)
done
81 changes: 81 additions & 0 deletions .github/workflows/unit_tests_python2.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,81 @@
# documentation: https://help.github.com/en/articles/workflow-syntax-for-github-actions
name: EasyBuild framework unit tests (python2)
on: [push, pull_request]

permissions:
contents: read # to fetch code (actions/checkout)

concurrency:
group: ${{format('{0}:{1}:{2}', github.repository, github.ref, github.workflow)}}
cancel-in-progress: true

jobs:
test_python2:
runs-on: ubuntu-20.04
container:
# CentOS 7.9 container that already includes Lmod & co,
# see https://github.com/easybuilders/easybuild-containers
image: ghcr.io/easybuilders/centos-7.9-amd64
steps:
- uses: actions/checkout@v3

- name: install Python packages
run: |
# Python packages
python2 -V
python2 -m pip --version
python2 -m pip install --upgrade pip
python2 -m pip --version
# strip out GC3Pie since installation with ancient setuptools (0.9.8) fails
sed -i '/GC3Pie/d' requirements.txt
python2 -m pip install -r requirements.txt
# git config is required to make actual git commits (cfr. tests for GitRepository)
sudo -u easybuild git config --global user.name "GitHub Actions"
sudo -u easybuild git config --global user.email "actions@github.com"
sudo -u easybuild git config --get-regexp 'user.*'
- name: install GitHub token (if available)
env:
# token (owned by @boegelbot) with gist permissions (required for some of the tests for GitHub integration);
# this token is not available in pull requests, so tests that require it are skipped in PRs,
# and are only run after the PR gets merged
GITHUB_TOKEN: ${{secrets.CI_UNIT_TESTS_GITHUB_TOKEN}}
run: |
# tests that require a GitHub token are skipped automatically when no GitHub token is available
if [ ! -z $GITHUB_TOKEN ]; then
sudo -u easybuild python2 -c "import keyring; import keyrings.alt.file; keyring.set_keyring(keyrings.alt.file.PlaintextKeyring()); keyring.set_password('github_token', 'easybuild_test', '$GITHUB_TOKEN')";
echo "GitHub token installed!"
else
echo "Installation of GitHub token skipped!"
fi
- name: install sources
run: |
# install from source distribution tarball, to test release as published on PyPI
python2 setup.py sdist
ls dist
export PREFIX=/tmp/$USER/$GITHUB_SHA
python2 -m pip install --prefix $PREFIX dist/easybuild-framework*tar.gz
- name: run test suite
run: |
# run tests *outside* of checked out easybuild-framework directory,
# to ensure we're testing installed version (see previous step)
cd $HOME
# make sure 'eb' is available via $PATH, and that $PYTHONPATH is set (some tests expect that)
export PREFIX=/tmp/$USER/$GITHUB_SHA
ENV_CMDS="export PATH=$PREFIX/bin:$PATH; export PYTHONPATH=$PREFIX/lib/python2.7/site-packages:$PYTHONPATH"
ENV_CMDS="${ENV_CMDS}; export EB_VERBOSE=1; export EB_PYTHON=python2; export TEST_EASYBUILD_SILENCE_DEPRECATION_WARNINGS=python2"
# run EasyBuild command via (non-root) easybuild user + login shell
sudo -u easybuild bash -l -c "${ENV_CMDS}; module --version; eb --version"
# show active EasyBuild configuration
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-config"
# gather some useful info on test system
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --show-system-info"
# check GitHub configuration
sudo -u easybuild bash -l -c "${ENV_CMDS}; eb --check-github --github-user=easybuild_test"
# create file owned by root but writable by anyone (used by test_copy_file)
sudo touch /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
sudo chmod o+w /tmp/file_to_overwrite_for_easybuild_test_copy_file.txt
# run test suite (via easybuild user + login shell)
sudo -u easybuild bash -l -c "${ENV_CMDS}; python2 -O -m test.framework.suite"
5 changes: 3 additions & 2 deletions easybuild/framework/easyconfig/templates.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,9 +76,9 @@
('installdir', "Installation directory"),
('start_dir', "Directory in which the build process begins"),
]
# software names for which to define <pref>ver and <pref>shortver templates
# software names for which to define <pref>ver, <pref>majver and <pref>shortver templates
TEMPLATE_SOFTWARE_VERSIONS = [
# software name, prefix for *ver and *shortver
# software name, prefix for *ver, *majver and *shortver
('CUDA', 'cuda'),
('CUDAcore', 'cuda'),
('Java', 'java'),
Expand Down Expand Up @@ -423,6 +423,7 @@ def template_documentation():
# step 2: add *ver/*shortver templates for software listed in TEMPLATE_SOFTWARE_VERSIONS
doc.append("Template names/values for (short) software versions")
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
doc.append("%s%%(%smajver)s: major version for %s" % (indent_l1, pref, name))
doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (indent_l1, pref, name))
doc.append("%s%%(%sver)s: full version for %s" % (indent_l1, pref, name))

Expand Down
4 changes: 3 additions & 1 deletion easybuild/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@
import os
import stat
import sys
import tempfile
import traceback

# IMPORTANT this has to be the first easybuild import as it customises the logging
Expand Down Expand Up @@ -253,8 +254,9 @@ def process_easystack(easystack_path, args, logfile, testing, init_session_state
easyconfig._easyconfigs_cache.clear()
easyconfig._easyconfig_files_cache.clear()

# restore environment
# restore environment and reset tempdir (to avoid tmpdir path getting progressively longer)
restore_env(init_env)
tempfile.tempdir = None

# If EasyConfig specific arguments were supplied in EasyStack file
# merge arguments with original command line args
Expand Down
5 changes: 5 additions & 0 deletions easybuild/tools/docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -441,6 +441,7 @@ def avail_easyconfig_templates_txt():
# step 2: add SOFTWARE_VERSIONS
doc.append('Template names/values for (short) software versions')
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
doc.append("%s%%(%smajver)s: major version for %s" % (INDENT_4SPACES, pref, name))
doc.append("%s%%(%sshortver)s: short version for %s (<major>.<minor>)" % (INDENT_4SPACES, pref, name))
doc.append("%s%%(%sver)s: full version for %s" % (INDENT_4SPACES, pref, name))
doc.append('')
Expand Down Expand Up @@ -495,8 +496,10 @@ def avail_easyconfig_templates_rst():
ver = []
ver_desc = []
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
ver.append('``%%(%smajver)s``' % pref)
ver.append('``%%(%sshortver)s``' % pref)
ver.append('``%%(%sver)s``' % pref)
ver_desc.append('major version for %s' % name)
ver_desc.append('short version for %s (<major>.<minor>)' % name)
ver_desc.append('full version for %s' % name)
table_values = [ver, ver_desc]
Expand Down Expand Up @@ -558,8 +561,10 @@ def avail_easyconfig_templates_md():
ver = []
ver_desc = []
for name, pref in TEMPLATE_SOFTWARE_VERSIONS:
ver.append('``%%(%smajver)s``' % pref)
ver.append('``%%(%sshortver)s``' % pref)
ver.append('``%%(%sver)s``' % pref)
ver_desc.append('major version for %s' % name)
ver_desc.append('short version for %s (``<major>.<minor>``)' % name)
ver_desc.append('full version for %s' % name)
table_values = [ver, ver_desc]
Expand Down
1 change: 1 addition & 0 deletions test/framework/docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -785,6 +785,7 @@ def test_avail_easyconfig_templates(self):
r"^Template names/values derived from easyconfig instance",
r"^\s+%\(version_major\)s: Major version",
r"^Template names/values for \(short\) software versions",
r"^\s+%\(pymajver\)s: major version for Python",
r"^\s+%\(pyshortver\)s: short version for Python \(<major>\.<minor>\)",
r"^Template constants that can be used in easyconfigs",
r"^\s+SOURCE_TAR_GZ: Source \.tar\.gz bundle \(%\(name\)s-%\(version\)s.tar.gz\)",
Expand Down
1 change: 1 addition & 0 deletions test/framework/easyblock.py
Original file line number Diff line number Diff line change
Expand Up @@ -1527,6 +1527,7 @@ def test_fetch_sources(self):

def test_download_instructions(self):
"""Test use of download_instructions easyconfig parameter."""

orig_test_ec = '\n'.join([
"easyblock = 'ConfigureMake'",
"name = 'software_with_missing_sources'",
Expand Down
15 changes: 9 additions & 6 deletions test/framework/easyconfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -1294,7 +1294,7 @@ def test_templating_doc(self):
# expected length: 1 per constant and 2 extra per constantgroup (title + empty line in between)
temps = [
easyconfig.templates.TEMPLATE_NAMES_EASYCONFIG,
easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 2,
easyconfig.templates.TEMPLATE_SOFTWARE_VERSIONS * 3,
easyconfig.templates.TEMPLATE_NAMES_CONFIG,
easyconfig.templates.TEMPLATE_NAMES_LOWER,
easyconfig.templates.TEMPLATE_NAMES_EASYBLOCK_RUN_STEP,
Expand Down Expand Up @@ -2140,12 +2140,15 @@ def eval_quoted_string(quoted_val, val):
Helper function to sanity check we can use the quoted string in Python contexts.
Returns the evaluated (i.e. unquoted) string
"""
globals = dict()
scope = dict()
try:
exec('res = %s' % quoted_val, globals)
except Exception as e: # pylint: disable=broad-except
self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, e))
return globals['res']
# this is needlessly complicated because we can't use 'exec' here without potentially running
# into a SyntaxError bug in old Python 2.7 versions (for example when running the tests in CentOS 7.9)
# cfr. https://stackoverflow.com/questions/4484872/why-doesnt-exec-work-in-a-function-with-a-subfunction
eval(compile('res = %s' % quoted_val, '<string>', 'exec'), dict(), scope)
except Exception as err: # pylint: disable=broad-except
self.fail('Failed to evaluate %s (from %s): %s' % (quoted_val, val, err))
return scope['res']

def assertEqual_unquoted(quoted_val, val):
"""Assert that evaluating the quoted_val yields the val"""
Expand Down
Loading

0 comments on commit 9c31fd5

Please sign in to comment.