From 5bfac15bcbfeb06750a0303fd681baaeed3f2283 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 5 Nov 2024 08:22:32 +0100 Subject: [PATCH 01/15] Update docker_launcher.sh --- docker_launcher.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/docker_launcher.sh b/docker_launcher.sh index ed914d9041d..fe055e52cc9 100755 --- a/docker_launcher.sh +++ b/docker_launcher.sh @@ -24,6 +24,7 @@ if [ "X$WORKSPACE" = "X" ] ; then export WORKSPACE=$(/bin/pwd) ; fi if [ "${X509_USER_PROXY}" = "" ] ; then x509_proxyfile=x509up_u`id -u` export X509_USER_PROXY=$WORKSPACE/${x509_proxyfile} + voms-proxy-init -voms cms || true #Make sure to delete /tmp/${x509_proxyfile} as dasgoclient prefer to read it instead of $X509_USER_PROXY #See https://github.com/dmwm/dasgoclient/issues/37 [ ! -e /tmp/${x509_proxyfile} ] || rm -f /tmp/${x509_proxyfile} From 7d723fa9ded849b0beb1a926daad3a808c4348b9 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 5 Nov 2024 08:25:26 +0100 Subject: [PATCH 02/15] Update run-ib-testbase.sh --- run-ib-testbase.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-ib-testbase.sh b/run-ib-testbase.sh index 6ef86bc23bf..6e7af8bf743 100755 --- a/run-ib-testbase.sh +++ b/run-ib-testbase.sh @@ -9,7 +9,7 @@ cvmfs_config probe || true for cvmfs_dir in cms-ci.cern.ch \$(grep CVMFS_REPOSITORIES= /etc/cvmfs/default.local | sed "s|.*=||;s|'||g" | sed 's|"||g' | tr ',' '\n' | grep cern.ch) ; do ls -l /cvmfs/\${cvmfs_dir} >/dev/null 2>&1 || true done -if [ "${X509_USER_PROXY}" = "" ] ; then +if [ "${X509_USER_PROXY}" = "" -o ! -e "${X509_USER_PROXY}" ] ; then voms-proxy-init -voms cms || true voms-proxy-info || true fi From e731d2587004f6b275c919ebb4785a276743cebe Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 5 Nov 2024 08:26:02 +0100 Subject: [PATCH 03/15] Update run-ib-testbase.sh --- run-ib-testbase.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-ib-testbase.sh b/run-ib-testbase.sh index 6e7af8bf743..030fe100c77 100755 --- a/run-ib-testbase.sh +++ b/run-ib-testbase.sh @@ -9,7 +9,7 @@ cvmfs_config probe || true for cvmfs_dir in cms-ci.cern.ch \$(grep CVMFS_REPOSITORIES= /etc/cvmfs/default.local | sed "s|.*=||;s|'||g" | sed 's|"||g' | tr ',' '\n' | grep cern.ch) ; do ls -l /cvmfs/\${cvmfs_dir} >/dev/null 2>&1 || true done -if [ "${X509_USER_PROXY}" = "" -o ! -e "${X509_USER_PROXY}" ] ; then +if [ "\${X509_USER_PROXY}" = "" -o ! -e "\${X509_USER_PROXY}" ] ; then voms-proxy-init -voms cms || true voms-proxy-info || true fi From a27266f043c697cac0df4596475f82212f4b0b00 Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 5 Nov 2024 08:32:26 +0100 Subject: [PATCH 04/15] Update docker_launcher.sh --- docker_launcher.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/docker_launcher.sh b/docker_launcher.sh index fe055e52cc9..12ed1ffea96 100755 --- a/docker_launcher.sh +++ b/docker_launcher.sh @@ -24,10 +24,14 @@ if [ "X$WORKSPACE" = "X" ] ; then export WORKSPACE=$(/bin/pwd) ; fi if [ "${X509_USER_PROXY}" = "" ] ; then x509_proxyfile=x509up_u`id -u` export X509_USER_PROXY=$WORKSPACE/${x509_proxyfile} - voms-proxy-init -voms cms || true #Make sure to delete /tmp/${x509_proxyfile} as dasgoclient prefer to read it instead of $X509_USER_PROXY #See https://github.com/dmwm/dasgoclient/issues/37 - [ ! -e /tmp/${x509_proxyfile} ] || rm -f /tmp/${x509_proxyfile} + if [ -e /tmp/${x509_proxyfile} ] ; then + mv /tmp/${x509_proxyfile} ${X509_USER_PROXY} + else + rm -f /tmp/${x509_proxyfile} + voms-proxy-init -voms cms || true + fi fi XPATH="" py3or2_dir="$HOME/bin" From 8e2e14c80b07f10243282ed72b2eb81a748b38c2 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 09:03:28 +0100 Subject: [PATCH 05/15] run voms-proxy-init from the container --- docker_launcher.sh | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/docker_launcher.sh b/docker_launcher.sh index 12ed1ffea96..ac2626a5e82 100755 --- a/docker_launcher.sh +++ b/docker_launcher.sh @@ -21,18 +21,10 @@ fi export DBS_URL=https://cmsweb.cern.ch:8443/dbs/prod/global/DBSReader export GIT_CONFIG_NOSYSTEM=1 if [ "X$WORKSPACE" = "X" ] ; then export WORKSPACE=$(/bin/pwd) ; fi -if [ "${X509_USER_PROXY}" = "" ] ; then - x509_proxyfile=x509up_u`id -u` - export X509_USER_PROXY=$WORKSPACE/${x509_proxyfile} - #Make sure to delete /tmp/${x509_proxyfile} as dasgoclient prefer to read it instead of $X509_USER_PROXY - #See https://github.com/dmwm/dasgoclient/issues/37 - if [ -e /tmp/${x509_proxyfile} ] ; then - mv /tmp/${x509_proxyfile} ${X509_USER_PROXY} - else - rm -f /tmp/${x509_proxyfile} - voms-proxy-init -voms cms || true - fi -fi +x509_proxyfile=x509up_u`id -u` +#Make sure to delete /tmp/${x509_proxyfile} as dasgoclient prefer to read it instead of $X509_USER_PROXY +#See https://github.com/dmwm/dasgoclient/issues/37 +[ ! -e /tmp/${x509_proxyfile} ] || rm -f /tmp/${x509_proxyfile} XPATH="" py3or2_dir="$HOME/bin" if [ ! -e ${py3or2_dir} ] ; then @@ -100,10 +92,7 @@ if [ "X$DOCKER_IMG" != X -a "X$RUN_NATIVE" = "X" ]; then if [ -d $HOME/bin ] ; then CMD2RUN="${CMD2RUN}export PATH=\$HOME/bin:\$PATH; " fi - if [ "$X509_USER_PROXY" = "" ] ; then - CMD2RUN="${CMD2RUN}voms-proxy-init -voms cms -rfc -valid 24:00 || true ; voms-proxy-info || true; " - fi - CMD2RUN="${CMD2RUN}echo \$HOME; cd $WORKSPACE; echo \$PATH; $@" + CMD2RUN="${CMD2RUN}export X509_USER_PROXY=${WORKSPACE}/${x509_proxyfile}; voms-proxy-init -voms cms -rfc -valid 24:00 || true ; voms-proxy-info || true; echo \$HOME; cd $WORKSPACE; echo \$PATH; $@" if $HAS_DOCKER ; then docker pull $DOCKER_IMG set +x From 562216375c36972570dd8d407e1f0ea046e15453 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 11:15:13 +0100 Subject: [PATCH 06/15] pass --proxy file to crab commands --- crab/ib-run-crab.sh | 9 ++++----- crab/ib-run-pr-crab.sh | 8 ++++---- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/crab/ib-run-crab.sh b/crab/ib-run-crab.sh index a932c6ade74..e69254e54d1 100755 --- a/crab/ib-run-crab.sh +++ b/crab/ib-run-crab.sh @@ -41,12 +41,11 @@ if [ -e ${thisdir}/${cmssw_queue}/pset.py ] ; then else export CMSRUN_PSET=${thisdir}/pset.py fi -if [ "${X509_USER_PROXY}" = "" ] ; then - voms-proxy-init -voms cms -fi +[ "${X509_USER_PROXY}" = "" ] && voms-proxy-init -voms cms +x509proxy=$(voms-proxy-info -path) pyver=$(${CMSBOT_PYTHON_CMD} -c 'import sys;print("python%s%s" % (sys.version_info[0],sys.version_info[1]))') if [ -e ${thisdir}/${pyver} ] ; then export PYTHONPATH="${thisdir}/${pyver}:${PYTHONPATH}"; fi -crab submit -c ${thisdir}/task.py +crab submit --proxy ${x509proxy} -c ${thisdir}/task.py rm -rf ${WORKSPACE}/crab mv crab_${CRAB_REQUEST} ${WORKSPACE}/crab echo "INPROGRESS" > $WORKSPACE/crab/statusfile @@ -63,7 +62,7 @@ while [ "${GRIDSITE}" = "N/Ayet" ] do sleep 10 echo "Gridsite has not been assigned yet!" - export GRIDSITE=$(crab status -d ./crab | grep "Grid scheduler - Task Worker:" | cut -d ":" -f2 | cut -d "-" -f1 | tr -d '\t' | tr -d " ") + export GRIDSITE=$(crab status --proxy ${x509proxy} -d ./crab | grep "Grid scheduler - Task Worker:" | cut -d ":" -f2 | cut -d "-" -f1 | tr -d '\t' | tr -d " ") done # Store information for the monitoring job diff --git a/crab/ib-run-pr-crab.sh b/crab/ib-run-pr-crab.sh index 9f8057abdeb..795c9824603 100755 --- a/crab/ib-run-pr-crab.sh +++ b/crab/ib-run-pr-crab.sh @@ -23,8 +23,6 @@ fi source $WORKSPACE/cms-bot/pr_testing/setup-pr-test-env.sh source $WORKSPACE/cms-bot/common/github_reports.sh -voms-proxy-init -voms cms - cd $RELEASE_FORMAT eval `scram run -sh` @@ -41,6 +39,8 @@ eval `scram run -sh` [ "${BUILD_ID}" != "" ] || export BUILD_ID=$(date +%s) CRABCLIENT_TYPES=$(ls ${PR_CVMFS_PATH}/share/cms/ | grep -Eo '(dev|prod|pre)' || true) [ "${CRABCLIENT_TYPES}" != "" ] || CRABCLIENT_TYPES="prod" +[ "${X509_USER_PROXY}" = "" ] && voms-proxy-init -voms cms +x509proxy=$(voms-proxy-info -path) for CRABCLIENT_TYPE in ${CRABCLIENT_TYPES} do # Report PR status @@ -50,7 +50,7 @@ do export CRAB_REQUEST="Jenkins_${CMSSW_VERSION}_${SCRAM_ARCH}_${CRABCLIENT_TYPE}_${BUILD_ID}" rm -rf crab_${CRAB_REQUEST} - crab-${CRABCLIENT_TYPE} submit -c $(dirname $0)/task.py + crab-${CRABCLIENT_TYPE} submit --proxy ${x509proxy} -c $(dirname $0)/task.py export ID=$(id -u) export TASK_ID=$(grep crab_${CRAB_REQUEST} crab_${CRAB_REQUEST}/.requestcache | sed 's|^V||') @@ -60,7 +60,7 @@ do while [ "${GRIDSITE}" = "" ] do sleep 10 - export GRIDSITE=$(curl -s -X GET --cert "${X509_USER_PROXY}" --key "${X509_USER_PROXY}" --capath "/etc/grid-security/certificates/" "https://cmsweb.cern.ch:8443/crabserver/prod/task?subresource=search&workflow=${TASK_ID}" | grep -o "http.*/${TASK_ID}") + export GRIDSITE=$(curl -s -X GET --cert "${x509proxy}" --key "${x509proxy}" --capath "/etc/grid-security/certificates/" "https://cmsweb.cern.ch:8443/crabserver/prod/task?subresource=search&workflow=${TASK_ID}" | grep -o "http.*/${TASK_ID}") done # Store information for the monitoring job From ca25a36dcfdc38e2079cbb19b7aad37152276f5d Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 11:20:24 +0100 Subject: [PATCH 07/15] use X509_USER_PROXY if set --- crab/ib-run-crab.sh | 10 ++++++---- crab/ib-run-pr-crab.sh | 10 ++++++---- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/crab/ib-run-crab.sh b/crab/ib-run-crab.sh index e69254e54d1..50d7bfeb896 100755 --- a/crab/ib-run-crab.sh +++ b/crab/ib-run-crab.sh @@ -41,11 +41,13 @@ if [ -e ${thisdir}/${cmssw_queue}/pset.py ] ; then else export CMSRUN_PSET=${thisdir}/pset.py fi -[ "${X509_USER_PROXY}" = "" ] && voms-proxy-init -voms cms -x509proxy=$(voms-proxy-info -path) +if [ "${X509_USER_PROXY}" = "" ] ; then + voms-proxy-init -voms cms + export X509_USER_PROXY=$(voms-proxy-info -path) +fi pyver=$(${CMSBOT_PYTHON_CMD} -c 'import sys;print("python%s%s" % (sys.version_info[0],sys.version_info[1]))') if [ -e ${thisdir}/${pyver} ] ; then export PYTHONPATH="${thisdir}/${pyver}:${PYTHONPATH}"; fi -crab submit --proxy ${x509proxy} -c ${thisdir}/task.py +crab submit --proxy ${X509_USER_PROXY} -c ${thisdir}/task.py rm -rf ${WORKSPACE}/crab mv crab_${CRAB_REQUEST} ${WORKSPACE}/crab echo "INPROGRESS" > $WORKSPACE/crab/statusfile @@ -62,7 +64,7 @@ while [ "${GRIDSITE}" = "N/Ayet" ] do sleep 10 echo "Gridsite has not been assigned yet!" - export GRIDSITE=$(crab status --proxy ${x509proxy} -d ./crab | grep "Grid scheduler - Task Worker:" | cut -d ":" -f2 | cut -d "-" -f1 | tr -d '\t' | tr -d " ") + export GRIDSITE=$(crab status --proxy ${X509_USER_PROXY} -d ./crab | grep "Grid scheduler - Task Worker:" | cut -d ":" -f2 | cut -d "-" -f1 | tr -d '\t' | tr -d " ") done # Store information for the monitoring job diff --git a/crab/ib-run-pr-crab.sh b/crab/ib-run-pr-crab.sh index 795c9824603..add4beade22 100755 --- a/crab/ib-run-pr-crab.sh +++ b/crab/ib-run-pr-crab.sh @@ -39,8 +39,10 @@ eval `scram run -sh` [ "${BUILD_ID}" != "" ] || export BUILD_ID=$(date +%s) CRABCLIENT_TYPES=$(ls ${PR_CVMFS_PATH}/share/cms/ | grep -Eo '(dev|prod|pre)' || true) [ "${CRABCLIENT_TYPES}" != "" ] || CRABCLIENT_TYPES="prod" -[ "${X509_USER_PROXY}" = "" ] && voms-proxy-init -voms cms -x509proxy=$(voms-proxy-info -path) +if [ "${X509_USER_PROXY}" = "" ] ; then + voms-proxy-init -voms cms + export X509_USER_PROXY=$(voms-proxy-info -path) +fi for CRABCLIENT_TYPE in ${CRABCLIENT_TYPES} do # Report PR status @@ -50,7 +52,7 @@ do export CRAB_REQUEST="Jenkins_${CMSSW_VERSION}_${SCRAM_ARCH}_${CRABCLIENT_TYPE}_${BUILD_ID}" rm -rf crab_${CRAB_REQUEST} - crab-${CRABCLIENT_TYPE} submit --proxy ${x509proxy} -c $(dirname $0)/task.py + crab-${CRABCLIENT_TYPE} submit --proxy ${X509_USER_PROXY} -c $(dirname $0)/task.py export ID=$(id -u) export TASK_ID=$(grep crab_${CRAB_REQUEST} crab_${CRAB_REQUEST}/.requestcache | sed 's|^V||') @@ -60,7 +62,7 @@ do while [ "${GRIDSITE}" = "" ] do sleep 10 - export GRIDSITE=$(curl -s -X GET --cert "${x509proxy}" --key "${x509proxy}" --capath "/etc/grid-security/certificates/" "https://cmsweb.cern.ch:8443/crabserver/prod/task?subresource=search&workflow=${TASK_ID}" | grep -o "http.*/${TASK_ID}") + export GRIDSITE=$(curl -s -X GET --cert "${X509_USER_PROXY}" --key "${X509_USER_PROXY}" --capath "/etc/grid-security/certificates/" "https://cmsweb.cern.ch:8443/crabserver/prod/task?subresource=search&workflow=${TASK_ID}" | grep -o "http.*/${TASK_ID}") done # Store information for the monitoring job From c18129bf9849973bc2657892bd26e8bc507d14e4 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 11:32:06 +0100 Subject: [PATCH 08/15] use proxyfrom workspace --- docker_launcher.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker_launcher.sh b/docker_launcher.sh index ac2626a5e82..6ab11b067b3 100755 --- a/docker_launcher.sh +++ b/docker_launcher.sh @@ -21,10 +21,11 @@ fi export DBS_URL=https://cmsweb.cern.ch:8443/dbs/prod/global/DBSReader export GIT_CONFIG_NOSYSTEM=1 if [ "X$WORKSPACE" = "X" ] ; then export WORKSPACE=$(/bin/pwd) ; fi -x509_proxyfile=x509up_u`id -u` -#Make sure to delete /tmp/${x509_proxyfile} as dasgoclient prefer to read it instead of $X509_USER_PROXY -#See https://github.com/dmwm/dasgoclient/issues/37 -[ ! -e /tmp/${x509_proxyfile} ] || rm -f /tmp/${x509_proxyfile} +if [ "${X509_USER_PROXY}" = "" ] ; then + export X509_USER_PROXY=${WORKSPACE}/x509up_u$(id -u) + voms-proxy-init -voms cms -rfc -valid 24:00 || true +fi + XPATH="" py3or2_dir="$HOME/bin" if [ ! -e ${py3or2_dir} ] ; then @@ -92,7 +93,7 @@ if [ "X$DOCKER_IMG" != X -a "X$RUN_NATIVE" = "X" ]; then if [ -d $HOME/bin ] ; then CMD2RUN="${CMD2RUN}export PATH=\$HOME/bin:\$PATH; " fi - CMD2RUN="${CMD2RUN}export X509_USER_PROXY=${WORKSPACE}/${x509_proxyfile}; voms-proxy-init -voms cms -rfc -valid 24:00 || true ; voms-proxy-info || true; echo \$HOME; cd $WORKSPACE; echo \$PATH; $@" + CMD2RUN="${CMD2RUN}export X509_USER_PROXY=${X509_USER_PROXY}; if [ ! -e ${X509_USER_PROXY} ] ; then voms-proxy-init -voms cms -rfc -valid 24:00 || true ; voms-proxy-info || true; fi; echo \$HOME; cd $WORKSPACE; echo \$PATH; $@" if $HAS_DOCKER ; then docker pull $DOCKER_IMG set +x From 3d7ebfbf268d12c6c943342e7af5b25f9963776e Mon Sep 17 00:00:00 2001 From: Malik Shahzad Muzaffar Date: Tue, 5 Nov 2024 12:00:00 +0100 Subject: [PATCH 09/15] Update black.yaml --- .github/workflows/black.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index af1ee5ce1d7..46c2159e886 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -16,4 +16,4 @@ jobs: - uses: psf/black@stable with: - options: "--check --diff --verbose -l 99 -t py36 -t py37 -t py38 -t py39 -t py310 -t py311" + options: "--exclude crab/python26/functools.py --check --diff --verbose -l 99 -t py36 -t py37 -t py38 -t py39 -t py310 -t py311" From 689184e0ad8c4a8e7fb9b5347e606c738c60f5ea Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 12:00:36 +0100 Subject: [PATCH 10/15] added py26/functools.py --- crab/python26/functools.py | 100 +++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 crab/python26/functools.py diff --git a/crab/python26/functools.py b/crab/python26/functools.py new file mode 100644 index 00000000000..53680b89466 --- /dev/null +++ b/crab/python26/functools.py @@ -0,0 +1,100 @@ +"""functools.py - Tools for working with functions and callable objects +""" +# Python module wrapper for _functools C module +# to allow utilities written in Python to be added +# to the functools module. +# Written by Nick Coghlan +# Copyright (C) 2006 Python Software Foundation. +# See C source code for _functools credits/copyright + +from _functools import partial, reduce + +# update_wrapper() and wraps() are tools to help write +# wrapper functions that can handle naive introspection + +WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__') +WRAPPER_UPDATES = ('__dict__',) +def update_wrapper(wrapper, + wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Update a wrapper function to look like the wrapped function + + wrapper is the function to be updated + wrapped is the original function + assigned is a tuple naming the attributes assigned directly + from the wrapped function to the wrapper function (defaults to + functools.WRAPPER_ASSIGNMENTS) + updated is a tuple naming the attributes of the wrapper that + are updated with the corresponding attribute from the wrapped + function (defaults to functools.WRAPPER_UPDATES) + """ + for attr in assigned: + setattr(wrapper, attr, getattr(wrapped, attr)) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + # Return the wrapper so this can be used as a decorator via partial() + return wrapper + +def wraps(wrapped, + assigned = WRAPPER_ASSIGNMENTS, + updated = WRAPPER_UPDATES): + """Decorator factory to apply update_wrapper() to a wrapper function + + Returns a decorator that invokes update_wrapper() with the decorated + function as the wrapper argument and the arguments to wraps() as the + remaining arguments. Default arguments are as for update_wrapper(). + This is a convenience function to simplify applying partial() to + update_wrapper(). + """ + return partial(update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + +def total_ordering(cls): + """Class decorator that fills in missing ordering methods""" + convert = { + '__lt__': [('__gt__', lambda self, other: not (self < other or self == other)), + ('__le__', lambda self, other: self < other or self == other), + ('__ge__', lambda self, other: not self < other)], + '__le__': [('__ge__', lambda self, other: not self <= other or self == other), + ('__lt__', lambda self, other: self <= other and not self == other), + ('__gt__', lambda self, other: not self <= other)], + '__gt__': [('__lt__', lambda self, other: not (self > other or self == other)), + ('__ge__', lambda self, other: self > other or self == other), + ('__le__', lambda self, other: not self > other)], + '__ge__': [('__le__', lambda self, other: (not self >= other) or self == other), + ('__gt__', lambda self, other: self >= other and not self == other), + ('__lt__', lambda self, other: not self >= other)] + } + roots = set(dir(cls)) & set(convert) + if not roots: + raise ValueError('must define at least one ordering operation: < > <= >=') + root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__ + for opname, opfunc in convert[root]: + if opname not in roots: + opfunc.__name__ = opname + opfunc.__doc__ = getattr(int, opname).__doc__ + setattr(cls, opname, opfunc) + return cls + +def cmp_to_key(mycmp): + """Convert a cmp= function into a key= function""" + class K(object): + __slots__ = ['obj'] + def __init__(self, obj, *args): + self.obj = obj + def __lt__(self, other): + return mycmp(self.obj, other.obj) < 0 + def __gt__(self, other): + return mycmp(self.obj, other.obj) > 0 + def __eq__(self, other): + return mycmp(self.obj, other.obj) == 0 + def __le__(self, other): + return mycmp(self.obj, other.obj) <= 0 + def __ge__(self, other): + return mycmp(self.obj, other.obj) >= 0 + def __ne__(self, other): + return mycmp(self.obj, other.obj) != 0 + def __hash__(self): + raise TypeError('hash not implemented') + return K From 77e2a8c062f63f7ff67da22aefc1c51a5486df06 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 12:12:05 +0100 Subject: [PATCH 11/15] ignore crab/python26/functools.py for black formatting --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 2e12ecf7cfc..abb9c18dd55 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [tool.black] line-length = 99 target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311'] +exclude = 'crab/python26/functools.py' [tool.pylint.main] # Analyse import fallback blocks. This can be used to support both Python 2 and 3 From 14f28b937aa5e959cf01ab9691148c436f0bacd3 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 12:12:46 +0100 Subject: [PATCH 12/15] cleanup black.yaml --- .github/workflows/black.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/black.yaml b/.github/workflows/black.yaml index 46c2159e886..af1ee5ce1d7 100644 --- a/.github/workflows/black.yaml +++ b/.github/workflows/black.yaml @@ -16,4 +16,4 @@ jobs: - uses: psf/black@stable with: - options: "--exclude crab/python26/functools.py --check --diff --verbose -l 99 -t py36 -t py37 -t py38 -t py39 -t py310 -t py311" + options: "--check --diff --verbose -l 99 -t py36 -t py37 -t py38 -t py39 -t py310 -t py311" From dc280d800970c9083440d6c461d36d14f019f6b3 Mon Sep 17 00:00:00 2001 From: Cms Build Date: Tue, 5 Nov 2024 12:13:18 +0100 Subject: [PATCH 13/15] Updated CMSSW L2 category information 14f28b937aa5e959cf01ab9691148c436f0bacd3 --- cmssw_l2/commit.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cmssw_l2/commit.txt b/cmssw_l2/commit.txt index ffd3dbb2ce9..e21f11412ea 100644 --- a/cmssw_l2/commit.txt +++ b/cmssw_l2/commit.txt @@ -16,3 +16,4 @@ b2da0596ab2006a2df73e3fe1a3d7f870b8c4cb4 ff90f10a35e09ef92f393ea3120a3ac912949a6a 677f11de98f5e87dc5ecdd5a7b28383ccbea69f2 bacc076e91b1b7fd7558b2568849dcf3b749430a +14f28b937aa5e959cf01ab9691148c436f0bacd3 From 4f3cc9d0077c88aae21b74ec71ec78dadb135d42 Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 15:57:07 +0100 Subject: [PATCH 14/15] add 8.0.X crab pset --- crab/CMSSW_5_3_X/pset.py | 18 ++++++++++++++++++ crab/CMSSW_8_0_X | 1 + 2 files changed, 19 insertions(+) create mode 120000 crab/CMSSW_8_0_X diff --git a/crab/CMSSW_5_3_X/pset.py b/crab/CMSSW_5_3_X/pset.py index 9db958e129b..2387d2e52f2 100644 --- a/crab/CMSSW_5_3_X/pset.py +++ b/crab/CMSSW_5_3_X/pset.py @@ -4,6 +4,24 @@ # Source: /local/reps/CMSSW/CMSSW/Configuration/PyReleaseValidation/python/ConfigBuilder.py,v # with command line options: MinBias_8TeV_cfi --conditions auto:startup -s GEN,SIM --datatier GEN-SIM -n 10 --relval 25000,250 --eventcontent RAWSIM --fileout file:step1.root # Testing cmsbot crab setup +# Testing cmsbot crab setup +def cmsbot_crab_test(): + import sys + try: + from subprocess import getstatusoutput + except: + from commands import getstatusoutput + from FWCore.Version.cmsbot_crab_test import CMSBOT_CRAB_TEST + + cmsbot_exit, cmsbot_out = getstatusoutput("cmsbot_crab_test.sh") + print("CMSBOT Crab Test:", CMSBOT_CRAB_TEST, cmsbot_exit, cmsbot_out) + if cmsbot_exit or (cmsbot_out != "OK") or (CMSBOT_CRAB_TEST != "OK"): + sys.exit(1) + + +cmsbot_crab_test() + + import FWCore.ParameterSet.Config as cms process = cms.Process("SIM") diff --git a/crab/CMSSW_8_0_X b/crab/CMSSW_8_0_X new file mode 120000 index 00000000000..e8a70b8827c --- /dev/null +++ b/crab/CMSSW_8_0_X @@ -0,0 +1 @@ +CMSSW_5_3_X \ No newline at end of file From 0cbd3d07ea0378e0e8b601ccdb2b8c9ef670b61c Mon Sep 17 00:00:00 2001 From: Shahzad Malik Muzaffar Date: Tue, 5 Nov 2024 16:31:22 +0100 Subject: [PATCH 15/15] download crab job log --- crab/ib-monitor-crab.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/crab/ib-monitor-crab.sh b/crab/ib-monitor-crab.sh index 253ae4be54e..47184e68063 100755 --- a/crab/ib-monitor-crab.sh +++ b/crab/ib-monitor-crab.sh @@ -25,6 +25,7 @@ while true ; do sleep 300 done rm -f $WORKSPACE/status-${CRABCLIENT_TYPE}.log +curl "${GRIDSITE}/job_out.1.0.txt" > $WORKSPACE/testsResults/job_out.log || true if [ $(echo "${status}" | grep 'finished' | wc -l) -gt 0 ] ; then echo "PASSED" > $WORKSPACE/testsResults/statusfile-${CRABCLIENT_TYPE} TEST_PASSED=true