Skip to content

Commit

Permalink
Merge remote-tracking branch 'upstream/master'
Browse files Browse the repository at this point in the history
  • Loading branch information
gartung committed Nov 5, 2024
2 parents bb81d17 + 0cbd3d0 commit 3b8921b
Show file tree
Hide file tree
Showing 10 changed files with 136 additions and 16 deletions.
1 change: 1 addition & 0 deletions cmssw_l2/commit.txt
Original file line number Diff line number Diff line change
Expand Up @@ -16,3 +16,4 @@ b2da0596ab2006a2df73e3fe1a3d7f870b8c4cb4
ff90f10a35e09ef92f393ea3120a3ac912949a6a
677f11de98f5e87dc5ecdd5a7b28383ccbea69f2
bacc076e91b1b7fd7558b2568849dcf3b749430a
14f28b937aa5e959cf01ab9691148c436f0bacd3
18 changes: 18 additions & 0 deletions crab/CMSSW_5_3_X/pset.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,24 @@
# Source: /local/reps/CMSSW/CMSSW/Configuration/PyReleaseValidation/python/ConfigBuilder.py,v
# with command line options: MinBias_8TeV_cfi --conditions auto:startup -s GEN,SIM --datatier GEN-SIM -n 10 --relval 25000,250 --eventcontent RAWSIM --fileout file:step1.root
# Testing cmsbot crab setup
# Testing cmsbot crab setup
def cmsbot_crab_test():
import sys
try:
from subprocess import getstatusoutput
except:
from commands import getstatusoutput
from FWCore.Version.cmsbot_crab_test import CMSBOT_CRAB_TEST

cmsbot_exit, cmsbot_out = getstatusoutput("cmsbot_crab_test.sh")
print("CMSBOT Crab Test:", CMSBOT_CRAB_TEST, cmsbot_exit, cmsbot_out)
if cmsbot_exit or (cmsbot_out != "OK") or (CMSBOT_CRAB_TEST != "OK"):
sys.exit(1)


cmsbot_crab_test()


import FWCore.ParameterSet.Config as cms

process = cms.Process("SIM")
Expand Down
1 change: 1 addition & 0 deletions crab/CMSSW_8_0_X
1 change: 1 addition & 0 deletions crab/ib-monitor-crab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ while true ; do
sleep 300
done
rm -f $WORKSPACE/status-${CRABCLIENT_TYPE}.log
curl "${GRIDSITE}/job_out.1.0.txt" > $WORKSPACE/testsResults/job_out.log || true
if [ $(echo "${status}" | grep 'finished' | wc -l) -gt 0 ] ; then
echo "PASSED" > $WORKSPACE/testsResults/statusfile-${CRABCLIENT_TYPE}
TEST_PASSED=true
Expand Down
7 changes: 4 additions & 3 deletions crab/ib-run-crab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -42,11 +42,12 @@ else
export CMSRUN_PSET=${thisdir}/pset.py
fi
if [ "${X509_USER_PROXY}" = "" ] ; then
voms-proxy-init -voms cms
voms-proxy-init -voms cms
export X509_USER_PROXY=$(voms-proxy-info -path)
fi
pyver=$(${CMSBOT_PYTHON_CMD} -c 'import sys;print("python%s%s" % (sys.version_info[0],sys.version_info[1]))')
if [ -e ${thisdir}/${pyver} ] ; then export PYTHONPATH="${thisdir}/${pyver}:${PYTHONPATH}"; fi
crab submit -c ${thisdir}/task.py
crab submit --proxy ${X509_USER_PROXY} -c ${thisdir}/task.py
rm -rf ${WORKSPACE}/crab
mv crab_${CRAB_REQUEST} ${WORKSPACE}/crab
echo "INPROGRESS" > $WORKSPACE/crab/statusfile
Expand All @@ -63,7 +64,7 @@ while [ "${GRIDSITE}" = "N/Ayet" ]
do
sleep 10
echo "Gridsite has not been assigned yet!"
export GRIDSITE=$(crab status -d ./crab | grep "Grid scheduler - Task Worker:" | cut -d ":" -f2 | cut -d "-" -f1 | tr -d '\t' | tr -d " ")
export GRIDSITE=$(crab status --proxy ${X509_USER_PROXY} -d ./crab | grep "Grid scheduler - Task Worker:" | cut -d ":" -f2 | cut -d "-" -f1 | tr -d '\t' | tr -d " ")
done

# Store information for the monitoring job
Expand Down
8 changes: 5 additions & 3 deletions crab/ib-run-pr-crab.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ fi
source $WORKSPACE/cms-bot/pr_testing/setup-pr-test-env.sh
source $WORKSPACE/cms-bot/common/github_reports.sh

voms-proxy-init -voms cms

cd $RELEASE_FORMAT
eval `scram run -sh`

Expand All @@ -41,6 +39,10 @@ eval `scram run -sh`
[ "${BUILD_ID}" != "" ] || export BUILD_ID=$(date +%s)
CRABCLIENT_TYPES=$(ls ${PR_CVMFS_PATH}/share/cms/ | grep -Eo '(dev|prod|pre)' || true)
[ "${CRABCLIENT_TYPES}" != "" ] || CRABCLIENT_TYPES="prod"
if [ "${X509_USER_PROXY}" = "" ] ; then
voms-proxy-init -voms cms
export X509_USER_PROXY=$(voms-proxy-info -path)
fi
for CRABCLIENT_TYPE in ${CRABCLIENT_TYPES}
do
# Report PR status
Expand All @@ -50,7 +52,7 @@ do

export CRAB_REQUEST="Jenkins_${CMSSW_VERSION}_${SCRAM_ARCH}_${CRABCLIENT_TYPE}_${BUILD_ID}"
rm -rf crab_${CRAB_REQUEST}
crab-${CRABCLIENT_TYPE} submit -c $(dirname $0)/task.py
crab-${CRABCLIENT_TYPE} submit --proxy ${X509_USER_PROXY} -c $(dirname $0)/task.py

export ID=$(id -u)
export TASK_ID=$(grep crab_${CRAB_REQUEST} crab_${CRAB_REQUEST}/.requestcache | sed 's|^V||')
Expand Down
100 changes: 100 additions & 0 deletions crab/python26/functools.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,100 @@
"""functools.py - Tools for working with functions and callable objects
"""
# Python module wrapper for _functools C module
# to allow utilities written in Python to be added
# to the functools module.
# Written by Nick Coghlan <ncoghlan at gmail.com>
# Copyright (C) 2006 Python Software Foundation.
# See C source code for _functools credits/copyright

from _functools import partial, reduce

# update_wrapper() and wraps() are tools to help write
# wrapper functions that can handle naive introspection

WRAPPER_ASSIGNMENTS = ('__module__', '__name__', '__doc__')
WRAPPER_UPDATES = ('__dict__',)
def update_wrapper(wrapper,
wrapped,
assigned = WRAPPER_ASSIGNMENTS,
updated = WRAPPER_UPDATES):
"""Update a wrapper function to look like the wrapped function
wrapper is the function to be updated
wrapped is the original function
assigned is a tuple naming the attributes assigned directly
from the wrapped function to the wrapper function (defaults to
functools.WRAPPER_ASSIGNMENTS)
updated is a tuple naming the attributes of the wrapper that
are updated with the corresponding attribute from the wrapped
function (defaults to functools.WRAPPER_UPDATES)
"""
for attr in assigned:
setattr(wrapper, attr, getattr(wrapped, attr))
for attr in updated:
getattr(wrapper, attr).update(getattr(wrapped, attr, {}))
# Return the wrapper so this can be used as a decorator via partial()
return wrapper

def wraps(wrapped,
assigned = WRAPPER_ASSIGNMENTS,
updated = WRAPPER_UPDATES):
"""Decorator factory to apply update_wrapper() to a wrapper function
Returns a decorator that invokes update_wrapper() with the decorated
function as the wrapper argument and the arguments to wraps() as the
remaining arguments. Default arguments are as for update_wrapper().
This is a convenience function to simplify applying partial() to
update_wrapper().
"""
return partial(update_wrapper, wrapped=wrapped,
assigned=assigned, updated=updated)

def total_ordering(cls):
"""Class decorator that fills in missing ordering methods"""
convert = {
'__lt__': [('__gt__', lambda self, other: not (self < other or self == other)),
('__le__', lambda self, other: self < other or self == other),
('__ge__', lambda self, other: not self < other)],
'__le__': [('__ge__', lambda self, other: not self <= other or self == other),
('__lt__', lambda self, other: self <= other and not self == other),
('__gt__', lambda self, other: not self <= other)],
'__gt__': [('__lt__', lambda self, other: not (self > other or self == other)),
('__ge__', lambda self, other: self > other or self == other),
('__le__', lambda self, other: not self > other)],
'__ge__': [('__le__', lambda self, other: (not self >= other) or self == other),
('__gt__', lambda self, other: self >= other and not self == other),
('__lt__', lambda self, other: not self >= other)]
}
roots = set(dir(cls)) & set(convert)
if not roots:
raise ValueError('must define at least one ordering operation: < > <= >=')
root = max(roots) # prefer __lt__ to __le__ to __gt__ to __ge__
for opname, opfunc in convert[root]:
if opname not in roots:
opfunc.__name__ = opname
opfunc.__doc__ = getattr(int, opname).__doc__
setattr(cls, opname, opfunc)
return cls

def cmp_to_key(mycmp):
"""Convert a cmp= function into a key= function"""
class K(object):
__slots__ = ['obj']
def __init__(self, obj, *args):
self.obj = obj
def __lt__(self, other):
return mycmp(self.obj, other.obj) < 0
def __gt__(self, other):
return mycmp(self.obj, other.obj) > 0
def __eq__(self, other):
return mycmp(self.obj, other.obj) == 0
def __le__(self, other):
return mycmp(self.obj, other.obj) <= 0
def __ge__(self, other):
return mycmp(self.obj, other.obj) >= 0
def __ne__(self, other):
return mycmp(self.obj, other.obj) != 0
def __hash__(self):
raise TypeError('hash not implemented')
return K
13 changes: 4 additions & 9 deletions docker_launcher.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,10 @@ export DBS_URL=https://cmsweb.cern.ch:8443/dbs/prod/global/DBSReader
export GIT_CONFIG_NOSYSTEM=1
if [ "X$WORKSPACE" = "X" ] ; then export WORKSPACE=$(/bin/pwd) ; fi
if [ "${X509_USER_PROXY}" = "" ] ; then
x509_proxyfile=x509up_u`id -u`
export X509_USER_PROXY=$WORKSPACE/${x509_proxyfile}
#Make sure to delete /tmp/${x509_proxyfile} as dasgoclient prefer to read it instead of $X509_USER_PROXY
#See https://github.com/dmwm/dasgoclient/issues/37
[ ! -e /tmp/${x509_proxyfile} ] || rm -f /tmp/${x509_proxyfile}
export X509_USER_PROXY=${WORKSPACE}/x509up_u$(id -u)
voms-proxy-init -voms cms -rfc -valid 24:00 || true
fi

XPATH=""
py3or2_dir="$HOME/bin"
if [ ! -e ${py3or2_dir} ] ; then
Expand Down Expand Up @@ -95,10 +93,7 @@ if [ "X$DOCKER_IMG" != X -a "X$RUN_NATIVE" = "X" ]; then
if [ -d $HOME/bin ] ; then
CMD2RUN="${CMD2RUN}export PATH=\$HOME/bin:\$PATH; "
fi
if [ "$X509_USER_PROXY" = "" ] ; then
CMD2RUN="${CMD2RUN}voms-proxy-init -voms cms -rfc -valid 24:00 || true ; voms-proxy-info || true; "
fi
CMD2RUN="${CMD2RUN}echo \$HOME; cd $WORKSPACE; echo \$PATH; $@"
CMD2RUN="${CMD2RUN}export X509_USER_PROXY=${X509_USER_PROXY}; if [ ! -e ${X509_USER_PROXY} ] ; then voms-proxy-init -voms cms -rfc -valid 24:00 || true ; voms-proxy-info || true; fi; echo \$HOME; cd $WORKSPACE; echo \$PATH; $@"
if $HAS_DOCKER ; then
docker pull $DOCKER_IMG
set +x
Expand Down
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
[tool.black]
line-length = 99
target-version = ['py36', 'py37', 'py38', 'py39', 'py310', 'py311']
exclude = 'crab/python26/functools.py'

[tool.pylint.main]
# Analyse import fallback blocks. This can be used to support both Python 2 and 3
Expand Down
2 changes: 1 addition & 1 deletion run-ib-testbase.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ cvmfs_config probe || true
for cvmfs_dir in cms-ci.cern.ch \$(grep CVMFS_REPOSITORIES= /etc/cvmfs/default.local | sed "s|.*=||;s|'||g" | sed 's|"||g' | tr ',' '\n' | grep cern.ch) ; do
ls -l /cvmfs/\${cvmfs_dir} >/dev/null 2>&1 || true
done
if [ "${X509_USER_PROXY}" = "" ] ; then
if [ "\${X509_USER_PROXY}" = "" -o ! -e "\${X509_USER_PROXY}" ] ; then
voms-proxy-init -voms cms || true
voms-proxy-info || true
fi
Expand Down

0 comments on commit 3b8921b

Please sign in to comment.