forked from Bioconductor/BBS
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathBBS-make-OUTGOING.py
executable file
·177 lines (158 loc) · 6.7 KB
/
BBS-make-OUTGOING.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
#!/usr/bin/env python3
##############################################################################
###
### This file is part of the BBS software (Bioconductor Build System).
###
### Author: Hervé Pagès <hpages.on.github@gmail.com>
### Last modification: Nov 22, 2023
###
import sys
import os
import time
import shutil
import bbs.fileutils
import bbs.parse
import BBSutils
import BBSvars
import BBSbase
def is_doing_buildbin(node_hostname):
return BBSutils.getNodeSpec(node_hostname, 'pkgType') != 'source'
def block_package(node_hostname, node_id, pkg):
nodes_path = BBSvars.products_in_rdir.path
node_path = os.path.join(nodes_path, node_id)
summary_file0 = '%s.%%s-summary.dcf' % pkg
## Extract Status from BUILD summary.
buildsrc_path = os.path.join(node_path, 'buildsrc')
summary_file = os.path.join(buildsrc_path, summary_file0 % 'buildsrc')
## Could happen that summary file is not available (because the node
## where that file is coming from didn't finish to build yet or failed
## to send the file back).
try:
dcf = open(summary_file, 'rb')
except IOError:
return True
status = bbs.parse.get_next_DCF_val(dcf, 'Status')
dcf.close()
if status != 'OK':
return True
## Not going further for workflows builds.
if BBSvars.buildtype == 'workflows':
return status != 'OK'
## Extract Status from CHECK summary.
checksrc_path = os.path.join(node_path, 'checksrc')
summary_file = os.path.join(checksrc_path, summary_file0 % 'checksrc')
try:
dcf = open(summary_file, 'rb')
except IOError:
return True
status = bbs.parse.get_next_DCF_val(dcf, 'Status')
dcf.close()
if status not in ['OK', 'WARNINGS']:
return True
if not is_doing_buildbin(node_hostname):
return False
## Extract Status from BUILD BIN summary.
buildbin_path = os.path.join(node_path, 'buildbin')
summary_file = os.path.join(buildbin_path, summary_file0 % 'buildbin')
try:
dcf = open(summary_file, 'rb')
except IOError:
return True
status = bbs.parse.get_next_DCF_val(dcf, 'Status')
dcf.close()
return status != 'OK'
### Copy the outgoing packages to the current directory.
def copy_outgoing_pkgs(products_in_subdir, source_node):
tmp = products_in_subdir.split('/')
if len(tmp) != 2:
sys.exit('ERROR: Invalid relative path to fresh pkgs %s (must be of the form node/subdir)' % products_in_subdir)
node_id = tmp[0]
node_hostname = node_id.split('-')[0]
fileext = BBSutils.getNodeSpec(node_hostname, 'pkgFileExt')
srcdir = os.path.join(BBSvars.products_in_rdir.path, products_in_subdir)
if not os.path.exists(srcdir):
errmsg = "Directory '%s' does not exist!\n\n" % srcdir + \
" %s is late or stopped building?" % node_hostname
raise FileExistsError(errmsg)
## Workflow and book packages do not have manuals/ because we do not run
## `R CMD check`.
manuals_dir = '../manuals'
if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']:
pass
elif source_node:
print('BBS> [stage6b] mkdir %s' % manuals_dir)
os.mkdir(manuals_dir)
print('BBS> [stage6b] BEGIN copying outgoing packages from %s.' % srcdir)
node_Arch = BBSutils.getNodeSpec(node_hostname, 'Arch')
node_pkgType = BBSutils.getNodeSpec(node_hostname, 'pkgType')
meat_index_path = os.path.join(BBSvars.Central_rdir.path,
BBSutils.meat_index_file)
pkgs = bbs.parse.get_meat_packages_for_node(meat_index_path, node_hostname,
node_Arch, node_pkgType)
meat_index = bbs.parse.get_meat_packages(meat_index_path, as_dict=True)
for pkg in pkgs:
if block_package(node_hostname, node_id, pkg):
continue
dcf_record = meat_index[pkg]
version = dcf_record['Version']
## Copy pkg from 'srcdir'.
pkg_file = '%s_%s.%s' % (pkg, version, fileext)
pkg_path = os.path.join(srcdir, pkg_file)
print('BBS> [stage6b] - copying %s to OUTGOING folder ...' % pkg_path)
if os.path.exists(pkg_path):
#shutil.copy(pkg_path, '.')
os.link(pkg_path, pkg_file) # create hard link to avoid making a copy
else:
print("BBS> [stage6b] SKIPPED (file %s doesn't exist)" % pkg_path)
## Get reference manual from pkg.Rcheck directory.
if BBSvars.buildtype in ['workflows', 'books', 'bioc-mac-arm64']:
pass
elif source_node:
pdf_file = os.path.join(BBSutils.getenv('BBS_WORK_TOPDIR'),
'meat',
'%s.Rcheck' % pkg,
'%s-manual.pdf' % pkg)
print('BBS> [stage6b] - copying %s manual to OUTGOING/manuals folder...' % pkg)
if os.path.exists(pdf_file):
dst = os.path.join(manuals_dir, '%s.pdf' % pkg)
#shutil.copy(pdf_file, dst)
os.link(pdf_file, dst) # create hard link to avoid making a copy
else:
print("BBS> [stage6b] SKIPPED (file %s doesn't exist)" % pdf_file)
print('BBS> [stage6b] END copying outgoing packages from %s.' % srcdir)
return
def stage6_make_OUTGOING():
## Create working directory
OUTGOING_dir = os.path.join(BBSvars.Central_rdir.path, 'OUTGOING')
print('BBS> [stage6b] remake_dir %s' % OUTGOING_dir)
bbs.fileutils.remake_dir(OUTGOING_dir)
## Loop over each element of the OUTGOING map
OUTGOING_MAP = BBSutils.getenv('BBS_OUTGOING_MAP')
map_elts = OUTGOING_MAP.split(' ')
for map_elt in map_elts:
tmp = map_elt.split(':')
if len(tmp) != 2:
sys.exit('ERROR: Invalid OUTGOING map element %s' % map_elt)
source_node = False
if tmp[0] == 'source':
source_node = True
OUTGOING_subdir = os.path.join(OUTGOING_dir, tmp[0])
print('BBS> [stage6b] mkdir %s' % OUTGOING_subdir)
os.mkdir(OUTGOING_subdir)
print('BBS> [stage6b] cd %s/' % OUTGOING_subdir)
os.chdir(OUTGOING_subdir)
try:
copy_outgoing_pkgs(tmp[1], source_node)
except FileExistsError as e:
BBSbase.kindly_notify_us('Postrun', e)
raise e
return
##############################################################################
### MAIN SECTION
##############################################################################
print()
print('BBS> ==================================================================')
print('BBS> [stage6b] STARTING stage6b on %s...' % time.asctime())
sys.stdout.flush()
stage6_make_OUTGOING()
print('BBS> [stage6b] DONE on %s.' % time.asctime())