-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmadrigal3_amisr.py
2351 lines (2012 loc) · 104 KB
/
madrigal3_amisr.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""The madrigal_amisr module is used to automate updating Madrigal with
SRI formated hdf5 files. Written by Bill Rideout; specification by
Bill Rideout and Todd Valentic. Meant to be used in realtime by a data
transport application.
This module will automatically create a new Madrigal experiment if needed.
Also allows for switching back to previously existing experiments. The start
method is used either to start a new experiment, or to switch to an old one.
The update method is used to add data.
Modified 2007-10-25 to also add batch updating.
Modified 2008-01-11 to match latest fitter output.
Modified 2008-01-22 to allow for plot directory creation and plots with same name
Modified 2018-04-18 to use Madrigal 3
Modified 2018-05-11 ??? brideout
Modified 2020-07-31 add chi-squared and SNR - brideout
Modified 2021-10-28 remove unused code, refactor, removed extraneous
dependencies - Ashton S. Reimer
Modified 2021-11-03 removed multiprocessing, optimized and simplified codebase
- Ashton S. Reimer
Modified 2021-11-18 added multiprocessing on a per file basis, instead of doing
multiple files in parallel - Ashton S. Reimer
Modified 2022-08-14 Changed experiments for experiments0 as the folder to hold
the data - Pablo M. Reyes
Modified 2023-02-15 change madFilename to include information about the radar
mode, the integration time and the type of process,
e.g. _lp_1min, or _vvels_5min - Pablo M. Reyes
Modified 2023-02-16 bring experimentsDirNum outside uploadExperiment() so that
one can specify the folder experiments0 like the place to save madrigal
files - Pablo M. Reyes
Modified 2023-03-20 Creating definition kindat2fname(kindat) in order to help
with filename formatting. - P. M. Reyes
# e.g. _lp_fit_5min
# e.g. _bc_nenotr_5min
# e.g. _5min-lp_vvels_5min
# e.g. _1min-lp_vvels_5min
Adding file_version to createNewExperimentFromIni and uploadExperiment
in order to add a file_version.
"""
#.......10........20........30........40........50........60........70........80
import os
import sys
import datetime
import traceback
import configparser
import logging
import glob
import shutil
import struct
import distutils.dir_util
import multiprocessing as mp
from multiprocessing import shared_memory
import dill # needed because we can't pickle
import tempfile
import tables
import numpy as np
import madrigal.metadata
import madrigal.cedar
import madrigal.admin
# global flag
WRITEHEADER = 1
POOLSIZE = 8
it2min = {
1:1, 2:2, 3:3, 4:4, 5:5, 6:6, 7:7, 8:8, 9:9, 10:10, 11:11,
12:12, 13:13, 14:14, 15:15, 16:16, 17:17, 18:18, 19:19, 20:20,
25 : 30,
30 : 45,
35 : 60,
40 : 90,
45 : 120,
60 : 45/60,
64 : 30/60,
68 : 15/60,
72 : 10/60,
76 : 5/60,
80 : 4/60,
}
def kindat2fname(kindat):
pc = kindat//10000
pt = (kindat - 10000 * pc) // 100
it = (kindat - 10000 * pc - 100 * pt)
if pc == 100:
pc_desc = "nenotr"
elif pc == 200:
pc_desc = "fit"
elif pc == 300:
pc_desc = "vvels"
else:
raise Exception(f'process type pc={pc} not available. '\
f'kindat = {kindat}. '\
f'valid values: 100,200,300')
if pc in [100,200, 300]:
if pt == 1:
pt_desc = "lp" # long pulse
elif pt == 2:
pt_desc = "ac" # alternating code
elif pt == 3:
pt_desc = "bc" # binary code
else:
raise Exception(f'pulse type pt={pt} not available. '\
f'kindat = {kindat}. '\
f'valid values: 1,2,3')
intg_min = it2min[it]
if intg_min < 1:
it_desc = f"{int(intg_min * 60):02d}sec"
elif intg_min >= 1:
it_desc = f"{intg_min:02d}min"
return f"_{pt_desc}_{pc_desc}_{it_desc}"
# e.g. _lp_fit_05min
# e.g. _bc_nenotr_05min
# e.g. _lp_vvels_05min
# e.g. _lp_vvels_05min
def update_typetab(kindat,ckindat,typetab_file='/opt/madrigal/madrigal3/metadata/typeTab.txt'):
# read the existing kindats from the typeTab
with open(typetab_file,'r') as f:
lines = f.readlines()
existing_kindats = list()
existing_ckindats = list()
for line in lines:
temp_k, temp_ck = line.strip('\n').split(',')
existing_kindats.append(int(temp_k))
existing_ckindats.append(temp_ck)
# check to see if input kindat is in typeTab
if not kindat in existing_kindats:
print("Adding kindat %s to typeTab.txt")
existing_kindats.append(kindat)
existing_ckindats.append(ckindat)
# sort them by kindat
sorted_kindats = sorted(existing_kindats)
sorted_ckindats = [y for _, y in sorted(zip(existing_kindats,
existing_ckindats), key=lambda pair: pair[0])]
with open(typetab_file,'w') as f:
for kindat,ckindat in zip(sorted_kindats,sorted_ckindats):
f.write('%s,%s\n' % (str(kindat),ckindat))
return
def update_cachedfiles(inst,kindat,cachedfiles_file='/opt/madrigal/madrigal3/cachedFiles.ini'):
# if the kindat is for resolved velocities, we need to specify the array splitting
mapping = {300: "{'array':'cgm_lat'}", # latitude binning resolved velocities
}
proc_config = int(kindat) // 10000
# if the kindat isn't a derived data product kindat we're done
if not proc_config in list(mapping.keys()):
return
# check to make sure the kindat is in the cachedFiles.ini
configfile = configparser.ConfigParser(interpolation=None)
configfile.read(cachedfiles_file)
existing_kindats = list(set([x.split('_')[0] for x in configfile.options(str(inst))]))
# if not, add it:
if not kindat in existing_kindats:
configfile.set(str(inst),"%s_params" % kindat,value='')
configfile.set(str(inst),"%s_format" % kindat,value=mapping[proc_config])
with open(cachedfiles_file,'w') as f:
configfile.write(f)
return
def uploadMadrigalFile(madAdminObj,expPath,fullMadFilename,fileDesc,
category,fileAnalyst,fileAnalystEmail):
# addMadrigalFile adds a new file to an experiment using metadata read from madFilename.
# Inputs:
# expDir - full path to experiment directory (as returned by createMadriogalExperiment)
# madFilename - full path to the complete Madrigal file. Basename will be maintained.
# permission - 0 (public) or 1 (private).
# fileDesc - file description
# category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)
# kindat - if not None (the default), use this kindat instead of what is found in the file.
# notify - if True (the default), send a message to all registered users. If False, do not.
# fileAnalyst - full name of file Analyst. Default is ''
# fileAnalystEmail - email of file Analyst. Default is ''
# createCachedText - if True, add cached text file in overview/<basename>.txt.gz. If False,
# no cached file.
# createCachedNetCDF4 - if True, add cached netCDF4 file in overview/<basename>.nc. If False,
# no cached file.
# updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
# convert madFilename to Madrigal with .hdf5 extension before loading.
# acceptOldSummary - if True, accept an old summary file. Used mainly for upgrading to Madrigal 3.
# Default is False.
madAdminObj.addMadrigalFile(expDir = expPath,
madFilename = fullMadFilename,
permission = 0,
fileDesc = fileDesc,
category = category,
kindat = None,
notify = True,
fileAnalyst = fileAnalyst,
fileAnalystEmail = fileAnalystEmail,
createCachedText = True,
createCachedNetCDF4 = True,
updateToMad3 = False,
acceptOldSummary = False)
def uploadMadrigalExp(madAdminObj,fullMadFilename,expTitle,
fileDesc,category,optChar, dirName ,experimentsDirNum,PI,PIEmail,
fileAnalyst,fileAnalystEmail):
"""
uploadMadrigalExp uploads an already created Madrigal file to the
corresponding Madrigal local path.
"""
# createMadrigalExperiment creates a new experiment on Madrigal using metadata read from madFilename.
# Inputs:
# madFilename - full path to the complete Madrigal file. Basename will be maintained.
# expTitle - experiment title
# permission - 0 (public) or 1 (private) or -1 (ignore).
# fileDesc - file description
# instCode - instrument code. If default (None), instrument code is taken from file,
# but error is thrown if more than one kinst found.
# category - 1=default, 2=variant, 3=history, or 4=realtime. Default is 1 (default file)
# optChar - optional character to be added to experiment directory if no dirName
# given. If dirName argument given, this argument ignored. optChar
# is used if the default directory name DDmmmYY is used for
# more than one experiment created for a given instrument on a given day.
# For example, if --optChar=h for a MLH experiment on September 12, 2005,
# then the experiment directory created would be experiments/2005/mlh/12sep05h.
# dirName - directory name to use for experiment. If None (the default), the directory
# name will be the default name DDmmmYY[optChar]. Cannot contain "/"
# kindat - if not None (the default), use this kindat instead of what is found in the file.
# experimentsDirNum - the number to be appended to the experiments directory, if experiments
# directory being used is of the form experiments[0-9]* instead of just
# experiments. For example, if experimentsDirNum is 7, then the experiment
# would be created in MADROOT/experiments7 instead of MADROOT/experiments.
# PI- full name of principal investigator. The default is ''
# PIEmail - email of principal investigator. The default is ''
# fileAnalyst -full name of file analyst. The default is ''
# fileAnalystEmail - email of file analyst,. The default is ''
# createCachedText - if True, add cached text file in overview/<basename>.txt.gz. If False,
# no cached file.
# createCachedNetCDF4 - if True, add cached netCDF4 file in overview/<basename>.nc. If False,
# no cached file.
# notify - if True (the default), send a message to all registered users. If False, do not.
# updateToMad3 - if False (the default), error raised if madFilename non-Hdf5 file. If True, try to
# convert madFilename to Madrigal with .hdf5 extension before loading.
expPath = madAdminObj.createMadrigalExperiment(
madFilename = fullMadFilename,
expTitle = expTitle,
permission = 0,
fileDesc = fileDesc,
instCode = None,
category = category,
optChar = optChar,
dirName = dirName,
kindat = None,
experimentsDirNum = experimentsDirNum,
PI = PI,
PIEmail = PIEmail,
fileAnalyst = fileAnalyst,
fileAnalystEmail = fileAnalystEmail,
createCachedText = True,
createCachedNetCDF4 = True,
notify = True,
updateToMad3 = False)
return expPath
def get_unique_fname(expPath, plotBasenames = []):
"""
Routine to get a unique plotX.html file name.
"""
plotFiles = glob.glob(os.path.join(expPath, 'plot*.html'))
for plotFile in plotFiles:
plotBasenames.append(os.path.basename(plotFile))
plotNum = 0
while True:
plotName = 'plot%.3i.html' % (plotNum)
if plotName not in plotBasenames:
break
plotNum += 1
return plotName, plotBasenames
def createMad3File(args):
"""createMad3File is the method creates a single Madrigal 3 in parallel.
args: hdf5Type, hdf5Filename, instrument, kindat,
fullMadFilename, thisLowerRange, thisUpperRange, writeHeader, iniData,
fileSection, principleInvestigator, expPurpose, expMode,
cycleTime, correlativeExp, sciRemarks
"""
hdf5Type, hdf5Filename, instrument, kindat, fullMadFilename, thisLowerRange, \
thisUpperRange, writeHeader, iniData, fileSection, principleInvestigator, expPurpose, \
expMode, cycleTime, correlativeExp, sciRemarks = args
print("Creating Madrigal hdf5 file...")
with tempfile.NamedTemporaryFile(delete=True,dir='/tmp') as tf:
tempfile_name = tf.name + ".hdf5"
fileHandler = hdf5Handler(hdf5Type)
fileHandler.createMadrigalFile(hdf5Filename,instrument,kindat,None,
tempfile_name,thisLowerRange,thisUpperRange)
# header
if writeHeader:
try: kindatDesc = iniData.get(fileSection, 'extend_ckindat')
except: kindatDesc = None
try: analyst = iniData.get(fileSection, 'analyst')
except: analyst = None
#try: comments = iniData.get(fileSection, 'comments')
try: comments = iniData.get(fileSection, 'fileComment')
except: comments = None
try: history = iniData.get(fileSection, 'history')
except: history = None
if not hdf5Filename is None:
if not comments is None:
comments = comments + '\n' + 'SOURCE_FILE %s' % (
os.path.basename(hdf5Filename))
else:
comments = 'SOURCE_FILE %s' % (
os.path.basename(hdf5Filename))
print("Creating Madrigal Catalog Header...")
now = datetime.datetime.now()
catHeadObj = madrigal.cedar.CatalogHeaderCreator(tempfile_name)
# principleInvestigator - Names of responsible Principal Investigator(s)
# or others knowledgeable about the experiment.
# expPurpose - Brief description of the experiment purpose
# expMode - Further elaboration of meaning of MODEXP; e.g. antenna
# patterns and pulse sequences.
# cycleTime - Minutes for one full measurement cycle
# correlativeExp - Correlative experiments (experiments with related data)
# sciRemarks - scientific remarks
# instRemarks - instrument remarks
catHeadObj.createCatalog(principleInvestigator=principleInvestigator,
expPurpose=expPurpose,
expMode=expMode,
cycleTime=cycleTime,
correlativeExp=correlativeExp,
sciRemarks=sciRemarks)
# kindatDesc - description of how this data was analyzed (the kind of data)
# analyst - name of person who analyzed this data
# comments - additional comments about data (describe any instrument-specific parameters)
# history - a description of the history of the processing of this file
catHeadObj.createHeader(kindatDesc=kindatDesc,
analyst=analyst,
comments=comments,
history=history)
catHeadObj.write()
shutil.copyfile(tempfile_name, fullMadFilename)
os.remove(tempfile_name)
print("This took %s seconds" % (str(datetime.datetime.now()-now)))
def parseExpId(expId):
"""parseExpId parses an experiment id in the form YYYYMMDD.<inst_code>.<number>, and
returns a tuple of (datetime, YYYYMMSS string, instId, optional char associated with number,
and full path to the Madrigal experiment.
Inputs: expId - experiment id string in the form YYYYMMDD.<inst_code>.<number>, where
the date represents the first day of the experiment, <inst_code> is the instrument
code, and the trailing <number> is between 0 and 26
Returns: a tuple with 5 items: 1. datetime represented by YYYYMMDD, 2. YYYYMMSS string
itself, 3) the inst id, 4) the optional char associated with the number (0='', 1='a', ...
26='z'), and 5) the string representing the full path to the Madrigal experiment in form
$MADROOT/experiments/YYYY/<3_letter_inst>/DDmmmYYY<char>.
Raises ValueError if expId not in proper format, instrument code not found,
or trailing number < 0 or > 26.
"""
madDBObj = madrigal.metadata.MadrigalDB()
madInstObj = madrigal.metadata.MadrigalInstrument(madDBObj)
try:
year = int(expId[0:4])
month = int(expId[4:6])
day = int(expId[6:8])
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
if year < 1900:
raise ValueError('expId <%s> has too early year %i' % (str(expId), year))
try:
thisDate = datetime.datetime(year, month, day)
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
try:
items = expId.split('.')
instCode = int(items[1])
num = int(items[2])
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
if num == 0:
extChar = ''
else:
if num <= 26:
extChar = chr(96 + num)
else:
extChar = chr(96 + (num - 1)//26) + chr(97 + (num - 1)%26)
# get 3 letter instrument mnemonic
mnem = madInstObj.getInstrumentMnemonic(instCode)
if mnem == None:
raise ValueError('unknown instrument code in expId: <%i>' % (instCode))
fulldirName = os.path.join(madDBObj.getMadroot(),'experiments0','%04i' % year,mnem,'%s%s' % (thisDate.strftime('%d%b%y').lower(), extChar))
return((thisDate, items[0], instCode, extChar, fulldirName))
class BatchExperiment:
"""BatchExperiment is a class to create and update AMISR Madrigal experiments
"""
# defines length of line in Cedar catalog/header file
__CEDAR_LEN__ = 80
def uploadExperiment(self,iniFile,plotsdir='plots',file_version=1,
removeTmpFiles=True, experimentsDirNum=None):
"""
file_version : the 3 digit version attached to each file
removeTmpFiles : When True, remove tmp file after uploading to Madrigal
"""
# create needed Madrigal objects
self.madDBObj = madrigal.metadata.MadrigalDB()
madExpObj = madrigal.metadata.MadrigalExperiment(self.madDBObj)
self.madInstObj = madrigal.metadata.MadrigalInstrument(self.madDBObj)
# read ini file
self.__iniData__ = configparser.ConfigParser()
self.__iniData__.read(iniFile)
# DEFAULT
ExperimentName = self.__iniData__.get('DEFAULT','ExperimentName')
# get reqiured experiment info
expTitle = self.__iniData__.get('Experiment', 'title')
self.instrument = int(self.__iniData__.get('Experiment', 'instrument'))
logFile = self.__iniData__.get('Experiment', 'logFile')
expId = self.__iniData__.get('Experiment', 'expId')
OutPath = self.__iniData__.get('Experiment','OutPath')
# PI, Analyst
PI = self.__iniData__.get('Experiment', 'pi')
PIEmail = self.__iniData__.get('Experiment', 'PIEmail')
fileAnalyst = self.__iniData__.get('Experiment', 'fileAnalyst')
fileAnalystEmail = self.__iniData__.get('Experiment', 'fileAnalystEmail')
# parse the expId
try:
items = expId.split('.')
date = int(items[0])
num = int(items[1])
expId = items[0] + '.' + str(self.instrument) + '.' + items[1]
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
# find the number of files being created
numFiles = 0
while True:
try:
self.__iniData__.get('File%i' % (numFiles + 1), 'hdf5Filename')
numFiles += 1
except configparser.NoSectionError:
break
# get optional character, if any
optChar = "" #parseExpId(expId)[3]
dirName = os.path.basename(parseExpId(expId)[4])
# create new madrigal file name template
instMnemonic = self.madInstObj.getInstrumentMnemonic(self.instrument).lower()
#madFilenameTemplate = '%s%02i%02i%02i.' % (instMnemonic, firstTime.year % 100,
# firstTime.month, firstTime.day)
#madFilenameTemplate = '%s%04i%02i%02i' % (instMnemonic, firstTime.year,
# firstTime.month, firstTime.day)
madFilenameTemplate = '%s%s' % (instMnemonic, ExperimentName)
madAdminObj = madrigal.admin.MadrigalDBAdmin()
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
kindat = int(self.__iniData__.get(self.fileSection, 'kindat'))
if False:
# previous version control
if type(file_version) == int:
ver2use = file_version
elif type(file_version) == type(None):
# here should be the last version created
allversions = sorted(glob.glob(os.path.join(OutPath,
madFilenameTemplate + kindat2fname(kindat) + ".???.h5")))
if len(allversions)>0:
ver2use = int(allversions[-1].split('.')[-2]) + 1
else:
ver2use = 1
else:
raise Exception("file_version needs to be int or None.")
madFilename = madFilenameTemplate + kindat2fname(kindat)\
+ f'.{ver2use:03d}.h5'
else:
fileOutVerTag = self.__iniData__.get(self.fileSection, 'fileOutVerTag')
madFilename = madFilenameTemplate + kindat2fname(kindat)\
+ f'.{fileOutVerTag}.h5'
fullMadFilename = os.path.join(OutPath,madFilename)
print(f"working on file: {fullMadFilename}")
if not os.path.exists(fullMadFilename):
raise Exception(f"file {fullMadFilename} does not exist.")
hdf5Type = self.__iniData__.get(self.fileSection, 'type')
status = self.__iniData__.get(self.fileSection, 'status')
fileDesc = status
fileComment = self.__iniData__.get(self.fileSection, 'fileComment')
history = self.__iniData__.get(self.fileSection, 'history')
category = int(self.__iniData__.get(self.fileSection, 'category'))
shutil.copyfile(fullMadFilename, os.path.join('/tmp',madFilename))
tmpfullMadFilename=os.path.join('/tmp',madFilename)
if fileNum==0:# create the experiment
try:
expPath = uploadMadrigalExp(madAdminObj,tmpfullMadFilename,
expTitle,fileDesc,category,optChar,dirName,experimentsDirNum,
PI,PIEmail,fileAnalyst,fileAnalystEmail)
except IOError:
x,y,z = sys.exc_info()
print(y)
expPath = str(y).split()[1]
info = input('Okay to Remove ' + expPath + '? type Yes: ')
if info=='Yes':
distutils.dir_util.remove_tree(expPath+'/',verbose=1)
expPath = uploadMadrigalExp(madAdminObj,tmpfullMadFilename,
expTitle,fileDesc,category,optChar,dirName,experimentsDirNum,
PI,PIEmail,fileAnalyst,fileAnalystEmail)
else:
raise IOError(y)
else:
uploadMadrigalFile(madAdminObj,expPath,tmpfullMadFilename,fileDesc,
category,fileAnalyst,fileAnalystEmail)
# see if links to images are desired
numLinks = 0
image_dict = dict()
while True:
try:
imageTitle = self.__iniData__.get(self.fileSection,
'imageTitle%i' % (numLinks + 1))
image = self.__iniData__.get(self.fileSection,
'image%i' % (numLinks + 1))
#if 'Geometry Plot' in imageTitle:
if imageTitle not in image_dict.keys():
image_dict.update({imageTitle:[]})
image_dict[imageTitle].append(image)
logging.info(f'Including image {image}')
numLinks += 1
except (configparser.NoSectionError, configparser.NoOptionError):
break
if len(image_dict.keys())>0:
self.createPlotLinks(madFilename,image_dict, expPath)
if removeTmpFiles:
logging.info(f'{tmpfullMadFilename} has been uploaded. Removing tmp file ...')
os.remove(tmpfullMadFilename)
self.expPath = expPath
# create link to ISR AMISR database
self.createLinkBack2amisr(self.instrument,ExperimentName,expPath)
#return(expPath)
def createNewExperimentFromIni(self, iniFile, skip_existing=False,
skip_doc_plots=False, file_version = 1):
"""createNewExperimentFromIni will try to create a single new Madrigal experiment using
information parsed from an input ini file.
This method will also allow importing summary plots created outside this script into Madrigal.
It will also allow the option of the automatic creation of individual record plots.
file_version : the 3 digit version attached to each file
Example ini file:
[Experiment]
title: World Day
instrument: 61
logFile: $BASE/amisr/pokerflat/face1/20071011-120000/log.txt
expId: 20071011.61.000
pi: Craig Heiselman
modexp: This is a one line experiment title
cmodexp: In this section you can write a multi-line description
of your experiment. An ini file recognizes multiline
descriptions as long as every continuation line begins
with whitespace (as in this example).
[File1]
hdf5Filename: $BASE/amisr/pokerflat/face1/20071011-120000/20071011.004_lp_3min.h5
kindat: 5950
type: standard
createRecPlots: True
imageTitle1: Electron density - long pulse - beam 1
image1: /tmp/elecDensBeam1_lp.jpg
imageTitle2: Electron density - long pulse - beam 2
image2: /tmp/elecDensBeam2_lp.jpg
ckindat: In this section you can write a multi-line description
of how this particular file was created. An ini file
recognizes multiline descriptions as long as every continuation line begins
with whitespace (as in this example)
[File2]
hdf5Filename: $BASE/amisr/pokerflat/face1/20071011-120000/20071011.004_ac_3min.h5
kindat: 5951
type: standard
createRecPlots: False
imageTitle1: Electron density - alternating code - beam 1
image1: /tmp/elecDensBeam1_ac.jpg
imageTitle2: Electron density - alternating code - beam 2
image2: /tmp/elecDensBeam2_ac.jpg
[File3]
hdf5Filename: $BASE/amisr/pokerflat/face1/20071011-120000/20071011.004_lp_3min-vvels.h5
kindat: 5953
type: velocity
The input ini file is made up of an [Experiment] section and one or more [File*] section.
No time values are used here. since the experiment times will be determined from the data itself.
The names title, instrument, logFile, and expId are all required in the Experiment section.
The expId name must be in the form YYYYMMDD.<inst_code>.<number>, where
the date represents the first day of the experiment, <inst_code> is the instrument
code, and the trailing <number> is between 0 and 26.
In addition to the required fields in the Experiment section, there are also some optional
fields designed to add experiment level information to the files' catalog record:
pi - the experiment principle investigator
modexp - a short experiment title
cmodexp - a full description of the experiment. These fields describe the experiment
as a whole, and so will be the same for each file. The field cmodexp will
typically be multiple lines. It is legal to have multiline values in an ini
file as long as each new line begins with some white space.
For each file to be added to the experiment, a section called File* is required, and the
numbering must be in increasing order starting with 1. The names hdf5Filename, kindat, and type
are required. It is highly recommended that every file in an experiment have a unique
kindat, because the kindat description is how the user determines the differences between
files. Madrigal administrators can always add additional kindats by editing the
$MADROOT/metadata.typeTab.txt file (kindat descriptions must not contain commas).
type deterimines the type of hdfs being loaded. Presently supported types are standard,
velocity, and uncorrected_ne_only.
In addition to the required fields in the File* section, there are also some optional fields:
createRecPlots -If set to True, the createRecPlots name will allow the creation of
individual record plots. If not given or False, these plots will not be created.
If type != standard, createRecPlots is ignored, since only works with standard data.
imageTitle%i and image%i - must be given as a pair with matching numbers. Allows images
relating to that experiment to be imported into Madrigal. The user will see a link to
the imported image with text set by imageTitle.
ckindat - a description of how this particular file was processed. Will be included in the
header record prepended to the file. The field ckindat will typically be multiple lines.
It is legal to have multiline values in an ini file as long as each new line begins
with some white space.
lowerRange - sets a lower range cutoff in km for uncorrected_ne_only files
upperRange - sets a upper range cutoff in km for uncorrected_ne_only files
"""
# create needed Madrigal objects
self.madDBObj = madrigal.metadata.MadrigalDB()
madExpObj = madrigal.metadata.MadrigalExperiment(self.madDBObj)
self.madInstObj = madrigal.metadata.MadrigalInstrument(self.madDBObj)
# read ini file
self.__iniData__ = configparser.ConfigParser()
self.__iniData__.read(iniFile)
# DEFAULT
ExperimentName = self.__iniData__.get('DEFAULT','ExperimentName')
# get reqiured experiment info
expTitle = self.__iniData__.get('Experiment', 'title')
self.instrument = int(self.__iniData__.get('Experiment', 'instrument'))
logFile = self.__iniData__.get('Experiment', 'logFile')
expId = self.__iniData__.get('Experiment', 'expId')
OutPath = self.__iniData__.get('Experiment','OutPath')
# PI, Analyst
PI = self.__iniData__.get('Experiment', 'pi')
PIEmail = self.__iniData__.get('Experiment', 'PIEmail')
fileAnalyst = self.__iniData__.get('Experiment', 'fileAnalyst')
fileAnalystEmail = self.__iniData__.get('Experiment', 'fileAnalystEmail')
# parse the expId
try:
items = expId.split('.')
date = int(items[0])
num = int(items[1])
expId = items[0] + '.' + str(self.instrument) + '.' + items[1]
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
filename=logFile,
filemode='w')
logging.info('Creating exp using ini file %s with instrument %i and title <%s> and expId <%s>' % \
(iniFile, self.instrument, expTitle, expId))
# find the number of files being created
numFiles = 0
while True:
try:
self.__iniData__.get('File%i' % (numFiles + 1), 'hdf5Filename')
numFiles += 1
except configparser.NoSectionError:
break
if numFiles == 0:
raise IOError('No File* section specified in ini file')
# next find the time range in the data
# firstTime = None
# lastTime = None
# for fileNum in range(numFiles):
# self.fileSection = 'File%i' % (fileNum + 1)
# hdf5Filename = self.__iniData__.get(self.fileSection, 'hdf5Filename')
# hdf5Type = self.__iniData__.get(self.fileSection, 'type')
# fileHandler = hdf5Handler(hdf5Type)
# startTime, endTime = fileHandler.getStartEndTimes(hdf5Filename)
# print(startTime, endTime, hdf5Filename)
# if firstTime == None:
# firstTime = startTime
# elif firstTime > startTime:
# firstTime = startTime
# if lastTime == None:
# lastTime = endTime
# elif lastTime < endTime:
# lastTime = endTime
# create new madrigal file name template
instMnemonic = self.madInstObj.getInstrumentMnemonic(self.instrument).lower()
#madFilenameTemplate = '%s%02i%02i%02i.' % (instMnemonic, firstTime.year % 100,
# firstTime.month, firstTime.day)
#madFilenameTemplate = '%s%04i%02i%02i' % (instMnemonic, firstTime.year,
# firstTime.month, firstTime.day)
madFilenameTemplate = '%s%s' % (instMnemonic, ExperimentName)
# header
try: principleInvestigator = self.__iniData__.get('Experiment', 'pi')
except: principleInvestigator = None
try: expPurpose = self.__iniData__.get('Experiment', 'modexp')
except: expPurpose = None
try: expMode = self.__iniData__.get('Experiment', 'cmodexp')
except: expMode = None
try: cycleTime = self.__iniData__.get('Experiment', 'cycletime')
except: cycleTime = None
try: correlativeExp = self.__iniData__.get('Experiment', 'correxp')
except: correlativeExp = None
try: sciRemarks = self.__iniData__.get('Experiment', 'remarks')
except: sciRemarks = None
args = [] # each will be a tuple of (hdf5Type,hdf5Filename, self.instrument, kindat,
# fullMadFilename, thisLowerRange, thisUpperRange, writeHeader, iniData,
# fileSection, principleInvestigator, expPurpose, expMode,
# cycleTime, correlativeExp, sciRemarks)
# loop through all the files, and add to madrigal
plots_links_dict = {}
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
kindat = int(self.__iniData__.get(self.fileSection, 'kindat'))
if False:
if type(file_version) == int:
madFilename = madFilenameTemplate + kindat2fname(kindat)\
+ f'.{file_version:03d}.h5'
fullMadFilename = os.path.join(OutPath,madFilename)
elif type(file_version) == type(None):
for fcount in range(1,1000):
madFilename = madFilenameTemplate + kindat2fname(kindat)\
+ f'.{fcount:03d}.h5'
fullMadFilename = os.path.join(OutPath,madFilename)
if not os.path.exists(fullMadFilename):
break
else:
raise Exception("file_version needs to be int or None.")
else:
fileOutVerTag = self.__iniData__.get(self.fileSection, 'fileOutVerTag')
madFilename = madFilenameTemplate + kindat2fname(kindat)\
+ f'.{fileOutVerTag}.h5'
fullMadFilename = os.path.join(OutPath,madFilename)
print(f"working on file: {fullMadFilename}")
hdf5Filename = self.__iniData__.get(self.fileSection, 'hdf5Filename')
hdf5Type = self.__iniData__.get(self.fileSection, 'type')
ckindat = self.__iniData__.get(self.fileSection, 'ckindat')
# update the madrigal/metadata/typeTab.txt and madrigal/cachedFiles.ini files
update_typetab(kindat, ckindat)
update_cachedfiles(self.instrument,kindat)
try:
thisLowerRange = float(self.__iniData__.get(
self.fileSection, 'lowerRange'))
except:
thisLowerRange = None
try:
thisUpperRange = float(self.__iniData__.get(
self.fileSection, 'upperRange'))
except:
thisUpperRange = None
args.append((hdf5Type, hdf5Filename, self.instrument, kindat,
fullMadFilename, thisLowerRange, thisUpperRange, WRITEHEADER,
self.__iniData__, self.fileSection, principleInvestigator,
expPurpose, expMode, cycleTime, correlativeExp, sciRemarks))
#logging.info(f'adding file from {str(firstTime)} to {str(lastTime)}'
logging.info(f'adding experiment {ExperimentName} '
f' using {hdf5Filename}, kindat {kindat:d}, type {hdf5Type}'
f', lowerRange={str(thisLowerRange)}'
f', upperRange={str(thisUpperRange)}')
# link plots associated with the file
numLinks = 0
image_dict = dict()
while True:
try:
imageTitle = self.__iniData__.get(self.fileSection,
'imageTitle%i' % (numLinks + 1))
image = self.__iniData__.get(self.fileSection,
'image%i' % (numLinks + 1))
if imageTitle not in image_dict.keys():
image_dict.update({imageTitle:[]})
image_dict[imageTitle].append(image)
numLinks += 1
except (configparser.NoSectionError, configparser.NoOptionError):
break
if len(image_dict.keys())>0:
plots_links_dict.update({madFilename:[image_dict, OutPath]})
# all file info has been gathered - kick off multiprocessing
print('Processing %i files' % (len(args)))
for i,arg in enumerate(args):
print("...working on file %d: %s" % (i+1,arg[1]))
if skip_existing and os.path.exists(arg[4]):
print('Skipping file %s - already exists\n' % arg[4])
continue
else:
createMad3File(arg)
if not skip_doc_plots:
print('Creating plot links...')
for madfname, [image_dict, OutPath] in plots_links_dict.items():
self.createPlotLinks(madfname,image_dict, OutPath)
self.createLinkBack2amisr(self.instrument,ExperimentName,OutPath)
# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
# def createNewExperimentFromIni
def createLinkBack2amisr(self,instrument,Experiment,expPath):
"""
Create a link to the amisr.com page of the current Experiment
Inputs:
instrument - int, instrument code e.g. 61 for PFISR
Experiment - YYYYMMDD.XXX unique AMISR experiment name
expPath - path to experiment directory
"""
# get a unique filename
plotName, plotBasenames = get_unique_fname(expPath)
baseurl = "https://data.amisr.com/database/"
amisr_isr_database = f"{baseurl}{instrument}/experiment/{Experiment}/3/"
output_file = os.path.join(expPath, plotName)
logging.info(f"Saving file {output_file}")
with open(output_file, 'w') as f:
f.write(f"""<html><head>
<TITLE>AMISR ISR Database</TITLE>
<meta http-equiv="refresh" content="0; url='{amisr_isr_database}'" />
</head>
<body>
<p>You will be redirected to data.amisr.com soon!</p>
</body>
</html>
""")
def createPlotLinks(self,hdf5Filename,image_dict, expPath):
"""createPlotLinks is a method to create an html file associated
with each hdf5 file and links to groups of plots.
Inputs:
hdf5Filename - the file that the plots will be associated with
image_dict - a dictionary with plot types as keys and plot images
as items
expPath - path to experiment directory
"""
# get a unique filename
plotName, plotBasenames = get_unique_fname(expPath)
if not os.path.exists(os.path.join(expPath, 'plots')):
os.mkdir(os.path.join(expPath, 'plots'))
f_links_html = f"""<html><head>
<TITLE>Plots associated with : {hdf5Filename}</TITLE>
"""
f_links_html += """
<style>
div.gallery {
margin: 2px;
border: 1px solid #ccc;
width: 400px;
float: left;
}
div.gallery:hover {
border: 1px solid #777;
}
div.gallery img {
width: 45%;
height: auto;
}
div.desc {
width: 55%;
float: right;
font-size : 14px;
text-align: center;
}
div.rowDiv {
overflow: hidden; /* add this to contain floated children */
}
h4 {
margin-top: 2px ;
margin-bottom: 0 ;
}
</style>
</head> <body>
"""
f_links_html += f'<h4>Plots associated with : {hdf5Filename}</h4>\n'
for imageTitle,imagefiles in image_dict.items():
if "Geometry Plot" in imageTitle:
assert len(image_dict) >1, "Need other plots, not only Geometry."
assert len(imagefiles) == 1, f"So far, {imageTitle} expects 1 fig"
logging.info("Creating link for imageTitle.")
self.createLink(hdf5Filename,imageTitle,imagefiles[0],expPath,
plotBasenames = [plotName])
continue
f_links_html += f' <div class="rowDiv">\n'
f_links_html += f' <h4>{imageTitle.split(" ",1)[1]}</h4>\n'
for imageFile in imagefiles:
imgFileBase = os.path.basename(imageFile)
# change base with madrigal fname
uniquename = imgFileBase.split(imageTitle.split(" ",1)[0],1)[-1]