-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathmadrigal3_amisr.py.orig
1897 lines (1592 loc) · 85.1 KB
/
madrigal3_amisr.py.orig
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
"""The madrigal_amisr module is used to automate updating Madrigal with
SRI formated hdf5 files. Written by Bill Rideout; specification by
Bill Rideout and Todd Valentic. Meant to be used in realtime by a data
transport application.
This module will automatically create a new Madrigal experiment if needed.
Also allows for switching back to previously existing experiments. The start
method is used either to start a new experiment, or to switch to an old one.
The update method is used to add data.
Modified 2007-10-25 to also add batch updating.
Modified 2008-01-11 to match latest fitter output.
Modified 2008-01-22 to allow for plot directory creation and plots with same name
Modified 2018-04-18 to use Madrigal 3
Modified 2018-05-11 ??? brideout
Modified 2020-07-31 add chi-squared and SNR - brideout
Modified 2021-10-28 remove unused code, refactor, removed extraneous dependencies - Ashton S. Reimer
Modified 2021-11-03 removed multiprocessing, optimized and simplified codebase - Ashton S. Reimer
Modified 2021-11-18 added multiprocessing on a per file basis, instead of doing multiple files in parallel - Ashton S. Reimer
Modified 2022-08-14 Changed experiments for experiments0 as the folder to hold the data - Pablo M. Reyes
"""
import os
import sys
import datetime
import traceback
import configparser
import logging
import glob
import shutil
import struct
import distutils.dir_util
import multiprocessing as mp
from multiprocessing import shared_memory
import dill # needed because we can't pickle
import tempfile
import tables
import numpy as np
import madrigal.metadata
import madrigal.cedar
import madrigal.admin
# global flag
WRITEHEADER = 1
POOLSIZE = 8
def update_typetab(kindat,ckindat,typetab_file='/opt/madrigal/madrigal3/metadata/typeTab.txt'):
# read the existing kindats from the typeTab
with open(typetab_file,'r') as f:
lines = f.readlines()
existing_kindats = list()
existing_ckindats = list()
for line in lines:
temp_k, temp_ck = line.strip('\n').split(',')
existing_kindats.append(int(temp_k))
existing_ckindats.append(temp_ck)
# check to see if input kindat is in typeTab
if not kindat in existing_kindats:
print("Adding kindat %s to typeTab.txt")
existing_kindats.append(kindat)
existing_ckindats.append(ckindat)
# sort them by kindat
sorted_kindats = sorted(existing_kindats)
sorted_ckindats = [y for _, y in sorted(zip(existing_kindats, existing_ckindats), key=lambda pair: pair[0])]
with open(typetab_file,'w') as f:
for kindat,ckindat in zip(sorted_kindats,sorted_ckindats):
f.write('%s,%s\n' % (str(kindat),ckindat))
return
def update_cachedfiles(inst,kindat,cachedfiles_file='/opt/madrigal/madrigal3/cachedFiles.ini'):
# if the kindat is for resolved velocities, we need to specify the array splitting
mapping = {300: "{'array':'cgm_lat'}", # latitude binning resolved velocities
}
proc_config = int(kindat) // 10000
# if the kindat isn't a derived data product kindat we're done
if not proc_config in list(mapping.keys()):
return
# check to make sure the kindat is in the cachedFiles.ini
configfile = configparser.ConfigParser(interpolation=None)
configfile.read(cachedfiles_file)
existing_kindats = list(set([x.split('_')[0] for x in configfile.options(str(inst))]))
# if not, add it:
if not kindat in existing_kindats:
configfile.set(str(inst),"%s_params" % kindat,value='')
configfile.set(str(inst),"%s_format" % kindat,value=mapping[proc_config])
with open(cachedfiles_file,'w') as f:
configfile.write(f)
return
def createMad3File(args):
"""createMad3File is the method creates a single Madrigal 3 in parallel.
args: hdf5Type, hdf5Filename, instrument, kindat,
fullMadFilename, thisLowerRange, thisUpperRange, writeHeader, iniData,
fileSection, principleInvestigator, expPurpose, expMode,
cycleTime, correlativeExp, sciRemarks
"""
hdf5Type, hdf5Filename, instrument, kindat, fullMadFilename, thisLowerRange, \
thisUpperRange, writeHeader, iniData, fileSection, principleInvestigator, expPurpose, \
expMode, cycleTime, correlativeExp, sciRemarks = args
print("Creating Madrigal hdf5 file...")
with tempfile.NamedTemporaryFile(delete=True,dir='/tmp') as tf:
tempfile_name = tf.name + ".hdf5"
fileHandler = hdf5Handler(hdf5Type)
fileHandler.createMadrigalFile(hdf5Filename,instrument,kindat,None,tempfile_name,thisLowerRange,thisUpperRange)
# header
if writeHeader:
try: kindatDesc = iniData.get(fileSection, 'ckindat')
except: kindatDesc = None
try: analyst = iniData.get(fileSection, 'analyst')
except: analyst = None
try: comments = iniData.get(fileSection, 'comments')
except: comments = None
try: history = iniData.get(fileSection, 'history')
except: history = None
print("Creating Madrigal Catalog Header...")
now = datetime.datetime.now()
catHeadObj = madrigal.cedar.CatalogHeaderCreator(tempfile_name)
catHeadObj.createCatalog(principleInvestigator=principleInvestigator, expPurpose=expPurpose, expMode=expMode, cycleTime=cycleTime, correlativeExp=correlativeExp, sciRemarks=sciRemarks)
catHeadObj.createHeader(kindatDesc=kindatDesc, analyst=analyst, comments=comments, history=history)
catHeadObj.write()
shutil.copyfile(tempfile_name, fullMadFilename)
os.remove(tempfile_name)
print("This took %s seconds" % (str(datetime.datetime.now()-now)))
def parseExpId(expId):
"""parseExpId parses an experiment id in the form YYYYMMDD.<inst_code>.<number>, and
returns a tuple of (datetime, YYYYMMSS string, instId, optional char associated with number,
and full path to the Madrigal experiment.
Inputs: expId - experiment id string in the form YYYYMMDD.<inst_code>.<number>, where
the date represents the first day of the experiment, <inst_code> is the instrument
code, and the trailing <number> is between 0 and 26
Returns: a tuple with 5 items: 1. datetime represented by YYYYMMDD, 2. YYYYMMSS string
itself, 3) the inst id, 4) the optional char associated with the number (0='', 1='a', ...
26='z'), and 5) the string representing the full path to the Madrigal experiment in form
$MADROOT/experiments/YYYY/<3_letter_inst>/DDmmmYYY<char>.
Raises ValueError if expId not in proper format, instrument code not found,
or trailing number < 0 or > 26.
"""
madDBObj = madrigal.metadata.MadrigalDB()
madInstObj = madrigal.metadata.MadrigalInstrument(madDBObj)
try:
year = int(expId[0:4])
month = int(expId[4:6])
day = int(expId[6:8])
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
if year < 1900:
raise ValueError('expId <%s> has too early year %i' % (str(expId), year))
try:
thisDate = datetime.datetime(year, month, day)
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
try:
items = expId.split('.')
instCode = int(items[1])
num = int(items[2])
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
if num == 0:
extChar = ''
else:
if num < 26:
extChar = chr(96 + num)
else:
extChar = chr(96 + int(num/26)) + chr(96 + num%26)
# get 3 letter instrument mnemonic
mnem = madInstObj.getInstrumentMnemonic(instCode)
if mnem == None:
raise ValueError('unknown instrument code in expId: <%i>' % (instCode))
dirName = os.path.join(madDBObj.getMadroot(),'experiments0','%04i' % year,mnem,'%s%s' % (thisDate.strftime('%d%b%y').lower(), extChar))
return((thisDate, items[0], instCode, extChar, dirName))
class BatchExperiment:
"""BatchExperiment is a class to create and update AMISR Madrigal experiments
"""
# defines length of line in Cedar catalog/header file
__CEDAR_LEN__ = 80
def uploadExperiment(self,iniFile,plotsdir='plots'):
# create needed Madrigal objects
self.madDBObj = madrigal.metadata.MadrigalDB()
madExpObj = madrigal.metadata.MadrigalExperiment(self.madDBObj)
self.madInstObj = madrigal.metadata.MadrigalInstrument(self.madDBObj)
# read ini file
self.__iniData__ = configparser.ConfigParser()
self.__iniData__.read(iniFile)
# get reqiured experiment info
expTitle = self.__iniData__.get('Experiment', 'title')
self.instrument = int(self.__iniData__.get('Experiment', 'instrument'))
logFile = self.__iniData__.get('Experiment', 'logFile')
expId = self.__iniData__.get('Experiment', 'expId')
OutPath = self.__iniData__.get('Experiment','OutPath')
# parse the expId
try:
items = expId.split('.')
date = int(items[0])
num = int(items[1])
expId = items[0] + '.' + str(self.instrument) + '.' + items[1]
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
# find the number of files being created
numFiles = 0
while True:
try:
self.__iniData__.get('File%i' % (numFiles + 1), 'hdf5Filename')
numFiles += 1
except configparser.NoSectionError:
break
# get optional character, if any
optChar = parseExpId(expId)[3]
# next find the time range in the data
firstTime = None
lastTime = None
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
hdf5Filename = self.__iniData__.get(self.fileSection, 'hdf5Filename')
hdf5Type = self.__iniData__.get(self.fileSection, 'type')
fileHandler = hdf5Handler(hdf5Type)
startTime, endTime = fileHandler.getStartEndTimes(hdf5Filename)
print(startTime, endTime, hdf5Filename)
if firstTime == None:
firstTime = startTime
elif firstTime > startTime:
firstTime = startTime
if lastTime == None:
lastTime = endTime
elif lastTime < endTime:
lastTime = endTime
# create new madrigal file name template
instMnemonic = self.madInstObj.getInstrumentMnemonic(self.instrument).lower()
madFilenameTemplate = '%s%02i%02i%02i.' % (instMnemonic, firstTime.year % 100,
firstTime.month, firstTime.day)
madAdminObj = madrigal.admin.MadrigalDBAdmin()
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
madFilename = madFilenameTemplate + '%03i' % (fileNum + 1) + '.hdf5'
fullMadFilename = os.path.join(OutPath,madFilename)
hdf5Filename = self.__iniData__.get(self.fileSection, 'hdf5Filename')
hdf5Type = self.__iniData__.get(self.fileSection, 'type')
status = self.__iniData__.get(self.fileSection, 'status')
category = int(self.__iniData__.get(self.fileSection, 'category'))
fileDesc=status
shutil.copyfile(fullMadFilename, os.path.join('/tmp',madFilename))
fullMadFilename=os.path.join('/tmp',madFilename)
if fileNum==0:# create the experiment
try:
expPath = madAdminObj.createMadrigalExperiment(fullMadFilename, expTitle, 0, fileDesc,None,category=category,optChar=optChar,
createCachedText=True, createCachedNetCDF4=True)
except IOError:
x,y,z = sys.exc_info()
print(y)
expPath = str(y).split()[1]
info = input('Okay to Remove ' + expPath + '? type Yes: ')
if info=='Yes':
distutils.dir_util.remove_tree(expPath+'/',verbose=1)
expPath = madAdminObj.createMadrigalExperiment(fullMadFilename, expTitle, 0, fileDesc,None,category=category,optChar=optChar,
createCachedText=True, createCachedNetCDF4=True)
else:
raise IOError(y)
else:
madAdminObj.addMadrigalFile(expPath,fullMadFilename,0, fileDesc,category=category,kindat=None,
createCachedText=True, createCachedNetCDF4=True)
# see if links to images are desired
numLinks = 0
while True:
try:
imageTitle = self.__iniData__.get(self.fileSection, 'imageTitle%i' % (numLinks + 1))
image = self.__iniData__.get(self.fileSection, 'image%i' % (numLinks + 1))
self.createLink(imageTitle, image, expPath)
logging.info('Created link with file %s' % (image))
numLinks += 1
except (configparser.NoSectionError, configparser.NoOptionError):
break
self.expPath = expPath
os.remove(fullMadFilename)
return(expPath)
def createNewExperimentFromIni(self, iniFile):
"""createNewExperimentFromIni will try to create a single new Madrigal experiment using
information parsed from an input ini file.
This method will also allow importing summary plots created outside this script into Madrigal.
It will also allow the option of the automatic creation of individual record plots.
Example ini file:
[Experiment]
title: World Day
instrument: 61
logFile: $BASE/amisr/pokerflat/face1/20071011-120000/log.txt
expId: 20071011.61.000
pi: Craig Heiselman
modexp: This is a one line experiment title
cmodexp: In this section you can write a multi-line description
of your experiment. An ini file recognizes multiline
descriptions as long as every continuation line begins
with whitespace (as in this example).
[File1]
hdf5Filename: $BASE/amisr/pokerflat/face1/20071011-120000/20071011.004_lp_3min.h5
kindat: 5950
type: standard
createRecPlots: True
imageTitle1: Electron density - long pulse - beam 1
image1: /tmp/elecDensBeam1_lp.jpg
imageTitle2: Electron density - long pulse - beam 2
image2: /tmp/elecDensBeam2_lp.jpg
ckindat: In this section you can write a multi-line description
of how this particular file was created. An ini file
recognizes multiline descriptions as long as every continuation line begins
with whitespace (as in this example)
[File2]
hdf5Filename: $BASE/amisr/pokerflat/face1/20071011-120000/20071011.004_ac_3min.h5
kindat: 5951
type: standard
createRecPlots: False
imageTitle1: Electron density - alternating code - beam 1
image1: /tmp/elecDensBeam1_ac.jpg
imageTitle2: Electron density - alternating code - beam 2
image2: /tmp/elecDensBeam2_ac.jpg
[File3]
hdf5Filename: $BASE/amisr/pokerflat/face1/20071011-120000/20071011.004_lp_3min-vvels.h5
kindat: 5953
type: velocity
The input ini file is made up of an [Experiment] section and one or more [File*] section.
No time values are used here. since the experiment times will be determined from the data itself.
The names title, instrument, logFile, and expId are all required in the Experiment section.
The expId name must be in the form YYYYMMDD.<inst_code>.<number>, where
the date represents the first day of the experiment, <inst_code> is the instrument
code, and the trailing <number> is between 0 and 26.
In addition to the required fields in the Experiment section, there are also some optional
fields designed to add experiment level information to the files' catalog record:
pi - the experiment principle investigator
modexp - a short experiment title
cmodexp - a full description of the experiment. These fields describe the experiment
as a whole, and so will be the same for each file. The field cmodexp will
typically be multiple lines. It is legal to have multiline values in an ini
file as long as each new line begins with some white space.
For each file to be added to the experiment, a section called File* is required, and the
numbering must be in increasing order starting with 1. The names hdf5Filename, kindat, and type
are required. It is highly recommended that every file in an experiment have a unique
kindat, because the kindat description is how the user determines the differences between
files. Madrigal administrators can always add additional kindats by editing the
$MADROOT/metadata.typeTab.txt file (kindat descriptions must not contain commas).
type deterimines the type of hdfs being loaded. Presently supported types are standard,
velocity, and uncorrected_ne_only.
In addition to the required fields in the File* section, there are also some optional fields:
createRecPlots -If set to True, the createRecPlots name will allow the creation of
individual record plots. If not given or False, these plots will not be created.
If type != standard, createRecPlots is ignored, since only works with standard data.
imageTitle%i and image%i - must be given as a pair with matching numbers. Allows images
relating to that experiment to be imported into Madrigal. The user will see a link to
the imported image with text set by imageTitle.
ckindat - a description of how this particular file was processed. Will be included in the
header record prepended to the file. The field ckindat will typically be multiple lines.
It is legal to have multiline values in an ini file as long as each new line begins
with some white space.
lowerRange - sets a lower range cutoff in km for uncorrected_ne_only files
upperRange - sets a upper range cutoff in km for uncorrected_ne_only files
"""
# create needed Madrigal objects
self.madDBObj = madrigal.metadata.MadrigalDB()
madExpObj = madrigal.metadata.MadrigalExperiment(self.madDBObj)
self.madInstObj = madrigal.metadata.MadrigalInstrument(self.madDBObj)
# read ini file
self.__iniData__ = configparser.ConfigParser()
self.__iniData__.read(iniFile)
# get reqiured experiment info
expTitle = self.__iniData__.get('Experiment', 'title')
self.instrument = int(self.__iniData__.get('Experiment', 'instrument'))
logFile = self.__iniData__.get('Experiment', 'logFile')
expId = self.__iniData__.get('Experiment', 'expId')
OutPath = self.__iniData__.get('Experiment','OutPath')
# parse the expId
try:
items = expId.split('.')
date = int(items[0])
num = int(items[1])
expId = items[0] + '.' + str(self.instrument) + '.' + items[1]
except:
traceback.print_exc()
raise ValueError('expId not in form YYYYMMDD.<inst_code>.<number>: <%s>' % (str(expId)))
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
filename=logFile,
filemode='w')
logging.info('Creating exp using ini file %s with instrument %i and title <%s> and expId <%s>' % \
(iniFile, self.instrument, expTitle, expId))
# find the number of files being created
numFiles = 0
while True:
try:
self.__iniData__.get('File%i' % (numFiles + 1), 'hdf5Filename')
numFiles += 1
except configparser.NoSectionError:
break
if numFiles == 0:
raise IOError('No File* section specified in ini file')
# next find the time range in the data
firstTime = None
lastTime = None
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
hdf5Filename = self.__iniData__.get(self.fileSection, 'hdf5Filename')
hdf5Type = self.__iniData__.get(self.fileSection, 'type')
fileHandler = hdf5Handler(hdf5Type)
startTime, endTime = fileHandler.getStartEndTimes(hdf5Filename)
print(startTime, endTime, hdf5Filename)
if firstTime == None:
firstTime = startTime
elif firstTime > startTime:
firstTime = startTime
if lastTime == None:
lastTime = endTime
elif lastTime < endTime:
lastTime = endTime
# create new madrigal file name template
instMnemonic = self.madInstObj.getInstrumentMnemonic(self.instrument).lower()
madFilenameTemplate = '%s%02i%02i%02i.' % (instMnemonic, firstTime.year % 100,
firstTime.month, firstTime.day)
# header
try: principleInvestigator = self.__iniData__.get('Experiment', 'pi')
except: principleInvestigator = None
try: expPurpose = self.__iniData__.get('Experiment', 'modexp')
except: expPurpose = None
try: expMode = self.__iniData__.get('Experiment', 'cmodexp')
except: expMode = None
try: cycleTime = self.__iniData__.get('Experiment', 'cycletime')
except: cycleTime = None
try: correlativeExp = self.__iniData__.get('Experiment', 'correxp')
except: correlativeExp = None
try: sciRemarks = self.__iniData__.get('Experiment', 'remarks')
except: sciRemarks = None
args = [] # each will be a tuple of (hdf5Type,hdf5Filename, self.instrument, kindat,
# fullMadFilename, thisLowerRange, thisUpperRange, writeHeader, iniData,
# fileSection, principleInvestigator, expPurpose, expMode,
# cycleTime, correlativeExp, sciRemarks)
# loop through all the files, and add to madrigal
acceptedFileNums = [] # used in link list
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
madFilename = madFilenameTemplate + '%03i' % (fileNum + 1) + '.hdf5'
fullMadFilename = os.path.join(OutPath,madFilename)
if os.path.exists(fullMadFilename):
print('Skipping file %s - already exists\n' % madFilename)
continue
acceptedFileNums.append(fileNum)
hdf5Filename = self.__iniData__.get(self.fileSection, 'hdf5Filename')
kindat = int(self.__iniData__.get(self.fileSection, 'kindat'))
hdf5Type = self.__iniData__.get(self.fileSection, 'type')
ckindat = self.__iniData__.get(self.fileSection, 'ckindat')
# update the madrigal/metadata/typeTab.txt and madrigal/cachedFiles.ini files
update_typetab(kindat,ckindat)
update_cachedfiles(self.instrument,kindat)
try:
thisLowerRange = float(self.__iniData__.get(self.fileSection, 'lowerRange'))
except:
thisLowerRange = None
try:
thisUpperRange = float(self.__iniData__.get(self.fileSection, 'upperRange'))
except:
thisUpperRange = None
args.append((hdf5Type, hdf5Filename, self.instrument, kindat, fullMadFilename,
thisLowerRange, thisUpperRange, WRITEHEADER, self.__iniData__,
self.fileSection, principleInvestigator, expPurpose, expMode,
cycleTime, correlativeExp, sciRemarks))
logging.info('adding file from %s to %s using %s, kindat %i, type %s, lowerRange=%s, upperRange=%s' % \
(str(firstTime), str(lastTime), hdf5Filename, kindat, hdf5Type, str(thisLowerRange), str(thisUpperRange)))
# all file info has been gathered - kick off multiprocessing
print('Processing %i files' % (len(args)))
for i,arg in enumerate(args):
print("...working on file %d: %s" % (i+1,arg[1]))
createMad3File(arg)
# separate loop for links when complete
for fileNum in range(numFiles):
self.fileSection = 'File%i' % (fileNum + 1)
# see if links to images are desired
numLinks = 0
while True:
try:
imageTitle = self.__iniData__.get(self.fileSection, 'imageTitle%i' % (numLinks + 1))
image = self.__iniData__.get(self.fileSection, 'image%i' % (numLinks + 1))
self.createLink(imageTitle, image, OutPath)
logging.info('Created link with file %s' % (image))
numLinks += 1
except (configparser.NoSectionError, configparser.NoOptionError):
break
def createLink(self,title,imageFile,expPath):
"""createLink is a method to create a new html file in expPath with a link to an image file.
Inputs:
title - title of plot (will show up in Madrigal)
imageFile - external image to be copied and displayed in Madrigal
expPath - path to experiment directory
"""
templateHtml = """<html><head>
<TITLE>%s</TITLE>
</head> <body><img src="plots/%s"></body></html>
"""
# get a unique filename
plotBasenames = []
plotFiles = glob.glob(os.path.join(expPath, 'plot*.html'))
for plotFile in plotFiles:
plotBasenames.append(os.path.basename(plotFile))
plotNum = 0
while True:
plotName = 'plot%i.html' % (plotNum)
if plotName not in plotBasenames:
break
plotNum += 1
if not os.path.exists(os.path.join(expPath, 'plots')):
os.mkdir(os.path.join(expPath, 'plots'))
imageFileBasename = os.path.basename(imageFile)
outName = os.path.join(expPath, 'plots',imageFileBasename)
if os.path.exists(outName):
imageFileTrailing = imageFileBasename.rsplit('.',1)
imgNum=1
while True:
outName = '%s-%i.%s' % (imageFileTrailing[0],imgNum,imageFileTrailing[1])
if not os.path.exists(os.path.join(expPath, 'plots',outName)):
break
imgNum+=1
outName = os.path.join(expPath,'plots',outName)
shutil.copyfile(imageFile, outName)
try:
os.chmod(outName,0o664)
except:
pass
with open(os.path.join(expPath, plotName), 'w') as f:
f.write(templateHtml % (title, os.path.basename(outName)))
class analyzeHdf5:
"""analyzeHdf5 is a class to analyze a SRI-formated hdf5 file containing standard ISR parameters
"""
def __init__(self,hdf5File):
"""__init__ gets summary information about hdf5File containing uncorrected electron density.
"""
self.__startTime = None # will be set to the earliest datetime
self.__endTime = None # will be set to the latest datetime
# read in all required data
with tables.open_file(hdf5File) as hdfObj:
start_time = hdfObj.root.Time.UnixTime[0,0]
end_time = hdfObj.root.Time.UnixTime[-1,1]
self.__startTime = datetime.datetime.utcfromtimestamp(start_time)
self.__endTime = datetime.datetime.utcfromtimestamp(end_time)
if self.__startTime > self.__endTime:
raise Exception("Problem with start and end times: %s, %s, %s" % (self.__startTime,self.__endTime,hdf5File))
def getStartEndTimes(self):
return (self.__startTime, self.__endTime)
class hdf5ToMadrigal:
"""hdf5ToMadrigal is a class to turn a standard SRI-formated hdf5 file into a Madrigal file
"""
def __init__(self,hdf5File,kinst,kindat,cedarObj,madrigalFile):
"""__init__ will write or update a Madrigal file using data in hdf5File
Inputs:
hdf5File - full path to hdf5 file with ISR data in SRI format
kinst - instrument code (integer)
kindat - data file kindat (integer)
cedarObj - existing madrigal.cedar.MadrigalCedarFile to append data to.
If None, new madrigal.cedar.MadrigalCedarFile
created using madrigalFile.
madrigalFile - name of Madrigal file to create or append to.
Sets attributes self.numRecs, self.numTimes, self.numBeams
"""
print("doing the fitted data stuff")
now = datetime.datetime.now()
# hard-coded indices defined by the format of the hdf file
o_index = 0
e_index = -1
fractIndex = 0
tempIndex = 1
colIndex = 2
velIndex = 3
# parameter boundaries
minTemp = 100.0 # MJN changed 06/05/2008
maxTemp = 10000.0
maxTempErr = 32765.0 # limit on Cedar format
minTempErr = 1.0 # limit on Cedar format
minNe = 1.0E9
maxNe = 1.0E13
maxNeErr = 3.2E13
minNeErr = 1.0
maxVo = 32765.0 # limit on Cedar format
maxVoErr = 32765.0 # limit on Cedar format
minVoErr = 0.01
maxFract = 1.0
minFract = 0.0
# read in all required data
with tables.open_file(hdf5File) as hdfObj:
# beam codes
beamCodes = hdfObj.root.BeamCodes
beamCodeArray = beamCodes.read()
self.numBeams = beamCodeArray.shape[0]
# geomag
plat = hdfObj.root.Geomag.MagneticLatitude
platArray = plat.read()
plong = hdfObj.root.Geomag.MagneticLongitude
plongArray = plong.read()
# ranges
ranges = hdfObj.root.FittedParams.Range
rangeArray = ranges.read()
numRanges = rangeArray.shape[1]
RangeTime=0
if rangeArray.ndim==3:
RangeTime=1
# electron density (ne)
ne = hdfObj.root.FittedParams.Ne
neArray = ne.read()
self.numTimes = neArray.shape[0]
# error in electron density
dne = hdfObj.root.FittedParams.dNe
dneArray = dne.read()
# ion info
fits = hdfObj.root.FittedParams.Fits
fitsArray = fits.read()
# ion error info
errors = hdfObj.root.FittedParams.Errors
errArray = errors.read()
# time info
days = hdfObj.root.Time.Day
dayArray = days.read()
months = hdfObj.root.Time.Month
monthArray = months.read()
years = hdfObj.root.Time.Year
yearArray = years.read()
dtimes = hdfObj.root.Time.dtime
dtimeArray = dtimes.read()
# number of tx, rx
numTxAeu = hdfObj.root.ProcessingParams.AeuTx
numTxAeuArray = numTxAeu.read()
numRxAeu = hdfObj.root.ProcessingParams.AeuRx
numRxAeuArray = numRxAeu.read()
# power info
txPower = hdfObj.root.ProcessingParams.TxPower
txPowerArray = txPower.read()
# baud length
baudLength = hdfObj.root.ProcessingParams.BaudLength.read()
# pulse length
pulseLength = hdfObj.root.ProcessingParams.PulseLength.read()
baudCount = int(pulseLength/baudLength)
if baudCount <= 0:
baudCount = 1
# tx freq
txFreq = hdfObj.root.ProcessingParams.TxFrequency.read()
# rx freq
rxFreq = hdfObj.root.ProcessingParams.RxFrequency.read()
# chisq
chisq = hdfObj.root.FittedParams.FitInfo.chi2
chisqArray = chisq.read()
# snr
snr = hdfObj.root.NeFromPower.SNR
snrArray = snr.read()
# create cedarObj if needed
if cedarObj == None:
cedarObj = madrigal.cedar.MadrigalCedarFile(madrigalFile,True,arraySplitParms=['beamid'])
# create all data records
# loop first through num records, then through num beams
print("Building list of data...")
inputs = list()
for recIndex in range(self.numTimes):
# get start and end times for this record
startYear = int(yearArray[recIndex][0])
endYear = int(yearArray[recIndex][1])
startMonth = int(monthArray[recIndex][0])
endMonth = int(monthArray[recIndex][1])
startDay = int(dayArray[recIndex][0])
endDay = int(dayArray[recIndex][1])
startDtime = dtimeArray[recIndex][0]
endDtime = dtimeArray[recIndex][1]
startHour = int(startDtime)
endHour = int(endDtime)
startMin = int(startDtime*60.0 - startHour*60.0)
endMin = int(endDtime*60.0 - endHour*60.0)
startSec = int(startDtime*3600.0) % 60
endSec = int(endDtime*3600.0) % 60
startTime = datetime.datetime(startYear, startMonth, startDay, startHour, startMin, startSec)
endTime = datetime.datetime(endYear, endMonth, endDay, endHour, endMin, endSec)
for beamIndex in range(self.numBeams):
beamId = beamCodeArray[beamIndex][0]
az = beamCodeArray[beamIndex][1]
el = beamCodeArray[beamIndex][2]
txpower = txPowerArray[recIndex]/1000.0
numtxaeu = numTxAeuArray[recIndex]
numrxaeu = numRxAeuArray[recIndex]
if RangeTime:
rangeValue = rangeArray[recIndex][0]
else:
rangeValue = rangeArray[0]
neValue = neArray[recIndex][beamIndex]
dneValue = dneArray[recIndex][beamIndex]
fitsValue = fitsArray[recIndex][beamIndex]
errValue = errArray[recIndex][beamIndex]
chisqValue = chisqArray[recIndex][beamIndex]
snrValue = snrArray[recIndex][beamIndex]
platValue = platArray[beamIndex]
plongValue = plongArray[beamIndex]
limits = (minTemp,maxTemp,maxTempErr,minTempErr,minNe,maxNe,maxNeErr,minNeErr,
maxVo,maxVoErr,minVoErr,maxFract,minFract)
indices = (o_index,e_index,fractIndex,tempIndex,colIndex,velIndex)
inputs.append((kinst,kindat,startTime,endTime,az,el,beamId,txpower,numtxaeu,
numrxaeu,baudCount,pulseLength,txFreq,rxFreq,rangeValue,neValue,
dneValue,fitsValue,errValue,chisqValue,snrValue,platValue,
plongValue,limits,indices)
)
print("There are %d jobs to execute..." % (len(inputs)))
# do one to get the size
dataRec = set_fitted_data_rec(inputs[0])
encoded = dill.dumps(dataRec)
datarecsize = len(encoded)
dtype = '|S%d' % (datarecsize)
output = np.zeros((len(inputs),), dtype=dtype)
shm = shared_memory.SharedMemory(create=True, size=output.nbytes)
shm_name = shm.name
output_array = np.ndarray((len(inputs),),dtype=dtype,buffer=shm.buf)
worker_pool = mp.Pool(POOLSIZE)
jobs = [worker_pool.apply_async(mp_wrapper,args=(set_fitted_data_rec,i,args,shm_name,len(inputs),dtype)) for i,args in enumerate(inputs)]
results = [p.get() for p in jobs]
for i in range(len(inputs)):
if i % 100 == 0:
print("Dumping data: %d/%d" % (i + 1,len(inputs)))
data_record = dill.loads(output_array[i])
cedarObj.append(data_record)
# dump records every 100
if i % 50 == 0:
cedarObj.dump()
# dump remaining records
cedarObj.dump()
shm.close()
shm.unlink()
print("This took %s seconds" % (str(datetime.datetime.now()-now)))
print("Closing out the file...")
now = datetime.datetime.now()
cedarObj.close()
self.numRecs = self.numTimes * self.numBeams
print("This took %s seconds" % (str(datetime.datetime.now()-now)))
class hdf5VelocityToMadrigal:
"""hdf5VelocityToMadrigal is a class to turn a SRI-formated hdf5 file with vector velocities
into a Madrigal file
"""
def __init__(self,hdf5File,kinst,kindat,cedarObj,madrigalFile):
"""__init__ will write or update a Madrigal file using data in hdf5File containing vector velocities
Inputs:
hdf5File - full path to hdf5 file with vector velocity data in SRI format
kinst - instrument code (integer)
kindat - data file kindat (integer)
cedarObj - existing madrigal.cedar.MadrigalCedarFile to append data to.
If None, new madrigal.cedar.MadrigalCedarFile
created using madrigalFile.
madrigalFile - name of Madrigal file to create or append to.
Sets attribute self.numRecs
"""
print("doing the fitted data stuff")
now = datetime.datetime.now()
# hard-coded indices defined by the format of the hdf file
north_index = 0
east_index = 1
parallel_index = 2
maxV = 32765.0 # limit on Cedar format
maxVErr = 32765.0 # limit on Cedar format
minVErr = 1.0
maxE = 32765.0*1e-5 # limit on Cedar format
maxEErr = 32765.0*1e-5 # limit on Cedar format
minEErr = 0.01*1e-5
maxDirErr = 360.0
# read in all required data
with tables.open_file(hdf5File) as hdfObj:
# time info
unixTimes = hdfObj.root.Time.UnixTime
timeArray = unixTimes.read()
self.numRecs = timeArray.shape[0]
# altitude info
minAlt = hdfObj.root.ProcessingParams.MinAlt.read()
maxAlt = hdfObj.root.ProcessingParams.MaxAlt.read()
# nmeas - number of measurments
nmeas = hdfObj.root.VectorVels.Nmeas
nmeasArray = nmeas.read()
params = ['nsmpta']
try:
byGeo = hdfObj.root.ProcessingParams.GeographicBinning.read()
except:
byGeo = 0
# integration time
integrationSecs = hdfObj.root.ProcessingParams.IntegrationTime.read()
# magnetic latitude or latitude bins
if byGeo!=0:
if byGeo==2:
platArray = hdfObj.root.VectorVels.Latitude.read()
params.extend(['gdlat'])
else:
platArray = hdfObj.root.VectorVels.MagneticLatitude.read()
params.extend(['cgm_lat'])
params.extend(['vi2','dvi2','vi1', 'dvi1', 'vi3', 'dvi3'])
else:
try:
platArray = hdfObj.root.VectorVels.Plat.read()
except:
platArray = hdfObj.root.VectorVels.MagneticLatitude.read()
params.extend(['cgm_lat','vipn','dvipn','vipe', 'dvipe', 'vi6', 'dvi6'])
numPlat = platArray.shape[0]
# velocity vectors and errors
try:
vestArray = hdfObj.root.VectorVels.Vest.read()
dvestArray = hdfObj.root.VectorVels.errVest.read()
except:
vestArray = None
dvestArray = None
xxxxxx
dvestArray=np.maximum(dvestArray,minVErr)
# velocity magnitude, dir and errors
try:
vmagArray = hdfObj.root.VectorVels.Vmag.read()
dvmagArray = hdfObj.root.VectorVels.errVmag.read()
vdirArray = hdfObj.root.VectorVels.Vdir.read()
dvdirArray = hdfObj.root.VectorVels.errVdir.read()
if byGeo:
params.extend(['magvel','dmagvel','gnangle','dgnangle'])
else: