Merge pull request #10 from PyCOMPLETE/LS2reorganization

LS2 Reorganization

000_get_data_for_hl_recalc.py

@@ -0,0 +1,125 @@
+from __future__ import division
+import os
+import cPickle as pickle
+import argparse
+import time
+
+import LHCMeasurementTools.lhc_log_db_query as lldb
+from LHCMeasurementTools.SetOfHomogeneousVariables import SetOfHomogeneousNumericVariables
+import LHCMeasurementTools.myfilemanager as mfm
+
+from GasFlowHLCalculator.h5_storage import H5_storage
+
+import GasFlowHLCalculator
+
+h5_storage = H5_storage(h5_dir = '/eos/user/l/lhcecld/heatload_data_storage')
+
+# Config
+dt_seconds = 60
+max_fill_hrs = 35
+blacklist = []
+blacklist.append(4948) # 116 hour long fill, exceeds memory
+blacklist.append(5488) # 40 hour long fill, also exceeds memory
+
+parser = argparse.ArgumentParser()
+parser.add_argument('-r', help='reversed', action='store_true')
+parser.add_argument('--year', choices=[2012, 2015, 2016, 2017, 2018, 2019], type=int, default=2019)
+
+args = parser.parse_args()
+year = args.year
+
+# File names
+h5_dir_0 = h5_storage.data_dir
+
+# [For all cells, for 3 special cells]
+gfolder = '/'.join(GasFlowHLCalculator.__file__.split('/')[:-1])
+variable_files = [gfolder+'/variable_list_complete.txt',
+	gfolder+'/variable_list_special.txt']
+h5_dirs = [h5_dir_0, h5_dir_0 + 'special_cells/']
+file_names = ['cryo_data_fill', 'special_data_fill']
+temp_filepaths = ['./tmp/' + f for f in file_names]
+temp_files = [t + '_%i.csv' for t in temp_filepaths]
+data_file_funcs = [h5_storage.get_data_file, h5_storage.get_special_data_file]
+
+if year == 2012:
+    fills_pkl_name = '/afs/cern.ch/project/spsecloud/LHC_2012_selected_periods/fills_and_bmodes.pkl'
+elif year == 2015:
+    fills_pkl_name = '/afs/cern.ch/project/spsecloud/LHC_2015_PhysicsAfterTS2/fills_and_bmodes.pkl'
+elif year == 2016:
+    fills_pkl_name = '/afs/cern.ch/project/spsecloud/LHC_2016_25ns/LHC_2016_25ns_beforeTS1/fills_and_bmodes.pkl'
+elif year == 2017:
+    fills_pkl_name = '/afs/cern.ch/project/spsecloud/LHC_2017_operation/LHC_2017_operation/fills_and_bmodes.pkl'
+elif year == 2018:
+    fills_pkl_name = '/afs/cern.ch/work/l/lhcscrub/LHC_2018_followup/fills_and_bmodes.pkl'
+elif year == 2019:
+    fills_pkl_name = '/afs/cern.ch/work/l/lhcecld/LHC_followup_download_scripts/fills_and_bmodes.pkl'
+else:
+    raise ValueError('Invalid year')
+
+with open(fills_pkl_name, 'rb') as fid:
+    dict_fill_bmodes = pickle.load(fid)
+
+for variable_file, h5_dir, file_name, temp_filepath, temp_file, data_file_func in \
+        zip(variable_files, h5_dirs, file_names, temp_filepaths, temp_files, data_file_funcs):
+
+    fill_sublist = sorted(dict_fill_bmodes.keys(), reverse=args.r)
+    fill_sublist_2 = []
+    data_files = os.listdir(os.path.dirname(data_file_func(0)))
+    for filln in fill_sublist:
+        if os.path.basename(data_file_func(filln)) in data_files:
+            pass
+        elif filln in blacklist:
+            print('Fill %i is blacklisted' % filln)
+        elif not dict_fill_bmodes[filln]['flag_complete']:
+            print('Fill %i is not completed' % filln)
+        else:
+            t_start_fill = dict_fill_bmodes[filln]['t_startfill']
+            t_end_fill   = dict_fill_bmodes[filln]['t_endfill']
+            fill_hrs = (t_end_fill - t_start_fill)/3600.
+            if fill_hrs < max_fill_hrs:
+                fill_sublist_2.append(filln)
+            else:
+                print('Fill %i exceeds %i hours and is skipped' % (filln, max_fill_hrs))
+    print('Processing %i fills!' % len(fill_sublist_2))
+    time.sleep(5)
+
+    with open(variable_file, 'r') as f:
+        varlist = f.read().splitlines()[0].split(',')
+    print '%i Timber variables' % len(varlist)
+    for ii, var in enumerate(varlist):
+        if '.POSST' not in var:
+            raise ValueError('%s does not have a .POSST' % var)
+
+    if not os.path.isdir(h5_dir):
+        os.mkdir(h5_dir)
+
+    for filln in fill_sublist_2:
+        h5_file = data_file_func(filln)
+        if h5_file in os.listdir(os.path.dirname(h5_file)):
+            continue
+        this_temp_file = temp_file % filln
+        print('Downloading csv for fill %i' % filln)
+        t_start_fill = dict_fill_bmodes[filln]['t_startfill']
+        t_end_fill   = dict_fill_bmodes[filln]['t_endfill']
+        lldb.dbquery(varlist, t_start_fill, t_end_fill, this_temp_file)
+        print('Aligning data for fill %i' % filln)
+        htd_ob = SetOfHomogeneousNumericVariables(varlist, this_temp_file).aligned_object(dt_seconds)
+        print('Creating h5 file for fill %i' % filln)
+        n_tries_max = 5
+        for n_try in xrange(n_tries_max):
+            try:
+                mfm.aligned_obj_to_h5(htd_ob, h5_file)
+                break
+            except Exception as e:
+                print('Saving of h5 failed')
+                time.sleep(10)
+        else:
+            print('Raise error after trying to save the h5 file %i times' % n_tries_max)
+            raise
+
+        if os.path.isfile(h5_file) and os.path.getsize(h5_file) > 500:
+            os.remove(this_temp_file)
+            print('Deleted temporary file %s!' % (this_temp_file))
+        else:
+            print('Warning! Something went wrong for file %s!\nKeeping temporary file %s.' % (h5_file % filln, temp_file % filln))
+

001a_store_recalculated.py

@@ -5,11 +5,17 @@
 import argparse
 import random
 
-import compute_QBS_LHC as cql
-import h5_storage
-from h5_storage import data_dir
+from GasFlowHLCalculator.h5_storage import H5_storage
+import GasFlowHLCalculator.recalc_multiple_circuits as rmc
+
+from GasFlowHLCalculator.calibration_config import calibration_config
+from GasFlowHLCalculator.calibration import Calibration, CalibrationManager
+
+h5_storage = H5_storage(h5_dir = '/eos/user/l/lhcecld/heatload_data_storage/')
+cal_manager = CalibrationManager(calibration_config=calibration_config)
+
+data_dir = h5_storage.data_dir
 
-new_version_default = 7
 use_dPs = (True,False)
 
 parser = argparse.ArgumentParser()
@@ -28,37 +34,37 @@
     if info is not None:
         filln = int(info.group(1))
 
-	if args.filln:
-	    if not int(filln)==int(args.filln):
-		#print 'Skipped fill', filln
-		continue  
+    if args.filln:
+        if not int(filln)==int(args.filln):
+            #print 'Skipped fill', filln
+            continue
+
+    for use_dP in use_dPs:
+        kwargs = {'use_dP': use_dP}
+        this_qbs_file = h5_storage.get_qbs_file(filln, **kwargs)
+        if not os.path.isfile(this_qbs_file):
+
+            print('\nCalculation for fill %i (usedP: %s) started...' % (filln, use_dP))
+            time_0 = time.mktime(time.localtime())
+            atd_ob = h5_storage.load_data_file(filln)
+
+            calibration = cal_manager.get_calibration(atd_ob.timestamps[0])
+
+            qbs_ob, other = rmc.recalc_multiple_circuits(atd_ob,
+                calibration, circuit_selection='full_lhc',
+                with_P_drop=use_dP)
 
-        for use_dP in use_dPs:
-            if filln < 3600:
-                new_version = -1
+            n_tries = 5
+            while n_tries > 0:
+                try:
+                    h5_storage.store_qbs(filln, qbs_ob, **kwargs)
+                    break
+                except IOError:
+                    n_tries -= 1
+                    time.sleep(60)
             else:
-                new_version = new_version_default
-            kwargs = {'use_dP': use_dP, 'version': new_version}
-            this_qbs_file = h5_storage.get_qbs_file(filln, **kwargs)
-            if not os.path.isfile(this_qbs_file):
-                if new_version != new_version_default:
-                    print 'Warning in GasflowHLCalculator.qbs_fill: special case for pre LS1 fills. Specified version is ignored.'
-
-                time_0 = time.time()
-                atd_ob = h5_storage.load_data_file(filln)
-                qbs_ob = cql.compute_qbs(atd_ob, **kwargs)
-
-                n_tries = 5
-                while n_tries > 0:
-                    try:
-                        h5_storage.store_qbs(filln, qbs_ob, **kwargs)
-                        break
-                    except IOError:
-                        n_tries -= 1
-                        time.sleep(60)
-                else:
-                    raise IOError('Saving failed for fill %i!' % filln)
-                dt = time.time() - time_0
-                n_timesteps = len(qbs_ob.timestamps)
-                print('Calculation for fill %i (usedP: %s) with %i timesteps finished in %i s.' % (filln, use_dP, n_timesteps, dt))
+                raise IOError('Saving failed for fill %i!' % filln)
+            dt = time.mktime(time.localtime()) - time_0
+            n_timesteps = len(qbs_ob.timestamps)
+            print('Calculation for fill %i (usedP: %s) with %i timesteps finished in %i s.' % (filln, use_dP, n_timesteps, dt))
 

001b_special_store_recalculated.py

@@ -5,9 +5,16 @@
 import argparse
 import random
 
-import compute_QBS_special as cqs
-import h5_storage
-from h5_storage import special_data_dir, special_version
+from GasFlowHLCalculator.h5_storage import H5_storage
+import GasFlowHLCalculator.recalc_multiple_circuits as rmc
+
+from GasFlowHLCalculator.calibration_config import calibration_config
+from GasFlowHLCalculator.calibration import Calibration, CalibrationManager
+
+h5_storage = H5_storage(h5_dir = '/eos/user/l/lhcecld/heatload_data_storage/')
+cal_manager = CalibrationManager(calibration_config=calibration_config)
+
+special_data_dir = h5_storage.special_data_dir
 
 parser = argparse.ArgumentParser()
 parser.add_argument('-r', help='random', action='store_true')
@@ -35,21 +42,26 @@
 
         this_qbs_file = h5_storage.get_special_qbs_file(filln)
         if not os.path.isfile(this_qbs_file):
-            time_0 = time.time()
+            time_0 = time.mktime(time.localtime())
             atd_ob = h5_storage.load_special_data_file(filln)
-            new_cell = filln > 5600
-            qbs_ob = cqs.compute_qbs_special(atd_ob, new_cell=new_cell, separate=True, aligned=True)
+
+            calibration = cal_manager.get_calibration(atd_ob.timestamps[0])
+
+            qbs_ob, other = rmc.recalc_multiple_circuits(atd_ob,
+                calibration, circuit_selection='all_instrumented',
+                with_P_drop=True)
+
             n_tries = 5
             while n_tries > 0:
                 try:
-                    h5_storage.store_special_qbs(filln, qbs_ob, special_version)
+                    h5_storage.store_special_qbs(filln, qbs_ob)
                     break
                 except IOError:
                     n_tries -= 1
                     time.sleep(5)
             else:
                 raise IOError('Saving failed for fill %i!' % filln)
-            dt = time.time() - time_0
+            dt = time.mktime(time.localtime()) - time_0
             n_timesteps = len(qbs_ob.timestamps)
             print('Calculation for fill %i with %i timesteps finished in %i s.' % (filln, n_timesteps, dt))
 

002_recalc_and_save_all_circuits_one_fill.py

@@ -0,0 +1,37 @@
+import numpy as np
+
+from calibration_config import calibration_config
+from calibration import Calibration, CalibrationManager
+from h5_storage import H5_storage
+import recalc_multiple_circuits as rmc
+
+filln = 6737
+with_P_drop = True
+
+circuit_selection = 'full_lhc'
+circuit_selection = 'all_instrumented'
+
+cal_manager = CalibrationManager(calibration_config=calibration_config)
+h5_storage = H5_storage(h5_dir='/eos/user/l/lhcecld/heatload_data_storage')
+
+if circuit_selection == 'full_lhc':
+    obraw = h5_storage.load_data_file(filln=filln)
+elif circuit_selection == 'all_instrumented':
+    obraw = h5_storage.load_special_data_file(filln=filln)
+else:
+    raise(ValueError('Invalid circuit selection!'))
+
+calibration = cal_manager.get_calibration(obraw.timestamps[0])
+
+obhl_store, other = rmc.recalc_multiple_circuits(obraw,
+        calibration, circuit_selection=circuit_selection,
+        with_P_drop=with_P_drop)
+
+h5_storage.store_qbs(filln=filln, qbs_ob=obhl_store, use_dP=with_P_drop)
+
+if circuit_selection == 'full_lhc':
+    h5_storage.store_qbs(filln=filln, qbs_ob=obhl_store, use_dP=with_P_drop)
+elif circuit_selection == 'all_instrumented':
+    h5_storage.store_special_qbs(filln=filln, qbs_ob=obhl_store)
+else:
+    raise(ValueError('Invalid circuit selection!'))