From f6fbc92c6ca244a8cd7ce18663945faa533ae066 Mon Sep 17 00:00:00 2001 From: Xiong-Hui Chen Date: Mon, 25 Apr 2022 12:35:22 +0800 Subject: [PATCH] feat: add a view script --- RLA/easy_log/log_tools.py | 122 +++++++++++++++++++++++++------------- rla_scripts/view_expt.py | 19 ++++++ test/test_scripts.py | 9 ++- 3 files changed, 107 insertions(+), 43 deletions(-) diff --git a/RLA/easy_log/log_tools.py b/RLA/easy_log/log_tools.py index 9db3d7c..59598a8 100644 --- a/RLA/easy_log/log_tools.py +++ b/RLA/easy_log/log_tools.py @@ -28,39 +28,13 @@ def __init__(self, optional_log_type=None): if optional_log_type is not None: self.log_types.extend(optional_log_type) -class DownloadLogTool(BasicLogTool): - def __init__(self, rla_config_path, proj_root, task, regex, *args, **kwargs): - fs = open(rla_config_path, encoding="UTF-8") - self.private_config = yaml.load(fs) - self.proj_root = proj_root - self.task_table_name = task - self.regex = regex - - def _download_log(self, show=False): - for log_type in self.log_types: - root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex) - empty = True - for root_dir in glob.glob(root_dir_regex): - - pass - -class DeleteLogTool(BasicLogTool): - def __init__(self, proj_root, task_table_name, regex, filter, *args, **kwargs): - self.proj_root = proj_root - self.task_table_name = task_table_name - self.regex = regex - assert isinstance(filter, Filter) - self.filter = filter - self.small_timestep_regs = [] - super(DeleteLogTool, self).__init__(*args, **kwargs) - - def _find_small_timestep_log(self): - root_dir_regex = osp.join(self.proj_root, LOG, self.task_table_name, self.regex) + def _find_small_timestep_log(self, proj_root, task_table_name, regex, timstep_upper_bound=np.inf, timestep_lower_bound=0): + small_timestep_regs = [] + root_dir_regex = osp.join(proj_root, LOG, task_table_name, regex) for root_dir in glob.glob(root_dir_regex): print("searching dirs", root_dir) if os.path.exists(root_dir): for file_list in os.walk(root_dir): - if re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]): target_reg = re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]).group(0) else: @@ -77,7 +51,8 @@ def _find_small_timestep_log(self): if file_list[1] == ['tb'] or os.path.exists(progress_csv_file): # in root of logdir if not os.path.exists(progress_csv_file) or os.path.getsize(progress_csv_file) == 0: print("[delete] find an experiment without progress.csv.", file_list[0]) - self.small_timestep_regs.append(target_reg) + if timestep_lower_bound <= 0: + small_timestep_regs.append([target_reg, file_list[0]]) else: try: reader = pd.read_csv(progress_csv_file, chunksize=100000, quoting=csv.QUOTE_NONE, @@ -88,9 +63,9 @@ def _find_small_timestep_log(self): raw_df = pd.concat([raw_df, slim_chunk], ignore_index=True) last_timestep = raw_df[DEFAULT_X_NAME].max() print("[found a log] time_step ", last_timestep, target_reg) - if last_timestep < self.filter.timstep_bound: - self.small_timestep_regs.append(target_reg) - print("[delete] find an experiment with too small number of logs. ", file_list[0]) + if timestep_lower_bound <= last_timestep <= timstep_upper_bound: + small_timestep_regs.append([target_reg, file_list[0]]) + print("[delete] find an experiment satisfied timestep range. ", file_list[0]) else: print("[valid]") except Exception as e: @@ -101,8 +76,35 @@ def _find_small_timestep_log(self): elif 'events' in file_list[0]: # in event dir pass else: # empty dir - self.small_timestep_regs.append(target_reg) + if timestep_lower_bound <= 0: + small_timestep_regs.append([target_reg, file_list[0]]) print("[delete] find an experiment without any files. ", file_list[0]) + return small_timestep_regs + +class DownloadLogTool(BasicLogTool): + def __init__(self, rla_config_path, proj_root, task, regex, *args, **kwargs): + fs = open(rla_config_path, encoding="UTF-8") + self.private_config = yaml.load(fs) + self.proj_root = proj_root + self.task_table_name = task + self.regex = regex + + def _download_log(self, show=False): + for log_type in self.log_types: + root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex) + empty = True + for root_dir in glob.glob(root_dir_regex): + pass + +class DeleteLogTool(BasicLogTool): + def __init__(self, proj_root, task_table_name, regex, filter, *args, **kwargs): + self.proj_root = proj_root + self.task_table_name = task_table_name + self.regex = regex + assert isinstance(filter, Filter) + self.filter = filter + self.small_timestep_regs = [] + super(DeleteLogTool, self).__init__(*args, **kwargs) def _delete_related_log(self, regex, show=False): log_found = 0 @@ -155,7 +157,7 @@ def delete_related_log(self, skip_ask=False): return 0 def delete_small_timestep_log(self, skip_ask=False): - self._find_small_timestep_log() + self.small_timestep_regs = self._find_small_timestep_log(self.proj_root, self.task_table_name, self.regex, timstep_upper_bound=self.filter.timstep_bound) print("complete searching.") if skip_ask: s = 'y' @@ -164,17 +166,17 @@ def delete_small_timestep_log(self, skip_ask=False): log_found = 0 if s == 'y' or skip_ask: - for reg in self.small_timestep_regs: - print("[delete small-timestep log] reg: ", reg) - self._delete_related_log(show=True, regex=reg + '*') + for res in self.small_timestep_regs: + print("[delete small-timestep log] reg: ", res[1]) + self._delete_related_log(show=True, regex=res[0] + '*') if skip_ask: s = 'y' else: s = input("delete these files? (y/n)") if s == 'y' or skip_ask: - for reg in self.small_timestep_regs: - print("do delete: ", reg) - log_found += self._delete_related_log(show=False, regex=reg + '*') + for res in self.small_timestep_regs: + print("do delete: ", res[1]) + log_found += self._delete_related_log(show=False, regex=res[0] + '*') return log_found class ArchiveLogTool(BasicLogTool): @@ -231,6 +233,44 @@ def archive_log(self, skip_ask=False): print("do archive ...") self._archive_log(show=False) +class ViewLogTool(BasicLogTool): + def __init__(self, proj_root, task_table_name, regex, *args, **kwargs): + self.proj_root = proj_root + self.task_table_name = task_table_name + self.regex = regex + super(ViewLogTool, self).__init__(*args, **kwargs) + + def _view_log(self, regex): + root_dir_regex = osp.join(self.proj_root, LOG, self.task_table_name, regex) + for root_dir in glob.glob(root_dir_regex): + if os.path.exists(root_dir): + for file_list in os.walk(root_dir): + if re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]): + target_reg = re.search(r'\d{4}/\d{2}/\d{2}/\d{2}-\d{2}-\d{2}-\d{6}', file_list[0]).group(0) + else: + target_reg = None + if target_reg is not None: + backup_file = file_list[0] + '/backup.txt' + if file_list[1] == ['tb'] or os.path.exists(backup_file): # in root of logdir + with open(backup_file) as f: + print(f.read()) + + def view_log(self, skip_ask=False): + found_regs = self._find_small_timestep_log(self.proj_root, self.task_table_name, self.regex, timestep_lower_bound=1) + for res in found_regs: + print("view experiments:", res[1]) + if skip_ask: + s = 'y' + else: + s = input("press y to view \n ") + if s == 'y': + self._view_log(regex=res[0] + '*') + + + + + + # if __name__ == '__main__': # dlt = DeleteLogTool("../", "var_seq_imitation", "self-transfer", "2019/11/29/01-11*") # dlt.delete_related_log() diff --git a/rla_scripts/view_expt.py b/rla_scripts/view_expt.py index e69de29..e29da0d 100644 --- a/rla_scripts/view_expt.py +++ b/rla_scripts/view_expt.py @@ -0,0 +1,19 @@ +""" +A script to view data of experiments. +""" + +from RLA.easy_log.log_tools import ViewLogTool +import argparse +from config import * + +def argsparser(): + parser = argparse.ArgumentParser("Archive Log") + parser.add_argument('--task_table_name', type=str) + parser.add_argument('--regex', type=str) + args = parser.parse_args() + return args + +if __name__=='__main__': + args = argsparser() + dlt = ViewLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table_name, regex=args.regex) + dlt.view_log() \ No newline at end of file diff --git a/test/test_scripts.py b/test/test_scripts.py index ae877ee..91d061a 100644 --- a/test/test_scripts.py +++ b/test/test_scripts.py @@ -1,6 +1,7 @@ from test._base import BaseTest from RLA.easy_log.log_tools import DeleteLogTool, Filter -from RLA.easy_log.log_tools import ArchiveLogTool +from RLA.easy_log.log_tools import ArchiveLogTool, ViewLogTool + class ScriptTest(BaseTest): @@ -51,4 +52,8 @@ def test_archive(self): dlt = DeleteLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name='archived', regex='2022/03/01/21-13*', filter=filter) log_found = dlt.delete_related_log(skip_ask=True) assert log_found == 10 - \ No newline at end of file + + def test_view(self): + self.remove_and_copy_data() + dlt = ViewLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name=self.TASK_NAME, regex='2022/03/01/21-13*') + dlt.view_log(skip_ask=True) \ No newline at end of file