Merge pull request #8 from xionghuichen/dev

Dev

.gitignore

@@ -8,6 +8,7 @@ RLA.egg-info**
 **/code/**
 **/results/**
 **/log/**
+**/arc/**
 **/.ipynb_checkpoints/*
 **/.DS_Store
 test/target_data_root/*

README.md

@@ -116,6 +116,7 @@ We build an example project for integrating RLA, which can be seen in ./example/
    **/code/**
    **/results/**
    **/log/**
+   **/arc/**
    ```
 ### Step2: record intermediate variables/checkpoints/other types of data.
 
@@ -208,7 +209,7 @@ Usually, it is unnecessary to change the content of experiment logs. In our prac
 **Batch Management**
 
 We manage the items in the database via toolkits in rla_scripts. Currently, the rla_scripts includes
-1. Archive: archive some important results into another table.
+1. Archive: archive important experiments into an archived database, which will be saved in DATA_ROOT/arc.
 2. Delete: delete all useless experiments at once.
 3. Send to remote [TODO]
 4. Download from remote [TODO]
@@ -241,7 +242,8 @@ PS:
 2. An alternative way is building your own NFS for your physical machines and locate data_root to the NFS.
 
 # TODO
+- [ ] video visualization.
 - [ ] add comments and documents to the functions.
 - [ ] add an auto integration script.
-- [ ] download / upload experiment logs through timestamp;
+- [ ] download / upload experiment logs through timestamp.
 

RLA/easy_log/const.py

@@ -3,7 +3,7 @@
 CHECKPOINT = 'checkpoint'
 ARCHIVE_TESTER = 'archive_tester'
 OTHER_RESULTS = 'results'
-
+ARCHIVED_TABLE = 'arc'
 default_log_types = [LOG, CODE, CHECKPOINT, ARCHIVE_TESTER, OTHER_RESULTS]
 
 class LOAD_TESTER_MODE:

RLA/easy_log/log_tools.py

@@ -140,7 +140,7 @@ def _delete_related_log(self, regex, show=False):
                             os.remove(root_dir)
                             print("--- delete root file {} ---".format(root_dir))
                 else:
-                    print("not dir {}".format(root_dir))
+                    print("no dir {}".format(root_dir))
             if empty: print("empty regex {}".format(root_dir_regex))
         return log_found
 
@@ -180,46 +180,39 @@ def delete_small_timestep_log(self, skip_ask=False):
         return log_found
 
 class ArchiveLogTool(BasicLogTool):
-    def __init__(self, proj_root, task_table_name, regex, archive_table_name, remove, *args, **kwargs):
+    def __init__(self, proj_root, task_table_name, regex, archive_table_name=ARCHIVED_TABLE, *args, **kwargs):
         self.proj_root = proj_root
         self.task_table_name = task_table_name
         self.regex = regex
-        self.remove = remove
         self.archive_table_name = archive_table_name
         super(ArchiveLogTool, self).__init__(*args, **kwargs)
 
     def _archive_log(self, show=False):
         for log_type in self.log_types:
             root_dir_regex = osp.join(self.proj_root, log_type, self.task_table_name, self.regex)
-            archive_root_dir = osp.join(self.proj_root, log_type, self.archive_table_name)
-            prefix_dir = osp.join(self.proj_root, log_type, self.task_table_name)
+            archive_root_dir = osp.join(self.proj_root, self.archive_table_name, log_type)
+            prefix_dir = osp.join(self.proj_root, log_type)
             prefix_len = len(prefix_dir)
             empty = True
             # os.system("chmod +x -R \"{}\"".format(prefix_dir))
             for root_dir in glob.glob(root_dir_regex):
                 empty = False
                 if os.path.exists(root_dir):
+                    # remove the overlapped path.
                     archiving_target = osp.join(archive_root_dir, root_dir[prefix_len+1:])
                     archiving_target_dir = '/'.join(archiving_target.split('/')[:-1])
                     os.makedirs(archiving_target_dir, exist_ok=True)
                     if os.path.isdir(root_dir):
                         if not show:
                             # os.makedirs(archiving_target, exist_ok=True)
                             shutil.copytree(root_dir, archiving_target)
-                            if self.remove:
-                                try:
-                                    shutil.rmtree(root_dir)
-                                except PermissionError as e:
-                                    print("skip the permission error file")
-                        print("move dir {}, to {}".format(root_dir, archiving_target))
+                        print("copy dir {}, to {}".format(root_dir, archiving_target))
                     else:
                         if not show:
                             shutil.copy(root_dir, archiving_target)
-                            if self.remove:
-                                os.remove(root_dir)
-                        print("move file {}, to {}".format(root_dir, archiving_target))
+                        print("copy file {}, to {}".format(root_dir, archiving_target))
                 else:
-                    print("not dir {}".format(root_dir))
+                    print("no dir {}".format(root_dir))
             if empty: print("empty regex {}".format(root_dir_regex))
         pass
 
@@ -265,12 +258,3 @@ def view_log(self, skip_ask=False):
                 s = input("press y to view \n ")
             if s == 'y':
                 self._view_log(regex=res[0] + '*')
-
-
-
-
-
-
-# if __name__ == '__main__':
-#     dlt = DeleteLogTool("../", "var_seq_imitation", "self-transfer", "2019/11/29/01-11*")
-#     dlt.delete_related_log()

rla_scripts/archive_expt.py

@@ -15,16 +15,12 @@ def argsparser():
     parser = argparse.ArgumentParser("Archive Log")
     # reduce setting
     parser.add_argument('--task_table_name', type=str)
-    parser.add_argument('--archive_table_name', type=str, default=ARCHIVED_TABLE)
     parser.add_argument('--regex', type=str)
-    parser.add_argument('--remove', action='store_true')
-
 
     args = parser.parse_args()
     return args
 
 if __name__=='__main__':
     args = argsparser()
-    dlt = ArchiveLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table_name, regex=args.regex,
-                         archive_table_name=args.archive_table_name, remove=args.remove)
+    dlt = ArchiveLogTool(proj_root=DATA_ROOT, task_table_name=args.task_table_name, regex=args.regex)
     dlt.archive_log()

rla_scripts/config.py

@@ -1,2 +1 @@
 DATA_ROOT = '../example/simplest_code/'
-ARCHIVED_TABLE = 'archived'

rla_scripts/delete_expt.py

@@ -12,6 +12,9 @@ def argsparser():
     parser.add_argument('--task_table_name', type=str, default="")
     parser.add_argument('--regex', type=str)
     parser.add_argument('--timestep_bound', type=int, default=100)
+    # Filter.ALL: delete all experiments satisfied regex
+    # Filter.SMALL_TIMESTEP: delete all experiments that the names satisfy regex
+    # and the recorded timesteps are less than args.timestep_bound.
     parser.add_argument('--delete_type', type=str, default=Filter.ALL)
 
     args = parser.parse_args()

rla_scripts/view_expt.py

@@ -7,7 +7,7 @@
 from config import *
 
 def argsparser():
-    parser = argparse.ArgumentParser("Archive Log")
+    parser = argparse.ArgumentParser("View Log")
     parser.add_argument('--task_table_name', type=str)
     parser.add_argument('--regex', type=str)
     args = parser.parse_args()

setup.py

@@ -5,7 +5,7 @@
 
 setup(
         name='RLA',
-        version="0.5.2",
+        version="0.5.3",
         description=(
             'RL assistant'
         ),
@@ -20,6 +20,7 @@
             "argparse",
             "dill",
             "seaborn",
-            "pathspec"
+            "pathspec",
+            'tensorboardX'
         ]
     )

test/test_scripts.py

@@ -43,13 +43,12 @@ def test_delete_reg_small_ts(self):
     def test_archive(self):
         self.remove_and_copy_data()
         # archive experiments.
-        dlt = ArchiveLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name=self.TASK_NAME, regex='2022/03/01/21-13*',
-                             archive_table_name='archived', remove=False)
+        dlt = ArchiveLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name=self.TASK_NAME, regex='2022/03/01/21-13*')
         dlt.archive_log(skip_ask=True)
         # remove the archived experiments.
         filter = Filter()
         filter.config(type=Filter.ALL, timstep_bound=1)
-        dlt = DeleteLogTool(proj_root=self.TARGET_DATA_ROOT, task_table_name='archived', regex='2022/03/01/21-13*', filter=filter)
+        dlt = DeleteLogTool(proj_root=self.TARGET_DATA_ROOT + '/arc', regex='2022/03/01/21-13*', filter=filter, task_table_name=self.TASK_NAME)
         log_found = dlt.delete_related_log(skip_ask=True)
         assert log_found == 10