From 2cc0e80675e5872bd1756ca32e0ff5a3aed59667 Mon Sep 17 00:00:00 2001 From: Victor Schmidt Date: Mon, 25 Sep 2023 14:58:38 -0400 Subject: [PATCH 1/6] handle logger notes in wandb --- gflownet/utils/logger.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/gflownet/utils/logger.py b/gflownet/utils/logger.py index e9556f9c5..50356b377 100644 --- a/gflownet/utils/logger.py +++ b/gflownet/utils/logger.py @@ -32,6 +32,7 @@ def __init__( run_name=None, tags: list = None, context: str = "0", + notes: str = None, ): self.config = config self.do = do @@ -60,7 +61,7 @@ def __init__( if slurm_job_id: wandb_config["slurm_job_id"] = slurm_job_id self.run = self.wandb.init( - config=wandb_config, project=project_name, name=run_name + config=wandb_config, project=project_name, name=run_name, notes=notes ) else: self.wandb = None From 0158c43d04239d2350277398dc25784e8cff99b6 Mon Sep 17 00:00:00 2001 From: Victor Schmidt Date: Mon, 25 Sep 2023 14:59:39 -0400 Subject: [PATCH 2/6] describe notes config --- config/logger/base.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/config/logger/base.yaml b/config/logger/base.yaml index bc95f20d6..640167c81 100644 --- a/config/logger/base.yaml +++ b/config/logger/base.yaml @@ -44,3 +44,4 @@ debug: False lightweight: False progress: True context: "0" +notes: null # wandb run notes (e.g. "baseline") From b10810add8ead4ce63316572c4a6bc0307aac850 Mon Sep 17 00:00:00 2001 From: Victor Schmidt Date: Mon, 25 Sep 2023 16:31:38 -0400 Subject: [PATCH 3/6] handle quotes in generated command-line --- mila/launch.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/mila/launch.py b/mila/launch.py index b1aa0f227..19ba41cd2 100644 --- a/mila/launch.py +++ b/mila/launch.py @@ -262,6 +262,18 @@ def find_jobs_conf(args): return jobs_conf_path, local_out_dir +def quote(value): + v = str(value) + if " " in v: + if "'" not in v: + v = f"'{v}'" + elif '"' not in v: + v = f'"{v}"' + else: + raise ValueError(f"Cannot quote {value}") + return v + + def script_dict_to_main_args_str(script_dict, is_first=True, nested_key=""): """ Recursively turns a dict of script args into a string of main.py args @@ -272,11 +284,14 @@ def script_dict_to_main_args_str(script_dict, is_first=True, nested_key=""): previous_str (str, optional): base string to append to. Defaults to "". """ if not isinstance(script_dict, dict): - return nested_key + "=" + str(script_dict) + " " + return f"{nested_key}={quote(script_dict)} " new_str = "" for k, v in script_dict.items(): if k == "__value__": - new_str += nested_key + "=" + str(v) + " " + value = str(v) + if " " in value: + value = f"'{value}'" + new_str += f"{nested_key}={quote(v)} " continue new_key = k if not nested_key else nested_key + "." + str(k) new_str += script_dict_to_main_args_str(v, nested_key=new_key, is_first=False) From ba57f7f00424261ec4c9b82195f1cc017887a0c1 Mon Sep 17 00:00:00 2001 From: Victor Schmidt Date: Mon, 25 Sep 2023 17:05:45 -0400 Subject: [PATCH 4/6] handle possible "=" in notes --- mila/launch.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mila/launch.py b/mila/launch.py index 19ba41cd2..2d09d0a45 100644 --- a/mila/launch.py +++ b/mila/launch.py @@ -264,7 +264,8 @@ def find_jobs_conf(args): def quote(value): v = str(value) - if " " in v: + v = v.replace("(", r"\(").replace(")", r"\)") + if " " in v or "=" in v: if "'" not in v: v = f"'{v}'" elif '"' not in v: @@ -557,13 +558,20 @@ def print_md_help(parser, defaults): sbatch_path.parent.mkdir(parents=True, exist_ok=True) # write template sbatch_path.write_text(templated) - print(f" 🏷 Created ./{sbatch_path.relative_to(Path.cwd())}") + print() # Submit job to SLURM out = popen(f"sbatch {sbatch_path}").read() # Identify printed-out job id job_id = re.findall(r"Submitted batch job (\d+)", out)[0] job_ids.append(job_id) print(" ✅ " + out) + # Rename sbatch file with job id + parts = sbatch_path.stem.split(f"_{now}") + new_name = f"{parts[0]}_{job_id}_{now}" + if len(parts) > 1: + new_name += f"_{parts[1]}" + sbatch_path = sbatch_path.rename(sbatch_path.parent / new_name) + print(f" 🏷 Created ./{sbatch_path.relative_to(Path.cwd())}") # Write job ID & output file path in the sbatch file job_output_file = str(outdir / f"{job_args['job_name']}-{job_id}.out") job_out_files.append(job_output_file) From 0ed9684b07f57b74a231838b82ec76b27ffb7c4e Mon Sep 17 00:00:00 2001 From: Victor Schmidt Date: Mon, 25 Sep 2023 17:19:05 -0400 Subject: [PATCH 5/6] quote both key AND value if = in CLI --- mila/launch.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/mila/launch.py b/mila/launch.py index 2d09d0a45..2b90f4575 100644 --- a/mila/launch.py +++ b/mila/launch.py @@ -265,11 +265,11 @@ def find_jobs_conf(args): def quote(value): v = str(value) v = v.replace("(", r"\(").replace(")", r"\)") - if " " in v or "=" in v: - if "'" not in v: - v = f"'{v}'" - elif '"' not in v: + if " " in v: + if '"' not in v: v = f'"{v}"' + elif "'" not in v: + v = f"'{v}'" else: raise ValueError(f"Cannot quote {value}") return v @@ -285,14 +285,24 @@ def script_dict_to_main_args_str(script_dict, is_first=True, nested_key=""): previous_str (str, optional): base string to append to. Defaults to "". """ if not isinstance(script_dict, dict): - return f"{nested_key}={quote(script_dict)} " + candidate = f"{nested_key}={quote(script_dict)}" + if candidate.count("=") > 1: + assert "'" not in candidate, """Keys cannot contain ` ` and `'` and `=` """ + candidate = f"'{candidate}'" + return candidate + " " new_str = "" for k, v in script_dict.items(): if k == "__value__": value = str(v) if " " in value: value = f"'{value}'" - new_str += f"{nested_key}={quote(v)} " + candidate = f"{nested_key}={quote(v)} " + if candidate.count("=") > 1: + assert ( + "'" not in candidate + ), """Keys cannot contain ` ` and `'` and `=` """ + candidate = f"'{candidate}'" + new_str += candidate continue new_key = k if not nested_key else nested_key + "." + str(k) new_str += script_dict_to_main_args_str(v, nested_key=new_key, is_first=False) From d698ea701b52684a8c05e8a3a5947fe764c22f4b Mon Sep 17 00:00:00 2001 From: Victor Schmidt Date: Mon, 25 Sep 2023 17:23:42 -0400 Subject: [PATCH 6/6] typo: removed first level quoting --- mila/launch.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mila/launch.py b/mila/launch.py index 2b90f4575..180a289b4 100644 --- a/mila/launch.py +++ b/mila/launch.py @@ -265,7 +265,7 @@ def find_jobs_conf(args): def quote(value): v = str(value) v = v.replace("(", r"\(").replace(")", r"\)") - if " " in v: + if " " in v or "=" in v: if '"' not in v: v = f'"{v}"' elif "'" not in v: