Version 1.0

gpt_plan_test/Executor/__init__.py

@@ -45,7 +45,7 @@ def replanning_domain_specific(self, harder=0, domain='blocksworld'):
                 if 'holding' in i:
                     current_block = i.split('_')[1]
                     to_remove.add(i)
-            selected_block = random.choice(list(all_blocks))
+            selected_block = random.choice(sorted(list(all_blocks)))
             to_remove.add('clear_' + selected_block)
             to_add = {'on_' + current_block + '_' + selected_block, 'handempty', 'clear_' + current_block}
             self.replanning_init = self.replanning_init.union(to_add)
@@ -126,7 +126,6 @@ def regress(self, harder):
     def random_prefix_execution(self, replan=False):
         print("PLAN", self.plan)
         self.prefix = random.choice(range(1, len(self.plan)))
-        # print("PREFIX", self.prefix)
         self.final_state = self.get_final_state(self.init_state, 0, self.prefix)
         self.all_preds = self.get_sets(self.model[PREDICATES])
         self.not_true_preds = self.all_preds.difference(self.final_state)

gpt_plan_test/ReasoningTasks.py

@@ -156,7 +156,7 @@ def t1_t4(self, config_file, t1_or_t4="1_reasoning"):
                       f"\n-------- Ground truth plan ---------\n{gt_plan_text}")
 
             self.save_output("task" + t1_or_t4, final_output)
-            break
+
 
         # --------------- Add to final output --------------- #
         final_output += f"[+]: The number of correct plans is " + \
@@ -297,7 +297,7 @@ def paraphrase_goal(exec):
             os.makedirs(f"outputs/{self.engine}/", exist_ok=True)
             with open(f"outputs/{self.engine}/task2_paraphrase.txt", 'w+') as f:
                 f.write(final_output)
-            break
+
 
         print(single_goal_instances)
 
@@ -389,11 +389,12 @@ def t5_optimality(self, config_file):
             _, gpt3_plan = text_to_plan_blocksworld(gpt3_response, problem.actions, self.gpt3_plan_file, self.data)
             # Apply VAL
             correct = int(validate_plan(domain, cur_instance, self.gpt3_plan_file))
-            final_output += "\n===================================SUCCESS===================================\n" if correct else "\n===================================FAILURE===================================\n"
-            final_output += f"{query}\n--------- GPT3 response ---------\n{gpt3_response}\n" + \
-                            f"--------- Extracted plan ---------\n{gpt3_plan}" + \
-                            f"\n-------- Ground truth plan ---------\n{gt_plan_text}"
-            final_output += "\n=============================================================================\n"
+            if not correct:
+                final_output += "\n===================================FAILURE===================================\n"
+                final_output += '\n--------Invalid Plan-------\n'
+
+
+
 
             if self.verbose:
                 print(f"{query}\n--------- GPT3 response ---------\n{gpt3_response}\n"
@@ -403,12 +404,22 @@ def t5_optimality(self, config_file):
 
             if correct:
                 cost = get_cost_gpt_3(gpt3_response)
+                plan_list = [len(pl) > 0 for pl in gpt3_plan.split('\n')]
+                actual_cost_gpt3 = sum(plan_list)
+                txt= "---------WRONG COST OUTPUT BY LLM---------" if cost != actual_cost_gpt3 else "---------CORRECT COST OUTPUT BY LLM---------"
                 print("COST OF GPT_3 PLAN", cost)
-                if cost == plan_executor.cost:
+                if actual_cost_gpt3 == plan_executor.cost:
                     correct_plans += 1
-                    final_output += '\n--------Optimal Plan-------\n'
+                    final_output += "\n===================================SUCCESS===================================\n"
+                    final_output += '\n----------------Optimal Plan----------------\n'
                 else:
-                    final_output += '\n--------Sub-Optimal Plan-------\n'
+                    final_output += "\n===================================FAILURE===================================\n"
+                    final_output += '\n----------------Sub-optimal Plan----------------\n'
+                final_output += txt
+            final_output += f"{query}\n--------- GPT3 response ---------\n{gpt3_response}\n" + \
+                            f"--------- Extracted plan ---------\n{gpt3_plan}" + \
+                            f"\n-------- Ground truth plan ---------\n{gt_plan_text}"
+            final_output += "\n=============================================================================\n"
             self.save_output("task5_optimality", final_output)
             break
 
@@ -444,10 +455,10 @@ def t6_replanning(self, config_file, harder):
                 final_output += f"\n Instance {cur_instance}\n"
                 print(f"Instance {cur_instance}")
                 # gt_plan = self.compute_plan(domain, cur_instance)
-                text, gt_plan_text = replanning(plan_executor, self.data, get_plan, harder)
+                text, plan = replanning(plan_executor, self.data, get_plan, harder)
                 query += text
             gpt3_response = send_query_gpt3(query, self.engine, self.max_gpt_response_length)
-
+            gt_plan_text = get_plan_as_text(self.data, plan)
             # Do text_to_plan procedure
             _, gpt3_plan = text_to_plan_blocksworld(gpt3_response, problem.actions, self.gpt3_plan_file, self.data)
             # Apply VAL

gpt_plan_test/execution.stats

@@ -1,5 +1,5 @@
 Domain=BLOCKSWORLD-4OPS
 Problem=BW-RAND-4
-Preprocessing=0.00216
+Preprocessing=0.00298
 Fluents=30
 Operators=42

gpt_plan_test/grounding.stats

@@ -1,5 +1,5 @@
-Parsing=0.000142
-Instantiation=0.00175
-Fluent_Copying=5.9e-05
-Init_And_Goal_Copying=8e-06
-Operators_Copying=0.000124
+Parsing=0.000283
+Instantiation=0.00236
+Fluent_Copying=7.1e-05
+Init_And_Goal_Copying=9e-06
+Operators_Copying=0.000148