revise benchmark

minestudio/benchmark/auto_eval/README.md

@@ -9,9 +9,9 @@ python rule_generation.py
 ## Evaluating videos with VLM
 ```bash
 #Compare two videos
-python video_comparison.py --video_path_a='./eval_video/build_pillar/build_pillar_0.mp4' --video_path_b='./eval_video/build_pillar/build_pillar_6.mp4' --criteria_path='./auto_eval/criteria_files/build_pillar.txt' 
+python video_comparison.py --video_path_a='./eval_video/build_gate/build_gate_5.mp4' --video_path_b='./eval_video/build_gate/build_gate_7.mp4' --criteria_path='./auto_eval/criteria_files/build_gate.txt' 
 #Individual video evaluation
-python individual_video_rating.py --video_path='./eval_video/build_pillar/build_pillar_0.mp4' --criteria_path='./auto_eval/criteria_files/build_pillar.txt' 
+python individual_video_rating.py --video_path='./eval_video/build_gate/build_gate_5.mp4' --criteria_path='./auto_eval/criteria_files/build_gate.txt' 
 #Batch video evaluation
 python batch_video_rating.py --videos_path='./eval_video/' --criteria_files_path='./auto_eval/criteria_files/' 
 ```

minestudio/benchmark/auto_eval/batch_video_rating.py

@@ -1,3 +1,9 @@
+'''
+Date: 2024-11-21 21:38:32
+LastEditors: zhengxinyue
+LastEditTime: 2024-12-06 16:38:33
+FilePath: /MineStudio/minestudio/benchmark/auto_eval/batch_video_rating.py
+'''
 import cv2  # We're using OpenCV to read video
 import base64
 import time

minestudio/benchmark/auto_eval/individual_video_rating.py

@@ -1,3 +1,9 @@
+'''
+Date: 2024-11-21 21:38:32
+LastEditors: zhengxinyue
+LastEditTime: 2024-12-06 16:38:55
+FilePath: /MineStudio/minestudio/benchmark/auto_eval/individual_video_rating.py
+'''
 import cv2  # We're using OpenCV to read video
 import base64
 import time

minestudio/benchmark/auto_eval/rule_generation.py

@@ -1,3 +1,9 @@
+'''
+Date: 2024-11-21 21:38:32
+LastEditors: zhengxinyue
+LastEditTime: 2024-12-06 16:39:05
+FilePath: /MineStudio/minestudio/benchmark/auto_eval/rule_generation.py
+'''
 import os
 import argparse
 import numpy as np

minestudio/benchmark/auto_eval/video_comparison.py

@@ -1,3 +1,9 @@
+'''
+Date: 2024-11-21 21:38:32
+LastEditors: zhengxinyue
+LastEditTime: 2024-12-06 16:39:13
+FilePath: /MineStudio/minestudio/benchmark/auto_eval/video_comparison.py
+'''
 import cv2  # We're using OpenCV to read video
 import base64
 import time

minestudio/benchmark/read_conf.py

@@ -0,0 +1,22 @@
+'''
+Date: 2024-12-06 16:42:49
+LastEditors: zhengxinyue
+LastEditTime: 2024-12-11 17:44:10
+FilePath: /MineStudio/minestudio/benchmark/read_conf.py
+'''
+import os
+import yaml
+
+def convert_yaml_to_callbacks(yaml_file):
+    with open(yaml_file, 'r') as f:
+        data = yaml.safe_load(f)
+    commands = data.get('custom_init_commands', [])
+    commands_callback = f"{commands}"
+
+    text = data.get('text', '')
+    task_name = os.path.splitext(os.path.basename(yaml_file))[0]
+    task_callback = (
+        f"{{'name': '{task_name}', 'text': '{text}'}}"
+    )
+
+    return commands_callback, task_callback

minestudio/benchmark/task_configs/simple/build_dig3fill1.yaml

@@ -0,0 +1,7 @@
+custom_init_commands:
+- /give @s minecraft:dirt 15
+- /give @s minecraft:stone_shovel
+defaults:
+- base
+- _self_
+text: Build a 3x3 area filled with dirt blocks.

minestudio/benchmark/test.py

@@ -0,0 +1,67 @@
+'''
+Date: 2024-12-06 16:35:39
+LastEditors: zhengxinyue
+LastEditTime: 2024-12-11 17:47:25
+FilePath: /MineStudio/minestudio/benchmark/test.py
+'''
+import sys
+sys.path.append('/home/user/code/MineStudio/')
+import os
+import ray
+from rich import print
+from minestudio.inference import EpisodePipeline, MineGenerator, InfoBaseFilter
+from minestudio.benchmark.read_conf import convert_yaml_to_callbacks
+from functools import partial
+from minestudio.models import load_openai_policy
+from minestudio.simulator import MinecraftSim
+from minestudio.simulator.callbacks import (
+    SpeedTestCallback, 
+    RecordCallback, 
+    SummonMobsCallback, 
+    MaskActionsCallback, 
+    RewardsCallback, 
+    CommandsCallback, 
+    TaskCallback,
+    FastResetCallback
+)
+
+import pdb
+
+
+if __name__ == '__main__':
+    ray.init()
+    conf_path = './task_configs/simple'
+
+    for file_name in os.listdir(conf_path):
+        if file_name.endswith('.yaml'):
+            file_path = os.path.join(conf_path, file_name)
+            commands_callback, task_callback = convert_yaml_to_callbacks(file_path)
+
+            env = MinecraftSim(
+                obs_size=(128, 128), 
+                preferred_spawn_biome="forest", 
+                callbacks=[
+                    RecordCallback(record_path=f"./output/{file_name}", fps=30, frame_type="pov"),
+                    CommandsCallback(commands_callback),
+                    TaskCallback(task_callback),
+                ]
+            )
+            # pdb.set_trace()
+            policy = load_openai_policy(
+                model_path="/nfs-shared/jarvisbase/pretrained/foundation-model-2x.model",
+                weights_path="/nfs-shared/jarvisbase/pretrained/foundation-model-2x.weights"
+            ).to("cuda")
+
+            memory = None
+            obs, info = env.reset()
+            for i in range(1200):
+                action, memory = policy.get_action(obs, memory, input_shape='*')
+                obs, reward, terminated, truncated, info = env.step(action)
+            env.close()
+
+
+
+
+
+
+