interpret works!

Author: NishanthJKumar

predicators/datasets/generate_atom_trajs_with_vlm.py

@@ -1126,10 +1126,10 @@ def create_ground_atom_data_from_generated_demos(
     else:
         # import pdb; pdb.set_trace()
         generate_func = _generate_ground_atoms_with_vlm_pure_visual_preds
-    # ground_atoms_trajs = generate_func(img_option_trajs, env, train_tasks,
-    #                                    known_predicates, all_task_objs, vlm)
-    # return Dataset(option_segmented_trajs, ground_atoms_trajs)
-    return Dataset(option_segmented_trajs, None)
+    ground_atoms_trajs = generate_func(img_option_trajs, env, train_tasks,
+                                       known_predicates, all_task_objs, vlm)
+    return Dataset(option_segmented_trajs, ground_atoms_trajs)
+    # return Dataset(option_segmented_trajs, None)
 
 
 def create_ground_atom_data_from_labelled_txt(

predicators/envs/pybullet_coffee.py

@@ -48,7 +48,7 @@
     create_single_arm_pybullet_robot
 from predicators.settings import CFG
 from predicators.structs import Action, Array, EnvironmentTask, Object, \
-    Predicate, State, Observation, Task
+    Predicate, State, Observation, Task, Type
 
 class PyBulletCoffeeEnv(PyBulletEnv, CoffeeEnv):
     """PyBullet Coffee domain.
@@ -214,6 +214,14 @@ def pour_z_offset(cls) -> float:
     _camera_pitch: ClassVar[float]
     _camera_target: ClassVar[Pose3D]
 
+    # Types
+    _table_type = Type("table", [])
+    _robot_type = Type("robot", ["x", "y", "z", "tilt", "wrist", "fingers"])
+    _jug_type = Type("jug", ["x", "y", "z", "rot", "is_held", "is_filled"])
+    _machine_type = Type("coffee_machine", ["is_on"])
+    _cup_type = Type("cup", ["x", "y", "z", "capacity_liquid", "target_liquid", "current_liquid"])
+    _plug_type = Type("plug", ["x", "y", "z", "plugged_in"])
+
     def __init__(self, use_gui: bool = True) -> None:
         super().__init__(use_gui)
 
@@ -626,7 +634,6 @@ def _get_state(self, render_obs: bool = False) -> State:
         assert set(state) == set(self._current_state), \
             (f"Reconstructed state has objects {set(state)}, but "
              f"self._current_state has objects {set(self._current_state)}.")
-
         return state
 
     def step(self, action: Action, render_obs: bool = False) -> State:

scripts/configs/pred_invention_vlm.yaml

@@ -36,16 +36,16 @@ APPROACHES:
   #   FLAGS:
   #     grammar_search_vlm_atom_proposal_prompt_type: demo_with_vlm_imgs
   #     grammar_search_invent_geo_predicates_only: False
-  # interpret:
-  #   NAME: "grammar_search_invention"
-  #   FLAGS:
-  #     offline_data_method: demo_with_vlm_imgs
-  #     vlm_predicate_vision_api_generate_ground_atoms: True
-  vila-with-fewshot:
-    NAME: "vlm_open_loop"
+  interpret:
+    NAME: "grammar_search_invention"
     FLAGS:
-      vlm_open_loop_use_training_demos: True
-      offline_data_method: geo_and_demo_with_vlm_imgs
+      offline_data_method: demo_with_vlm_imgs
+      vlm_predicate_vision_api_generate_ground_atoms: True
+  # vila-with-fewshot:
+  #   NAME: "vlm_open_loop"
+  #   FLAGS:
+  #     vlm_open_loop_use_training_demos: True
+  #     offline_data_method: geo_and_demo_with_vlm_imgs
 
 
 ENVS: