slight changes for debugging.

Author: ashayathalye

predicators/approaches/grammar_search_invention_approach.py

@@ -487,9 +487,9 @@ def enumerate(self) -> Iterator[Tuple[Predicate, float]]:
         "Cut0",
     ],
     "pybullet_coffee": [
-        "JugInMachine",
-        "Holding",
-        "HandEmpty",
+        "JugInMachine0",
+        "Holding0",
+        "HandEmpty0",
     ]
 }
 _DEBUG_VLM_PREDICATES = defaultdict(list, _DEBUG_VLM_PREDICATES)
@@ -1107,6 +1107,7 @@ def rename(p: str) -> str:  # pragma: no cover
                 self._learned_predicates = set(
                     p for p in candidates.keys()
                     if p.name in debug_predicate_names)
+                import pdb; pdb.set_trace()
             else:
                 # Create the score function that will be used to guide search.
                 score_function = create_score_function(
@@ -1123,6 +1124,7 @@ def rename(p: str) -> str:  # pragma: no cover
         elif CFG.grammar_search_pred_selection_approach == "no_select":
             self._learned_predicates = set(candidates.keys())
         logging.info("Done.")
+
         # Finally, learn NSRTs via superclass, using all the kept predicates.
         annotations = None
         if dataset.has_annotations:

predicators/datasets/generate_atom_trajs_with_vlm.py

@@ -113,6 +113,7 @@ def _sample_vlm_atom_proposals_from_trajectories(
     """Given a list of ImageOptionTrajectories, query a VLM to generate a list
     of names of ground atoms from which we can extract predicates that might be
     relevant for planning to recreate these trajectories."""
+    import pdb; pdb.set_trace()
     aggregated_vlm_output_strs = []
     all_vlm_queries_list = []
     for traj in trajectories:

predicators/datasets/vlm_input_data_prompts/atom_labelling/img_option_diffs_label_history.txt

@@ -1,5 +1,7 @@
 You are a vision system for a robot provided with two images: a before image showing the state before a skill is executed, an after image showing the state after the skill is executed. You are given a list of predicates below, and you are given the values of these predicates in the image before the skill is executed. Your job is to output the values of the following predicates in the image after the skill is executed. Pay careful attention to the visual changes between the two images to figure out which predicates change and which predicates do not change. Note that some or all of the predicates don't necessary have to change. First, output a description of what changes you expect to happen based on the skill that was just run, explicitly noting the skill that was run. Second, output a description of what visual changes you see happen between the before and after images, looking specifically at the objects involved in the skill's arguments, noting what objects these are. From these two descriptions, for each predicate labeled in the previous timestep, note whether you expect its value to change or stay the same. Next, for each predicate given in the list of predicates to label, output each predicate value in the after image as a bulleted list (use '*' for the bullets) with each predicate and value on a different line. Ensure there is a period ('.') after the truth value of the predicate. For each predicate value, provide an explanation as to why you labelled this predicate as having this particular value, and note what value this predicate had in the previous timestep, which is given to you in the prompt. Use the format: `* <predicate>: <truth_value>. <explanation>`. When labeling the value of a predicate, if you don't see the objects involved in that predicate, retain its truth value from the previous timestep. Also, if your description of changes you expect to happen, and your description of visual changes you saw happen, have nothing to do with the predicate you are trying to label, retain its truth value from the previous timestep. For example, if in the previous timestep I paint an object, and in the current timestamp I sit on it, we don't expect its color to change after sitting on it. 
 
+Note that the coffee machine is black and the table is white. 
+
 Your response should have three sections. Here is an outline of what your response should look like:
 [START OULTLINE]
 # Expected changes based on the executed skill

predicators/datasets/vlm_input_data_prompts/atom_labelling/per_scene_naive.txt

@@ -1,2 +1,7 @@
-You are a vision system for a robot. Your job is to output the values of the following predicates based on the provided visual scene. For each predicate, output True, False, or Unknown if the relevant objects are not in the scene or the value of the predicate simply cannot be determined. Output each predicate value as a bulleted list (use '*' for the bullets) with each predicate and value on a different line. Use the format: `<predicate>(<obj1>, <obj2>, ...): <truth_value>` given the provided predicates and arguments exactly. Ensure there is a period ('.') after every list item. Do not output any text except the names and truth values of predicates.
+You are a vision system for a robot. Your job is to output the values of the following predicates based on the provided visual scene. For each predicate, output True, False, or Unknown if the relevant objects are not in the scene or the value of the predicate simply cannot be determined. Output each predicate value as a bulleted list (use '*' for the bullets) with each predicate and value on a different line. Use the format: `<predicate>(<obj1>, <obj2>, ...): <truth_value>` given the provided predicates and arguments exactly. Ensure there is a period ('.') after every list item. Do not output any text except the names and truth values of predicates. 
+
+Before output predicate values, give a description of the scene and explain why you think the predicate values are what they are.
+
+Note that the coffee machine is black and the table is white.
+
 Predicates:

predicators/envs/pybullet_coffee.py

@@ -286,21 +286,23 @@ def predicates(self) -> Set[Predicate]:
     def agent_goal_predicates(self) -> Set[Predicate]:
         return self.goal_predicates
 
-    # def get_vlm_debug_atom_strs(self,
-    #                             train_tasks: List[Task]) -> List[List[str]]:
-    #     # Convert the default value from List[List[str]] to List[str] to match
-    #     # the other entries we'll put into the dictionary.
-    #     default = [a[0] for a in super().get_vlm_debug_atom_strs(train_tasks)]
-    #     atom_strs_by_task_type = {
-    #         "more_stacks": ["Cooked(patty1)"],
-    #         "fatter_burger": ["Cooked(patty1)"],
-    #         "combo_burger":
-    #         ["Cooked(patty1)", "Cut(lettuce1)", "Whole(lettuce1)"]
-    #     }
-    #     atom_strs_by_task_type = defaultdict(lambda: default,
-    #                                          atom_strs_by_task_type)
-    #     atom_strs = atom_strs_by_task_type[CFG.burger_no_move_task_type]
-    #     return [[a] for a in atom_strs]
+    def get_vlm_debug_atom_strs(self,
+                                train_tasks: List[Task]) -> List[List[str]]:
+        # # Convert the default value from List[List[str]] to List[str] to match
+        # # the other entries we'll put into the dictionary.
+        # default = [a[0] for a in super().get_vlm_debug_atom_strs(train_tasks)]
+        # atom_strs_by_task_type = {
+        #     "more_stacks": ["Cooked(patty1)"],
+        #     "fatter_burger": ["Cooked(patty1)"],
+        #     "combo_burger":
+        #     ["Cooked(patty1)", "Cut(lettuce1)", "Whole(lettuce1)"]
+        # }
+        # atom_strs_by_task_type = defaultdict(lambda: default,
+        #                                      atom_strs_by_task_type)
+        # atom_strs = atom_strs_by_task_type[CFG.burger_no_move_task_type]
+        # return [[a] for a in atom_strs]
+        return [["JugInMachine(jug, coffee_machine)"], ["Holding(robby, jug)"], ["HandEmpty(robby)"]]
+        
 
     @property
     def oracle_proposed_predicates(self) -> Set[Predicate]:

predicators/settings.py

@@ -333,9 +333,12 @@ class GlobalSettings:
     # pybullet _coffee env parameters
     pybullet_coffee_update_camera = True
     pybullet_coffee_camera_distance = 0.8
-    pybullet_coffee_camera_yaw = -60
-    pybullet_coffee_camera_pitch = -30
-    pybullet_coffee_camera_target = (0.75, 1.25, 0.42)
+    # pybullet_coffee_camera_yaw = -60
+    # pybullet_coffee_camera_pitch = -30
+    # pybullet_coffee_camera_target = (0.75, 1.25, 0.42)
+    pybullet_coffee_camera_yaw = 40
+    pybullet_coffee_camera_pitch = -40
+    pybullet_coffee_camera_target = (1, 1.25, 0.42)
 
     # satellites env parameters
     satellites_num_sat_train = [2, 3]