sort of works

Author: wmcclinton

2402.15025v2.pdf

@@ -0,0 +1,237 @@
+#######
+backward-forward
+lis_thesis_pickplace
+#######
+NSRT-move0:
+    Parameters: [?x0:object]
+    Preconditions: []
+    Add Effects: [infront2(?x0:object)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x0:object)
+NSRT-move1:
+    Parameters: [?x0:object]
+    Preconditions: [IsNotHolding0(?x0:object)]
+    Add Effects: [infront0(?x0:object)]
+    Delete Effects: []
+    Ignore Effects: [infront0, infront2]
+    Option Spec: move(?x0:object)
+NSRT-pick0:
+    Parameters: [?x0:ball]
+    Preconditions: [infront0(?x0:ball)]
+    Add Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball)]
+    Delete Effects: [IsNotHolding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: pick(?x0:ball)
+NSRT-place_on0:
+    Parameters: [?x0:ball, ?x1:table]
+    Preconditions: [holding0(?x0:ball)]
+    Add Effects: [OnTable(?x0:ball, ?x1:table)]
+    Delete Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: place_on(?x0:ball, ?x1:table)
+
+#######
+pnad_search
+lis_thesis_pickplace
+#######
+NSRT-move0:
+    Parameters: [?x0:object]
+    Preconditions: []
+    Add Effects: [infront2(?x0:object)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x0:object)
+NSRT-move1:
+    Parameters: [?x0:object]
+    Preconditions: [IsNotHolding0(?x0:object)]
+    Add Effects: [infront0(?x0:object)]
+    Delete Effects: []
+    Ignore Effects: [infront0, infront2]
+    Option Spec: move(?x0:object)
+NSRT-pick0:
+    Parameters: [?x0:ball]
+    Preconditions: [IsNotHolding0(?x0:ball), infront0(?x0:ball)]
+    Add Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball)]
+    Delete Effects: [IsNotHolding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: pick(?x0:ball)
+NSRT-place_on0:
+    Parameters: [?x0:ball, ?x1:table]
+    Preconditions: [IsPickedUp0(?x0:ball), holding0(?x0:ball), infront0(?x0:ball), infront2(?x1:table)]
+    Add Effects: [OnTable(?x0:ball, ?x1:table)]
+    Delete Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: place_on(?x0:ball, ?x1:table)
+
+#######
+cluster_and_intersect
+lis_thesis_pickplace
+#######
+NSRT-Op0:
+    Parameters: [?x0:ball, ?x1:robot, ?x2:ball, ?x3:table]
+    Preconditions: [IsNotHolding0(?x0:ball), IsNotHolding0(?x2:ball), infront0(?x0:ball), infront0(?x2:ball), infront2(?x3:table)]
+    Add Effects: [IsClearOfObjects0(?x1:robot), IsStationary0(?x1:robot)]
+    Delete Effects: [infront0(?x0:ball), infront2(?x3:table)]
+    Ignore Effects: []
+    Option Spec: move(?x2:ball)
+NSRT-Op1:
+    Parameters: [?x0:ball, ?x1:table]
+    Preconditions: [IsNotHolding0(?x0:ball), infront0(?x0:ball)]
+    Add Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball), infront2(?x1:table)]
+    Delete Effects: [IsNotHolding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: pick(?x0:ball)
+NSRT-Op2:
+    Parameters: [?x0:table]
+    Preconditions: [infront2(?x0:table)]
+    Add Effects: []
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x0:table)
+NSRT-Op3:
+    Parameters: [?x0:ball, ?x1:table]
+    Preconditions: [IsPickedUp0(?x0:ball), holding0(?x0:ball), infront0(?x0:ball), infront2(?x1:table)]
+    Add Effects: [IsNotHolding0(?x0:ball), IsOnTopOf0(?x0:ball, ?x1:table), OnTable(?x0:ball, ?x1:table)]
+    Delete Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: place_on(?x0:ball, ?x1:table)
+NSRT-Op4:
+    Parameters: [?x0:ball, ?x1:ball, ?x2:table]
+    Preconditions: [IsNotHolding0(?x0:ball), IsNotHolding0(?x1:ball), IsOnTopOf0(?x1:ball, ?x2:table), OnTable(?x1:ball, ?x2:table), infront0(?x1:ball), infront2(?x2:table)]
+    Add Effects: [infront0(?x0:ball)]
+    Delete Effects: [infront0(?x1:ball), infront2(?x2:table)]
+    Ignore Effects: []
+    Option Spec: move(?x0:ball)
+NSRT-Op5:
+    Parameters: [?x0:ball]
+    Preconditions: [IsNotHolding0(?x0:ball), infront0(?x0:ball)]
+    Add Effects: [IsPickedUp0(?x0:ball), holding0(?x0:ball)]
+    Delete Effects: [IsNotHolding0(?x0:ball)]
+    Ignore Effects: []
+    Option Spec: pick(?x0:ball)
+NSRT-Op6:
+    Parameters: [?x0:ball, ?x1:table]
+    Preconditions: [IsNotHolding0(?x0:ball), IsOnTopOf0(?x0:ball, ?x1:table), OnTable(?x0:ball, ?x1:table)]
+    Add Effects: [infront0(?x0:ball), infront2(?x1:table)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x1:table)
+
+
+#######
+cluster_and_intersect
+lis_thesis_sweep
+#######
+
+NSRT-Op0:
+    Parameters: [?x0:robot, ?x1:tool]
+    Preconditions: []
+    Add Effects: [on1(?x1:tool, ?x0:robot)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x1:tool)
+NSRT-Op1:
+    Parameters: [?x0:surface, ?x1:robot, ?x2:tool]
+    Preconditions: [on0(?x2:tool, ?x0:surface), on1(?x2:tool, ?x1:robot)]
+    Add Effects: [holding0(?x2:tool)]
+    Delete Effects: [on0(?x2:tool, ?x0:surface), on1(?x2:tool, ?x1:robot)]
+    Ignore Effects: []
+    Option Spec: pick(?x2:tool)
+NSRT-Op2:
+    Parameters: [?x0:surface]
+    Preconditions: []
+    Add Effects: []
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x0:surface)
+NSRT-Op3:
+    Parameters: [?x0:container, ?x1:toy, ?x2:toy, ?x3:surface]
+    Preconditions: []
+    Add Effects: [InContainer(?x1:toy, ?x0:container), InContainer(?x2:toy, ?x0:container), NoObjectsOnTop0(?x3:surface), TableSwept(?x3:surface), in0(?x1:toy, ?x0:container), in0(?x2:toy, ?x0:container)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: sweep(?x3:surface)
+NSRT-Op4:
+    Parameters: [?x0:surface, ?x1:tool]
+    Preconditions: [holding0(?x1:tool)]
+    Add Effects: [on0(?x1:tool, ?x0:surface)]
+    Delete Effects: [holding0(?x1:tool)]
+    Ignore Effects: []
+    Option Spec: place_on(?x1:tool, ?x0:surface)
+NSRT-Op5:
+    Parameters: [?x0:tool, ?x1:toy]
+    Preconditions: []
+    Add Effects: [not_infront2(?x0:tool)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x1:toy)
+NSRT-Op6:
+    Parameters: [?x0:toy]
+    Preconditions: []
+    Add Effects: [holding1(?x0:toy)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: pick(?x0:toy)
+NSRT-Op7:
+    Parameters: [?x0:container]
+    Preconditions: []
+    Add Effects: []
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x0:container)
+NSRT-Op8:
+    Parameters: [?x0:container, ?x1:toy]
+    Preconditions: [holding1(?x1:toy)]
+    Add Effects: [InContainer(?x1:toy, ?x0:container), in0(?x1:toy, ?x0:container)]
+    Delete Effects: [holding1(?x1:toy)]
+    Ignore Effects: []
+    Option Spec: place_in(?x1:toy, ?x0:container)
+
+#######
+backward-forward
+lis_thesis_sweep
+#######
+
+NSRT-move0:
+    Parameters: [?x0:object]
+    Preconditions: []
+    Add Effects: []
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: move(?x0:object)
+NSRT-pick0:
+    Parameters: [?x0:object]
+    Preconditions: []
+    Add Effects: [holding1(?x0:object)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: pick(?x0:object)
+NSRT-pick1:
+    Parameters: [?x0:object]
+    Preconditions: []
+    Add Effects: []
+    Delete Effects: []
+    Ignore Effects: [on0, on1]
+    Option Spec: pick(?x0:object)
+NSRT-place_in0:
+    Parameters: [?x1:container, ?x0:toy]
+    Preconditions: [holding1(?x0:toy)]
+    Add Effects: [InContainer(?x0:toy, ?x1:container)]
+    Delete Effects: [holding1(?x0:toy)]
+    Ignore Effects: []
+    Option Spec: place_in(?x0:toy, ?x1:container)
+NSRT-place_on0:
+    Parameters: [?x0:object, ?x1:surface]
+    Preconditions: []
+    Add Effects: []
+    Delete Effects: [holding0(?x0:object)]
+    Ignore Effects: []
+    Option Spec: place_on(?x0:object, ?x1:surface)
+NSRT-sweep0:
+    Parameters: [?x0:surface, ?x1:container, ?x2:toy, ?x3:toy]
+    Preconditions: []
+    Add Effects: [InContainer(?x2:toy, ?x1:container), InContainer(?x3:toy, ?x1:container)]
+    Delete Effects: []
+    Ignore Effects: []
+    Option Spec: sweep(?x0:surface)

learned_operators.txt

@@ -1291,7 +1291,7 @@ def _select_predicates_by_clustering(
             # that are add effects in those clusters, letting pnad_search
             # downstream handle chainability.
 
-            assert CFG.segmenter == "option_changes"
+            assert CFG.segmenter == "option_changes" or CFG.segmenter == "every_step"
             segments = [
                 seg for ll_traj, atom_seq in atom_dataset for seg in
                 segment_trajectory(ll_traj, initial_predicates, atom_seq)
@@ -1321,10 +1321,13 @@ def _select_predicates_by_clustering(
                     # To cluster on type, there must be types. That is, there
                     # must be add effects in the segment and the object
                     # arguments for at least one add effect must be nonempty.
-                    assert len(types_in_effects) > 0 and len(
-                        set.union(*types_in_effects)) > 0
-                    types = tuple(sorted(list(set.union(*types_in_effects))))
-                    types_to_segments.setdefault(types, []).append(seg)
+                    try:
+                        assert len(types_in_effects) > 0 and len(
+                                set.union(*types_in_effects)) > 0
+                        types = tuple(sorted(list(set.union(*types_in_effects))))
+                        types_to_segments.setdefault(types, []).append(seg)
+                    except AssertionError:
+                        import ipdb; ipdb.set_trace()
                 logging.info(
                     f"STEP 2: generated {len(types_to_segments.keys())} "
                     f"type-based clusters for cluster {i+1} from STEP 1 "

predicators/approaches/grammar_search_invention_approach.py

@@ -59,6 +59,10 @@ def _generate_prompt_for_atom_proposals(
         str(obj.name) for obj in sorted(traj.objects)
         if obj.name != 'dummy_goal_obj'
     ])
+    # TODO
+    prompt += "\nBe sure to include reachable, infront, holding, handempty in invented predicate set.\n"
+    prompt += "\nAlso include atleast three invented predicates that change from True to False directly from a consequence of this action and do this for each action.\n"
+    prompt += "\nBe sure to propose over 100 predicates.\n"
 
     if CFG.grammar_search_vlm_atom_proposal_prompt_type == "naive_each_step":
         i = 0
@@ -87,6 +91,7 @@ def _generate_prompt_for_atom_proposals(
     else:  # pragma: no cover.
         raise ValueError("Unknown VLM prompting option " +
                          f"{CFG.grammar_search_vlm_atom_proposal_prompt_type}")
+    
     return ret_list
 
 

predicators/datasets/generate_atom_trajs_with_vlm.py

@@ -0,0 +1,15 @@
+You are a vision system for a robot. You are provided with two images corresponding to the states before and after a particular skill is executed. You are given a list of predicates below, and you are given the values of these predicates in the image before the skill is executed. Your job is to output the values of the following predicates in the image after the skill is executed. Pay careful attention to the visual changes between the two images to figure out which predicates change and which predicates do not change. For the predicates that change, list these separately at the end of your response. Note that in some scenes, there might be no changes. First, output a description of what changes you expect to happen based on the skill that was just run, explicitly noting the skill that was run. Second, output a description of what visual changes you see happen between the before and after images, looking specifically at the objects involved in the skill's arguments, noting what objects these are. Next, output each predicate value in the after image as a bulleted list with each predicate and value on a different line. For each predicate value *that changes*, provide an explanation as to why you labelled this predicate as having this particular value (no explanation is necessary if you believe the value has not changed). Lastly, some predicates may not be visually apparent in the images - be sure to carry their same truth valuses over to the next state. Atleast one predicate should change to become True each step. Use the format: <predicate>: <truth_value>. <explanation>.
+
+Your response should have three sections. Here is an outline of what your response should look like:
+[START OUTLINE]
+# Expected changes based on the executed skill
+[insert your analysis on the expected changes you will see based on the skill that was executed]
+
+# Visual changes observed between the images
+[insert your analysis on the visual changes observed between the images]
+
+# Predicate values in the after image
+[insert your bulleted list of `* <predicate>: <truth value>. <explanation>`]
+[END OUTLINE]
+
+Predicates:

predicators/datasets/vlm_input_data_prompts/atom_labelling/img_option_diffs_label_history_thesis.txt

@@ -0,0 +1,7 @@
+You are a robotic vision system whose job is to output a structured set of predicates useful for describing important concepts in the following demonstration of a task. You will be provided with a list of actions used during the task, as well as images of states before and after every action execution. Please provide predicates in terms of the following objects: {objs}. For each predicate, output it in the following format: predicate_name(obj1, obj2, obj3...). 
+Start by providing an overview description of the demonstration and the changes you observe in it, as well as a list of what concepts you think are relevant. 
+For each action, generate both "dynamic" predicates (i.e., those that you expect/see to change value before and after the action invocation) and "static" predicates (i.e., those that you expect/see stay constant, but are still important preconditions of the action and need to hold for the action to be correctly executed).
+Finally, generate any additional predicates that might be important to describing the overall demonstration. Try to provide any synonyms and antonyms for the most important static and dynamic predicates: the more ways to describe a concept, the better!
+Think very carefully about what relations/conditions (in the form of predicates) might be important for each of the particular actions to be executed.
+When generating these predicate names, try to be as unambiguous as possible. For instance, instead of generating a predicate `Free(table)` to express the concept that there are no objects/clutter on the table, instead generate `NoObjectsOnTop(table)`, since this is far more precise.
+Be careful to only use the objects listed above. DO NOT introduce any additional objects or concepts; if you want to do this, just change the predicate name. For instance, if you want to express the concept `IsColor(crayon, red)`, but the allowed object set only contains `crayon`, then this would be invalid (since there is no `red` object); instead you should output `IsColorRed(crayon)`.

predicators/datasets/vlm_input_data_prompts/atom_proposal/options_labels_whole_traj_diverse_spot_thesis.txt

@@ -596,8 +596,8 @@ def _get_tasks(self, num: int,
         wooden_table_obj = Object("wooden_table", self._surface_type)
         floor_obj = Object("floor", self._surface_type)
         toy_caterpillar_obj = Object("toy_caterpillar", self._toy_type)
-        toy_elephant_obj = Object("toy_elephant", self._toy_type)
-        toy_frog_obj = Object("toy_frog", self._toy_type)
+        toy_panda_bear_obj = Object("toy_panda_bear", self._toy_type)
+        toy_brown_bear_obj = Object("toy_brown_bear", self._toy_type)
         plastic_bin_obj = Object("plastic_bin", self._container_type)
 
         ret_tasks = []
@@ -608,18 +608,18 @@ def _get_tasks(self, num: int,
                 wooden_table_obj: np.array([]),
                 floor_obj: np.array([]),
                 toy_caterpillar_obj: np.array([]),
-                toy_elephant_obj: np.array([]),
-                toy_frog_obj: np.array([]),
+                toy_panda_bear_obj: np.array([]),
+                toy_brown_bear_obj: np.array([]),
                 plastic_bin_obj: np.array([]),
             }
             # Goal: all three toys in the plastic bin
             goal = {
                 GroundAtom(self._InContainer,
                            [toy_caterpillar_obj, plastic_bin_obj]),
                 GroundAtom(self._InContainer,
-                           [toy_elephant_obj, plastic_bin_obj]),
+                           [toy_panda_bear_obj, plastic_bin_obj]),
                 GroundAtom(self._InContainer,
-                           [toy_frog_obj, plastic_bin_obj]),
+                           [toy_brown_bear_obj, plastic_bin_obj]),
             }
             ret_tasks.append(EnvironmentTask(State(init_state_dict), goal))
         return ret_tasks
@@ -631,11 +631,11 @@ def get_vlm_debug_atom_strs(self,
         atom_strs = set([
             "holding(squeegee)",
             "holding(toy_caterpillar)",
-            "holding(toy_elephant)",
-            "holding(toy_frog)",
+            "holding(toy_panda_bear)",
+            "holding(toy_brown_bear)",
             "InContainer(toy_caterpillar, plastic_bin)",
-            "InContainer(toy_elephant, plastic_bin)",
-            "InContainer(toy_frog, plastic_bin)",
+            "InContainer(toy_panda_bear, plastic_bin)",
+            "InContainer(toy_brown_bear, plastic_bin)",
             "TableSwept(wooden_table)",
             "OnSurface(squeegee, floor)",
             "OnSurface(squeegee, wooden_table)",

predicators/envs/vlm_envs.py

@@ -138,6 +138,13 @@ def learn_nsrts_from_data(
         logging.info(nsrt)
     logging.info("")
 
+    # Save learned NSRTs to a text file.
+    save_path = f"saved_results/{CFG.strips_learner}_{CFG.env}_learned_nsrts.txt"
+    with open(save_path, "w") as f:
+        for nsrt in nsrts:
+            f.write(str(nsrt) + "\n")
+    logging.info("Saved learned NSRTs to %s", save_path)
+
     return set(nsrts), segmented_trajs, seg_to_nsrt