some progress, but things still don't work well enough...

Author: NishanthJKumar

predicators/envs/spot_env.py

@@ -1180,18 +1180,23 @@ def _actively_construct_initial_object_views(
 
         # Capture images from the current position and run detection
         # without moving the robot.
-        rgbds = capture_images(self._robot, self._localizer)
-        detection_ids = set(self._detection_id_to_obj.keys())
-        detections, artifacts = detect_objects(
-            detection_ids, rgbds, self._allowed_regions)
-
-        if CFG.spot_render_perception_outputs:
-            outdir = Path(CFG.spot_perception_outdir)
-            time_str = time.strftime("%Y%m%d-%H%M%S")
-            detections_outfile = outdir / f"detections_{time_str}.png"
-            no_detections_outfile = outdir / f"no_detections_{time_str}.png"
-            visualize_all_artifacts(artifacts, detections_outfile,
-                                    no_detections_outfile)
+        # rgbds = capture_images(self._robot, self._localizer)
+        # detection_ids = set(self._detection_id_to_obj.keys())
+        # detections, artifacts = detect_objects(
+        #     detection_ids, rgbds, self._allowed_regions)
+
+        # if CFG.spot_render_perception_outputs:
+        #     outdir = Path(CFG.spot_perception_outdir)
+        #     time_str = time.strftime("%Y%m%d-%H%M%S")
+        #     detections_outfile = outdir / f"detections_{time_str}.png"
+        #     no_detections_outfile = outdir / f"no_detections_{time_str}.png"
+        #     visualize_all_artifacts(artifacts, detections_outfile,
+        #                             no_detections_outfile)
+
+        # Detection is currently disabled (commented out above), so default
+        # to empty so the fallback to known poses below still works.
+        detections: Dict[ObjectDetectionID, math_helpers.SE3Pose] = {}
+        artifacts: Dict[str, Any] = {}
 
         obj_to_se3_pose = {
             self._detection_id_to_obj[det_id]: val

predicators/ground_truth_models/spot_env/options.py

@@ -522,7 +522,9 @@ def _fn() -> None:
         time.sleep(0.5)  # Wait for the hand image to settle
         while True:
             robot, localizer, lease_client = get_robot()
-            rgbds = capture_images(robot, localizer, relocalize=True)
+            rgbds = capture_images(robot, localizer,
+                                   camera_names=["hand_color_image"],
+                                   relocalize=True)
             pick_obj_id = get_detection_id_for_object(objects[target_obj_idx])
             _, artifacts = detect_objects([pick_obj_id], rgbds)
             try:

predicators/pretrained_model_interface.py

@@ -8,6 +8,7 @@
 import base64
 import logging
 import os
+import signal
 from io import BytesIO
 from typing import Collection, Dict, List, Optional, Union
 
@@ -198,6 +199,26 @@ def call_openai_api(self,
         return completion.choices[0].message.content
 
 
+class _GeminiTimeout:
+    """Context manager that raises TimeoutError after a deadline (seconds)."""
+
+    def __init__(self, seconds: int = 30) -> None:
+        self._seconds = seconds
+
+    def _handler(self, signum, frame):
+        raise TimeoutError(
+            f"Gemini API call timed out after {self._seconds}s")
+
+    def __enter__(self):
+        self._old_handler = signal.signal(signal.SIGALRM, self._handler)
+        signal.alarm(self._seconds)
+        return self
+
+    def __exit__(self, *args):
+        signal.alarm(0)
+        signal.signal(signal.SIGALRM, self._old_handler)
+
+
 class GoogleGeminiModel():
     """Common interface and methods for all Gemini-based models.
 
@@ -260,8 +281,8 @@ class GoogleGeminiLLM(LargeLanguageModel, GoogleGeminiModel):
     necessary API key to query the particular model name.
     """
 
-    # @retry(wait=wait_random_exponential(min=1, max=60),
-    #        stop=stop_after_attempt(10))
+    @retry(wait=wait_random_exponential(min=1, max=30),
+           stop=stop_after_attempt(5))
     def _sample_completions(
             self,
             prompt: str,
@@ -275,10 +296,11 @@ def _sample_completions(
         config = types.GenerateContentConfig(
             temperature=temperature,
             candidate_count=num_completions)
-        response = self._client.models.generate_content(
-            model=self._model_name,
-            contents=[prompt],
-            config=config)
+        with _GeminiTimeout(seconds=30):
+            response = self._client.models.generate_content(
+                model=self._model_name,
+                contents=[prompt],
+                config=config)
         return [response.text]
 
     def get_id(self) -> str:
@@ -292,8 +314,8 @@ class GoogleGeminiVLM(VisionLanguageModel, GoogleGeminiModel):
     necessary API key to query the particular model name.
     """
 
-    # @retry(wait=wait_random_exponential(min=1, max=60),
-    #        stop=stop_after_attempt(10))
+    @retry(wait=wait_random_exponential(min=1, max=30),
+           stop=stop_after_attempt(5))
     def _sample_completions(
             self,
             prompt: str,
@@ -306,12 +328,12 @@ def _sample_completions(
         assert imgs is not None
         config = types.GenerateContentConfig(
             temperature=temperature,
-            candidate_count=num_completions,
-            http_options=types.HttpOptions(timeout=15_000))
-        response = self._client.models.generate_content(
-            model=self._model_name,
-            contents=[prompt] + imgs,
-            config=config)
+            candidate_count=num_completions)
+        with _GeminiTimeout(seconds=30):
+            response = self._client.models.generate_content(
+                model=self._model_name,
+                contents=[prompt] + imgs,
+                config=config)
         return [response.text]
 
     def get_id(self) -> str:

predicators/spot_utils/graph_nav_maps/b45-621-cleanup/metadata.yaml

@@ -181,7 +181,7 @@ approach_angle_bounds:
   clear_plastic_container: [1.47, 1.67]
   recycling_bin: [-0.1, 0.1]
   pink_furry_eraser: [-1.67, -1.47]
-  green_and_blue_furry_eraser: [-1.67, -1.47]
+  green_and_blue_furry_eraser: [-0.1, 0.1]
   soda_can: [-1.67, -1.47]
 
 # Wiping skill params.