chore: sync upstream browser-use target (#183)

Author: haasonsaas

README.md

@@ -8,7 +8,7 @@ translation of the Python internals.
 Current frozen upstream target:
 
 ```text
-browser-use/browser-use@157779338afdcc03023010ec3c24ad63d820453c
+browser-use/browser-use@834269609082d187ca0250de2c06d93799dac92d
 ```
 
 The detailed support matrix lives in [docs/RELEASE.md](docs/RELEASE.md). The

crates/browser-use-core/src/lib.rs

@@ -109,7 +109,7 @@ pub(crate) use urls::{
 pub(crate) use usage::TokenUsageTracker;
 
 /// Version of the upstream browser-use source that this crate initially targets.
-pub const INITIAL_UPSTREAM_COMMIT: &str = "157779338afdcc03023010ec3c24ad63d820453c";
+pub const INITIAL_UPSTREAM_COMMIT: &str = "834269609082d187ca0250de2c06d93799dac92d";
 
 #[cfg(test)]
 mod tests;

crates/browser-use-core/src/prompt.rs

@@ -34,6 +34,8 @@ use browser_use_llm::{ChatMessage, ChatRequest, ContentPart, ImageDetailLevel, M
 use serde_json::Value;
 use std::collections::{BTreeMap, BTreeSet};
 
+const PLACEHOLDER_4PX_SCREENSHOT: &str = "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII=";
+
 mod history;
 mod schema;
 
@@ -279,22 +281,22 @@ pub fn build_step_request_with_file_system(
     let agent_history = render_previous_results(history, settings.max_history_items);
     let page_stats = render_page_stats(state);
     let agent_state =
-        render_agent_state_description(task, &page_stats, history, state, settings, file_system);
+        render_agent_state_description(&page_stats, history, state, settings, file_system);
     let read_state = render_read_state_description(history)
         .map(|description| format!("\n<read_state>\n{description}\n</read_state>\n"))
         .unwrap_or_default();
+    let step_meta = render_step_meta_description(history);
     let sensitive_values = collect_sensitive_data_values(&settings.sensitive_data);
     let user_text = redact_sensitive_string(
         &format!(
-            "<agent_history>\n{agent_history}\n</agent_history>\n\n<agent_state>\n{agent_state}\n</agent_state>\n<browser_state>\n{state_json}\n</browser_state>{read_state}"
+            "{}<agent_history>\n{agent_history}\n</agent_history>\n\n<agent_state>\n{agent_state}\n</agent_state>\n<browser_state>\n{state_json}\n</browser_state>{read_state}{step_meta}",
+            render_user_request_description(task)
         ),
         &sensitive_values,
     );
     let mut user_content = vec![ContentPart::Text { text: user_text }];
     user_content.extend(settings.sample_images.iter().cloned());
-    if settings.use_vision.accepts_prompt_image()
-        && let Some(screenshot) = state.screenshot.as_deref()
-    {
+    if let Some(screenshot) = prompt_visible_screenshot(state, settings) {
         user_content.push(ContentPart::ImageUrl {
             image_url: prompt_screenshot_data_url(screenshot, settings.llm_screenshot_size),
             detail: Some(settings.vision_detail_level),
@@ -559,6 +561,39 @@ fn render_judge_trajectory(history: &AgentHistory) -> String {
         .join("\n\n")
 }
 
+fn render_user_request_description(task: &str) -> String {
+    format!("<user_request>\n{task}\n</user_request>\n\n")
+}
+
+fn render_step_meta_description(history: &AgentHistory) -> String {
+    let next_step = latest_history_step_number(history).unwrap_or(0) + 1;
+    format!(
+        "<step_info>Step {next_step}\nToday:{}</step_info>\n",
+        utc_date_string(now_seconds() as i64)
+    )
+}
+
+fn utc_date_string(unix_seconds: i64) -> String {
+    let days = unix_seconds.div_euclid(86_400);
+    let (year, month, day) = civil_from_days(days);
+    format!("{year:04}-{month:02}-{day:02}")
+}
+
+fn civil_from_days(days_since_unix_epoch: i64) -> (i32, u32, u32) {
+    // Howard Hinnant's civil-from-days algorithm, using a 1970-01-01 epoch.
+    let z = days_since_unix_epoch + 719_468;
+    let era = if z >= 0 { z } else { z - 146_096 }.div_euclid(146_097);
+    let doe = z - era * 146_097;
+    let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096).div_euclid(365);
+    let mut year = yoe + era * 400;
+    let doy = doe - (365 * yoe + yoe / 4 - yoe / 100);
+    let mp = (5 * doy + 2).div_euclid(153);
+    let day = doy - (153 * mp + 2).div_euclid(5) + 1;
+    let month = mp + if mp < 10 { 3 } else { -9 };
+    year += if month <= 2 { 1 } else { 0 };
+    (year as i32, month as u32, day as u32)
+}
+
 fn truncate_judge_text(text: &str) -> String {
     const MAX_CHARS: usize = 40_000;
     if text.chars().count() <= MAX_CHARS {
@@ -649,6 +684,24 @@ fn screenshot_base64_payload(screenshot: &str) -> &str {
     }
 }
 
+fn prompt_visible_screenshot<'a>(
+    state: &'a BrowserStateSummary,
+    settings: &AgentSettings,
+) -> Option<&'a str> {
+    if !settings.use_vision.accepts_prompt_image() || is_new_tab_page(&state.url) {
+        return None;
+    }
+    state
+        .screenshot
+        .as_deref()
+        .filter(|screenshot| !screenshot.trim().is_empty())
+        .filter(|screenshot| !is_placeholder_4px_screenshot(screenshot))
+}
+
+fn is_placeholder_4px_screenshot(screenshot: &str) -> bool {
+    screenshot_base64_payload(screenshot).trim() == PLACEHOLDER_4PX_SCREENSHOT
+}
+
 fn append_latest_action_result_images(
     content: &mut Vec<ContentPart>,
     history: &AgentHistory,
@@ -774,14 +827,13 @@ fn fallback_page_stats(state: &BrowserStateSummary) -> browser_use_dom::DomPageS
 }
 
 fn render_agent_state_description(
-    task: &str,
     page_stats: &str,
     history: &AgentHistory,
     state: &BrowserStateSummary,
     settings: &AgentSettings,
     file_system: Option<&ManagedFileSystem>,
 ) -> String {
-    let mut description = format!("Task:\n{task}\n\nPage stats:\n{page_stats}");
+    let mut description = format!("Page stats:\n{page_stats}");
     if let Some(file_system) = file_system {
         let todo_contents = file_system.get_todo_contents();
         let todo_contents = if todo_contents.is_empty() {

crates/browser-use-core/src/tests.rs

@@ -45,7 +45,7 @@ impl Drop for CurrentDirGuard {
 fn target_commit_is_pinned() {
     assert_eq!(
         INITIAL_UPSTREAM_COMMIT,
-        "157779338afdcc03023010ec3c24ad63d820453c"
+        "834269609082d187ca0250de2c06d93799dac92d"
     );
 }
 
@@ -4551,6 +4551,11 @@ fn request_text(request: &ChatRequest) -> String {
         .join("\n")
 }
 
+fn text_pos(text: &str, needle: &str) -> usize {
+    text.find(needle)
+        .unwrap_or_else(|| panic!("{needle} not found in text"))
+}
+
 #[async_trait]
 impl ChatModel for QueueModel {
     fn provider(&self) -> &str {
@@ -4642,6 +4647,54 @@ async fn agent_history_records_token_usage_summary_and_costs() {
     assert!((usage.total_cost - 0.0000536).abs() < 0.0000000001);
 }
 
+#[tokio::test]
+async fn agent_cost_pricing_maps_bu_latest_to_bu_2_0_like_upstream() {
+    let done_output = serde_json::json!({
+        "current_state": {
+            "thinking": "done"
+        },
+        "action": [
+            {
+                "done": {
+                    "text": "finished",
+                    "success": true
+                }
+            }
+        ]
+    });
+    let usage = ChatUsage {
+        prompt_tokens: 100,
+        prompt_cached_tokens: Some(40),
+        prompt_cache_creation_tokens: None,
+        prompt_image_tokens: None,
+        completion_tokens: 20,
+        total_tokens: 120,
+    };
+    let settings = AgentSettings {
+        calculate_cost: true,
+        use_judge: false,
+        ..AgentSettings::default()
+    };
+    let mut agent = Agent::with_settings(
+        "finish",
+        settings,
+        QueueModel::with_model_outputs_and_usages(
+            "bu-latest",
+            vec![done_output],
+            vec![Some(usage)],
+        ),
+        MockSession::new(),
+    );
+
+    let history = agent.run(1).await.expect("run");
+    let usage = history.usage.as_ref().expect("usage summary");
+
+    assert!((usage.total_prompt_cost - 0.0000384).abs() < 0.0000000001);
+    assert!((usage.total_prompt_cached_cost - 0.0000024).abs() < 0.0000000001);
+    assert!((usage.total_completion_cost - 0.00007).abs() < 0.0000000001);
+    assert!((usage.total_cost - 0.0001108).abs() < 0.0000000001);
+}
+
 #[tokio::test]
 async fn agent_extract_action_uses_llm_result_sections() {
     let agent_output = serde_json::json!({
@@ -5873,6 +5926,7 @@ async fn agent_step_saves_conversation_transcript() {
         ]
     });
     let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
     state.screenshot = Some("abc123".to_owned());
     let settings = AgentSettings {
         save_conversation_path: Some(transcript_dir.display().to_string()),
@@ -8111,6 +8165,90 @@ fn step_request_includes_previous_results() {
     assert!(request_text.contains("Avoid repeating the same action sequence"));
 }
 
+#[test]
+fn step_request_places_user_request_before_history_and_step_meta_at_suffix() {
+    let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
+    let history = AgentHistory {
+        items: vec![AgentHistoryItem {
+            model_output: None,
+            result: vec![ActionResult::extracted("first step")],
+            state: blank_state(),
+            metadata: Some(StepMetadata {
+                step_start_time: 1.0,
+                step_end_time: 2.0,
+                step_number: 3,
+                step_interval: None,
+            }),
+        }],
+        ..AgentHistory::default()
+    };
+
+    let request = build_step_request(
+        "Do the cacheable thing",
+        &state,
+        &history,
+        &AgentSettings::default(),
+    )
+    .expect("step request");
+    let user_text = request_text(&request);
+
+    assert!(text_pos(&user_text, "<user_request>") < text_pos(&user_text, "<agent_history>"));
+    assert!(text_pos(&user_text, "</agent_history>") < text_pos(&user_text, "<agent_state>"));
+    assert!(text_pos(&user_text, "</agent_state>") < text_pos(&user_text, "<browser_state>"));
+    assert!(text_pos(&user_text, "</browser_state>") < text_pos(&user_text, "<step_info>"));
+    assert!(user_text.ends_with("</step_info>\n"));
+    assert!(user_text.contains("<step_info>Step 4\nToday:"));
+
+    let agent_state =
+        &user_text[text_pos(&user_text, "<agent_state>")..text_pos(&user_text, "</agent_state>")];
+    assert!(!agent_state.contains("<user_request>"));
+    assert!(!agent_state.contains("<step_info>"));
+    assert!(!agent_state.contains("Do the cacheable thing"));
+}
+
+#[test]
+fn step_request_prefix_before_step_info_is_stable_across_steps() {
+    let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
+    let history_with_step = |step_number| AgentHistory {
+        items: vec![AgentHistoryItem {
+            model_output: None,
+            result: vec![ActionResult::extracted("result")],
+            state: blank_state(),
+            metadata: Some(StepMetadata {
+                step_start_time: 1.0,
+                step_end_time: 2.0,
+                step_number,
+                step_interval: None,
+            }),
+        }],
+        ..AgentHistory::default()
+    };
+    let first = request_text(
+        &build_step_request(
+            "Keep the stable prefix",
+            &state,
+            &history_with_step(3),
+            &AgentSettings::default(),
+        )
+        .expect("first request"),
+    );
+    let second = request_text(
+        &build_step_request(
+            "Keep the stable prefix",
+            &state,
+            &history_with_step(4),
+            &AgentSettings::default(),
+        )
+        .expect("second request"),
+    );
+
+    let prefix_end = text_pos(&first, "<step_info>");
+    assert_eq!(&first[..prefix_end], &second[..prefix_end]);
+    assert_ne!(&first[prefix_end..], &second[prefix_end..]);
+}
+
 #[test]
 fn step_request_includes_loading_page_stats_hint_like_upstream() {
     let mut state = blank_state();
@@ -8607,6 +8745,7 @@ fn step_request_redacts_sensitive_values_from_state_and_history() {
 #[test]
 fn step_request_attaches_screenshot_as_image_part() {
     let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
     state.screenshot = Some("abc123".to_owned());
 
     let request = build_step_request(
@@ -8639,6 +8778,7 @@ fn step_request_attaches_screenshot_as_image_part() {
 fn step_request_resizes_screenshot_for_llm_prompt_only() {
     let original_screenshot = test_png_base64(240, 160);
     let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
     state.screenshot = Some(original_screenshot.clone());
     let settings = AgentSettings {
         llm_screenshot_size: Some(LlmScreenshotSize::new(120, 100).expect("valid size")),
@@ -8669,6 +8809,54 @@ fn step_request_resizes_screenshot_for_llm_prompt_only() {
     );
 }
 
+#[test]
+fn step_request_omits_screenshot_for_new_tab_pages() {
+    let mut state = blank_state();
+    state.url = "chrome://new-tab-page/".to_owned();
+    state.screenshot = Some("abc123".to_owned());
+
+    let request = build_step_request(
+        "inspect new tab",
+        &state,
+        &AgentHistory::default(),
+        &AgentSettings::default(),
+    )
+    .expect("step request");
+    let user_message = request
+        .messages
+        .iter()
+        .find(|message| message.role == MessageRole::User)
+        .expect("user message");
+
+    assert_eq!(user_message.content.len(), 1);
+    assert!(!request_text(&request).contains("abc123"));
+}
+
+#[test]
+fn step_request_omits_placeholder_4px_screenshot() {
+    let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
+    state.screenshot = Some(
+        "iVBORw0KGgoAAAANSUhEUgAAAAQAAAAECAIAAAAmkwkpAAAAFElEQVR4nGP8//8/AwwwMSAB3BwAlm4DBfIlvvkAAAAASUVORK5CYII="
+            .to_owned(),
+    );
+
+    let request = build_step_request(
+        "inspect placeholder",
+        &state,
+        &AgentHistory::default(),
+        &AgentSettings::default(),
+    )
+    .expect("step request");
+    let user_message = request
+        .messages
+        .iter()
+        .find(|message| message.role == MessageRole::User)
+        .expect("user message");
+
+    assert_eq!(user_message.content.len(), 1);
+}
+
 #[test]
 fn step_request_attaches_latest_action_result_images_as_image_parts() {
     let history = AgentHistory {
@@ -8748,6 +8936,7 @@ fn step_request_attaches_latest_action_result_images_as_image_parts() {
 #[test]
 fn step_request_inserts_sample_images_before_runtime_images() {
     let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
     state.screenshot = Some("screen-data".to_owned());
     let history = AgentHistory {
         items: vec![AgentHistoryItem {
@@ -8920,6 +9109,7 @@ fn step_request_omits_screenshot_when_vision_disabled() {
 #[test]
 fn step_request_includes_screenshot_when_auto_vision_state_has_one() {
     let mut state = blank_state();
+    state.url = "https://example.test/app".to_owned();
     state.screenshot = Some("abc123".to_owned());
     let settings = AgentSettings {
         use_vision: VisionMode::Auto,