Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions cli/src/native/actions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7247,6 +7247,9 @@ async fn handle_type(cmd: &Value, state: &mut DaemonState) -> Result<Value, Stri
&state.iframe_sessions,
)
.await?;
// Let the page run the frame and macrotask the keystrokes queued
// before returning, so the next command cannot land before them.
interaction::wait_for_settle(&mgr.client, &session_id).await;
Ok(json!({ "typed": text }))
}

Expand All @@ -7262,6 +7265,9 @@ async fn handle_press(cmd: &Value, state: &mut DaemonState) -> Result<Value, Str
let (actual_key, modifiers) = parse_key_chord(key);

interaction::press_key_with_modifiers(&mgr.client, &session_id, &actual_key, modifiers).await?;
// Let the page run the frame and macrotask the key events queued
// before returning, so the next command cannot land before them.
interaction::wait_for_settle(&mgr.client, &session_id).await;
Ok(json!({ "pressed": key }))
}

Expand Down Expand Up @@ -8333,6 +8339,9 @@ async fn handle_keyboard(cmd: &Value, state: &DaemonState) -> Result<Value, Stri
.ok_or("Missing 'text' parameter")?;
interaction::type_text_into_active_context(&mgr.client, &session_id, text, None)
.await?;
// Let the page run the frame and macrotask the keystrokes queued
// before returning, so the next command cannot land before them.
interaction::wait_for_settle(&mgr.client, &session_id).await;
return Ok(json!({ "typed": text }));
}
Some("insertText") => {
Expand All @@ -8347,6 +8356,9 @@ async fn handle_keyboard(cmd: &Value, state: &DaemonState) -> Result<Value, Stri
Some(&session_id),
)
.await?;
// Let the page run the frame and macrotask the insertion queued
// before returning, so the next command cannot land before them.
interaction::wait_for_settle(&mgr.client, &session_id).await;
return Ok(json!({ "inserted": true }));
}
_ => {}
Expand Down Expand Up @@ -15089,6 +15101,9 @@ async fn handle_inserttext(cmd: &Value, state: &DaemonState) -> Result<Value, St
Some(&session_id),
)
.await?;
// Let the page run the frame and macrotask the insertion queued
// before returning, so the next command cannot land before them.
interaction::wait_for_settle(&mgr.client, &session_id).await;
Ok(json!({ "inserted": true }))
}

Expand Down
212 changes: 212 additions & 0 deletions cli/src/native/e2e_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3906,6 +3906,218 @@ async fn e2e_press_dispatches_single_keystroke() {
assert_success(&resp);
}

/// `press` must not return until the page has run the frame and macrotask a
/// keydown handler queued. Regression test for #2510: a key sequence used to
/// return as soon as Chrome had dispatched the key events, so a click issued
/// right after could land before a `setTimeout(0)` the keys scheduled had
/// fired (e.g. MUI's ClickAwayListener arming itself on mount).
#[tokio::test]
#[ignore]
async fn e2e_press_waits_for_queued_macrotasks() {
let mut state = DaemonState::new();

let resp = execute_command(
&json!({ "id": "1", "action": "launch", "headless": true, "args": ["--no-sandbox", "--disable-dev-shm-usage"] }),
&mut state,
)
.await;
assert_success(&resp);

// The keydown handler arms the flag behind one animation frame and one
// macrotask, but the frame callback first burns 120ms of render work,
// like the React app in the issue (whose keystrokes delayed the
// mount-time timer by 86-95ms). Headless frames run too fast for a
// plain frame gate to fail without the fix, so the busy frame makes the
// race deterministic: `press` must have waited for both before
// returning, so the flag is already true; without the settle it would
// still be false.
let html = concat!(
"data:text/html,<html><body><script>",
"window.__armed=false;",
"addEventListener('keydown',e=>{if(e.key==='Enter'){requestAnimationFrame(()=>{var t=performance.now();while(performance.now()-t<120){}setTimeout(()=>{window.__armed=true;},0);});}});",
"</script></body></html>"
);
let resp = execute_command(
&json!({ "id": "2", "action": "navigate", "url": html }),
&mut state,
)
.await;
assert_success(&resp);

let resp = execute_command(
&json!({ "id": "3", "action": "press", "key": "Enter" }),
&mut state,
)
.await;
assert_success(&resp);

// The flag can only be true if `press` waited out the busy frame and
// the macrotask queued after it.
let resp = execute_command(
&json!({ "id": "4", "action": "evaluate", "script": "window.__armed" }),
&mut state,
)
.await;
assert_success(&resp);
assert_eq!(
get_data(&resp)["result"],
json!(true),
"press must wait for the frame and macrotask the keydown handler queued"
);

let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
assert_success(&resp);
}

/// `press` must return promptly when a key handler opens a `confirm`
/// dialog inside an animation-frame callback. Regression test for the
/// Vercel review on #2521: the settle probe's five-second fallback timer
/// is paused while a modal dialog is open, so the old settle evaluation
/// blocked for the full 30-second CDP timeout. The settle must be bounded
/// by a timer the dialog cannot pause, so `press` returns promptly once
/// the dialog opens (and within six seconds even if it was already open)
/// and the dialog stays pending for the agent.
#[tokio::test]
#[ignore]
async fn e2e_press_returns_when_dialog_opens_in_animation_frame() {
let mut state = DaemonState::new();

let resp = execute_command(
&json!({ "id": "1", "action": "launch", "headless": true, "args": ["--no-sandbox", "--disable-dev-shm-usage"] }),
&mut state,
)
.await;
assert_success(&resp);

// The keydown handler opens a confirm dialog inside a rAF callback.
// `confirm` is not auto-dismissed (only `alert` and `beforeunload`
// are), so the dialog stays open and pauses the page's timers while
// the settle probe is waiting.
let html = concat!(
"data:text/html,<html><body><script>",
"addEventListener('keydown',e=>{if(e.key==='Enter'){requestAnimationFrame(()=>{confirm('from frame');});}});",
"</script></body></html>"
);
let resp = execute_command(
&json!({ "id": "2", "action": "navigate", "url": html }),
&mut state,
)
.await;
assert_success(&resp);

let started = std::time::Instant::now();
let resp = execute_command(
&json!({ "id": "3", "action": "press", "key": "Enter" }),
&mut state,
)
.await;
let elapsed = started.elapsed();
assert_success(&resp);
// The settle returns as soon as the dialog opens, and is capped at
// six seconds by a host-side timer the dialog cannot pause; without
// the fix the probe waits out the 30-second CDP timeout. Fifteen
// seconds leaves ample headroom for slow runners.
assert!(
elapsed < std::time::Duration::from_secs(15),
"press blocked {elapsed:?} on a dialog the settle should have given up on"
);

// The dialog is still open for the agent: dismiss it and confirm the
// page recovers.
let resp = execute_command(
&json!({ "id": "4", "action": "dialog", "response": "dismiss" }),
&mut state,
)
.await;
assert_success(&resp);
let resp = execute_command(
&json!({ "id": "5", "action": "evaluate", "script": "document.title" }),
&mut state,
)
.await;
assert_success(&resp);

let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
assert_success(&resp);
}

#[tokio::test]
#[ignore]
async fn e2e_press_returns_promptly_when_key_opens_dialog() {
let mut state = DaemonState::new();

let resp = execute_command(
&json!({ "id": "1", "action": "launch", "headless": true, "args": ["--no-sandbox", "--disable-dev-shm-usage"] }),
&mut state,
)
.await;
assert_success(&resp);

// The keydown handler opens a confirm dialog from an animation-frame
// callback. A dialog pauses the page's timers, so without the
// dialog-aware settle the press would hang until the 30s CDP timeout.
let html = concat!(
"data:text/html,<html><body><script>",
"addEventListener('keydown',e=>{if(e.key==='Enter'){requestAnimationFrame(()=>{confirm('are you sure?');});}});",
"</script></body></html>"
);
let resp = execute_command(
&json!({ "id": "2", "action": "navigate", "url": html }),
&mut state,
)
.await;
assert_success(&resp);

let start = std::time::Instant::now();
let resp = execute_command(
&json!({ "id": "3", "action": "press", "key": "Enter" }),
&mut state,
)
.await;
assert_success(&resp);
assert!(
start.elapsed() < std::time::Duration::from_secs(15),
"press must return when a dialog opens instead of hanging for the 30s CDP timeout; took {:?}",
start.elapsed()
);

// The dialog stays open for the agent to handle: wait until it is
// tracked as pending, then dismiss it and check the page is alive.
let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5);
loop {
let resp = execute_command(
&json!({ "id": "4", "action": "dialog", "response": "status" }),
&mut state,
)
.await;
assert_success(&resp);
if get_data(&resp)["hasDialog"] == json!(true) {
break;
}
assert!(
std::time::Instant::now() < deadline,
"dialog opened by the key handler must be pending"
);
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
}

let resp = execute_command(
&json!({ "id": "5", "action": "dialog", "response": "dismiss" }),
&mut state,
)
.await;
assert_success(&resp);
let resp = execute_command(
&json!({ "id": "6", "action": "evaluate", "script": "document.title" }),
&mut state,
)
.await;
assert_success(&resp);

let resp = execute_command(&json!({ "id": "99", "action": "close" }), &mut state).await;
assert_success(&resp);
}

// ---------------------------------------------------------------------------
// Raw mouse regressions
// ---------------------------------------------------------------------------
Expand Down
78 changes: 78 additions & 0 deletions cli/src/native/interaction.rs
Original file line number Diff line number Diff line change
Expand Up @@ -406,6 +406,84 @@ pub async fn press_key_with_modifiers(
Ok(())
}

/// Wait until the page has run one animation frame and one macrotask.
///
/// Key handlers often schedule work behind a frame or a `setTimeout(0)`
/// (e.g. MUI's ClickAwayListener arms itself on a mount-time timer), so
/// returning right after the key events are dispatched lets the next command
/// land before that work has run. Evaluating a promise that resolves after
/// one frame plus one timer covers exactly the tasks the keys queued: timers
/// of equal delay run in queue order, so a `setTimeout(0)` scheduled by a
/// key handler runs before this one.
///
/// Best-effort: if the page has no script context (e.g. a `chrome://`
/// page) the evaluation fails, and the key events were already dispatched,
/// so a failed settle probe must not fail the command. The fallback timer
/// bounds the wait at 5s for pages whose frames never fire (e.g. a hidden
/// tab).
///
/// A JavaScript dialog opened by the key handler ends the settle at once.
/// `confirm`/`prompt` dialogs pause the page's timers, so neither the frame
/// callback nor the fallback timer can fire while one is open, and the
/// evaluation would hang until the 30s CDP timeout. The dialog stays open
/// for the agent to handle with the `dialog` command; there is nothing to
/// wait for while the page is blocked. A dialog that is already open when
/// the settle starts fires no event, so the whole probe is also capped just
/// above the 5s in-page fallback.
pub async fn wait_for_settle(client: &CdpClient, session_id: &str) {
use tokio::sync::broadcast::error::RecvError;

// Subscribe before evaluating so a dialog that opens mid-settle cannot
// slip past us, the same race `dispatch_mouse_or_dialog` uses.
let mut events = client.subscribe();
// The params must outlive the future: `send_command_typed` borrows them,
// and the future is polled across awaits below.
let params = EvaluateParams {
expression: "new Promise((resolve) => { setTimeout(() => resolve(false), 5000); requestAnimationFrame(() => setTimeout(() => resolve(true), 0)); })"
.to_string(),
return_by_value: Some(true),
await_promise: Some(true),
};
let settle = client.send_command_typed::<_, EvaluateResult>(
"Runtime.evaluate",
&params,
Some(session_id),
);
tokio::pin!(settle);
let deadline = tokio::time::sleep(std::time::Duration::from_secs(6));
tokio::pin!(deadline);
loop {
tokio::select! {
_ = &mut settle => return,
_ = &mut deadline => return,
event = events.recv() => {
match event {
Ok(e) if e.method == "Page.javascriptDialogOpening" => {
// Only a dialog on this settle's session ends it; a
// background-tab dialog must not. A session-less
// event has no flat session and is treated as the
// top-level page (i.e. ours).
let ours = match e.session_id.as_deref() {
Some(sid) => sid == session_id,
None => true,
};
if ours {
return;
}
continue;
}
Ok(_) => continue,
Err(RecvError::Lagged(_)) => continue,
Err(RecvError::Closed) => {
let _ = (&mut settle).await;
return;
}
}
}
}
}
}

pub async fn scroll(
client: &CdpClient,
session_id: &str,
Expand Down
3 changes: 3 additions & 0 deletions cli/src/output.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2292,6 +2292,7 @@ agent-browser type - Type text into an element
Usage: agent-browser type <selector> [--clear] [--delay <ms>] [--] <text>

Types text into the specified element character by character.
Returns after the page has run one animation frame and one macrotask, so timers queued by a key handler have already run before the next command.
Use --clear to clear existing content first. Use --delay to set the delay between keystrokes in milliseconds.
Use -- before text that looks like a flag to pass it literally. Put options before --.

Expand Down Expand Up @@ -2466,6 +2467,7 @@ agent-browser press - Press a key or key combination
Usage: agent-browser press <key>

Presses a key or key combination. Supports special keys and modifiers.
Returns after the page has run one animation frame and one macrotask, so timers queued by a key handler have already run before the next command.

Aliases: key

Expand Down Expand Up @@ -2535,6 +2537,7 @@ Sends keyboard input to whatever element currently has focus.
Unlike 'type' which requires a selector, 'keyboard' operates on
the current focus — essential for contenteditable editors like
Lexical, ProseMirror, CodeMirror, and Monaco.
The 'type' and 'inserttext' subcommands return after the page has run one animation frame and one macrotask, so timers queued by a key handler have already run before the next command.

Subcommands:
type <text> Type text character-by-character with real
Expand Down
2 changes: 2 additions & 0 deletions docs/content/docs/commands.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,8 @@ agent-browser mcp call <tool> <json> # Run one MCP tool and print its result

For `fill` and `type`, put options before the optional `--` marker. Text after the marker is literal, including values that look like flags. For example, `agent-browser fill "#search" -- --json` fills with `--json`, and `agent-browser --json type "#search" --clear --delay 10 -- --clear` clears the field, then types `--clear`. MCP `agent_browser_fill` and `agent_browser_type` pass typed `text` values literally, so they do not need the marker.

`type`, `press`, `keyboard type`, `keyboard inserttext`, and `inserttext` return only after the page has run one animation frame and one macrotask, so timers queued by a key handler have already run before the next command.

Browser responses automatically announce WebMCP tools on first discovery and when the catalog changes. Summaries contain only names, brief descriptions, origins, and frame IDs. Choose a relevant tool, then fetch its full schema with `agent-browser webmcp list <tool> --frame <frame-id> --json` before invoking it. Schemas and annotations are never included proactively. Unchanged catalogs and pages without tools add no context. JSON exposes updates as `data.webmcp`; CLI and MCP text include the same bounded summary. See [WebMCP](/webmcp) for update semantics and security.

Clicks fail before dispatch when another element covers the target's click point. The error names the covering element, for example `covered by <div#consent-banner>`. Dismiss or interact with that element, take a fresh snapshot, then retry the original action. Click and dblclick fail, instead of reporting success, when the element is hidden, disabled, or outside the viewport. `click --new-tab` opens the link's URL directly, so these checks do not apply to it.
Expand Down
2 changes: 2 additions & 0 deletions skill-data/core/references/commands.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,8 @@ agent-browser upload @e1 file.pdf # Upload files

For `fill` and `type`, put options before the optional `--` marker. Text after the marker is literal, including values that look like flags. For example, `agent-browser fill @e2 -- --json` fills with `--json`, and `agent-browser type @e2 --clear --delay 10 -- --clear` clears the field, then types `--clear`. MCP `agent_browser_fill` and `agent_browser_type` pass typed `text` values literally, so they do not need the marker.

`type`, `press`, `keyboard type`, `keyboard inserttext`, and `inserttext` return only after the page has run one animation frame and one macrotask, so timers queued by a key handler have already run before the next command.

Visible-label matching treats non-breaking and ordinary spaces equivalently.

Clicks fail before dispatch when another element covers the target's click point. The error names the covering element, for example `covered by <div#consent-banner>`. Dismiss or interact with that element, run a fresh snapshot, then retry the original action. Click and dblclick fail, instead of reporting success, when the element is hidden, disabled, or outside the viewport. `click --new-tab` opens the link's URL directly, so these checks do not apply to it.
Expand Down