observe()

Observe

See how to use observe() to discover actionable elements and analyze web page structure

Method Signatures

TypeScript
Python

// With ObserveOptions
await page.observe(options: ObserveOptions): Promise<ObserveResult[]>

// String instruction shorthand
await page.observe(instruction: string): Promise<ObserveResult[]>

ObserveOptions Interface:

interface ObserveOptions {
  instruction?: string;
  modelName?: AvailableModel;
  modelClientOptions?: ClientOptions;
  domSettleTimeoutMs?: number;
  drawOverlay?: boolean;
  iframes?: boolean;
}

# With parameters
await page.observe(
    instruction: str,
    dom_settle_timeout_ms: int = None,
    iframes: bool = None,
    model_name: AvailableModel = None,
    model_client_options: Dict = None
) -> List[ObserveResult]

Parameters

instruction

string

Natural language description of elements or actions to discover.

modelName

AvailableModel

Override the default LLM model for this observation.

modelClientOptions

ClientOptions

Model-specific configuration options.

domSettleTimeoutMs

number

Maximum time to wait for DOM to stabilize before analysis.Default: 30000

drawOverlay

boolean

Whether to draw visual overlays on discovered elements (debugging).Default: false

iframes

boolean

Set to true to search within iframes.Default: false

Response

Returns: Promise<ObserveResult[]> Array of discovered actionable elements, ordered by relevance. ObserveResult Interface:

interface ObserveResult {
  selector: string;        // XPath selector to locate element
  description: string;     // Human-readable description
  method?: string;         // Suggested action method
  arguments?: string[];    // Additional action parameters
}

selector

string

XPath selector that precisely locates the element.

description

string

Human-readable description of the element and its purpose.

method

string

Suggested interaction method: "click", "fill", "selectOptionFromDropdown", "nextChunk", "scrollTo", "prevChunk".

arguments

string[]

Additional parameters for the suggested action.

Code Examples

// Basic element discovery
const buttons = await page.observe("find all clickable buttons");
const formFields = await page.observe("locate form input fields");

// Advanced configuration
const elements = await page.observe({
  instruction: "find important call-to-action buttons",
  modelName: "gpt-4.1-mini",
  domSettleTimeoutMs: 45000,
  drawOverlay: true
});

// Working with results
const [loginButton] = await page.observe("find the login button");
if (loginButton) {
  console.log("Found:", loginButton.description);
  console.log("Selector:", loginButton.selector);
  await page.act(loginButton); // Execute the action
}

// Filter results
const submitButtons = await page.observe("find all submit buttons");
const primarySubmit = submitButtons.find(btn => 
  btn.description.toLowerCase().includes('primary')
);

// Iframe search
const iframeElements = await page.observe({
  instruction: "find form fields inside the iframe",
  iframes: true
});

# Basic element discovery
buttons = await page.observe("find all clickable buttons")
form_fields = await page.observe("locate the form fields")

# Advanced configuration  
elements = await page.observe(
    instruction="find important call-to-action buttons",
    model_name="gpt-4.1-mini",
    dom_settle_timeout_ms=45000
)

# Working with results
login_buttons = await page.observe("find the login button")
if login_buttons:
    button = login_buttons[0]
    print("Found:", button.description)
    print("Selector:", button.selector)
    await page.act(button)  # Execute the action

# Filter results
submit_buttons = await page.observe("find all submit buttons")
primary_submit = next((
    btn for btn in submit_buttons 
    if 'primary' in btn.description.lower()
), None)

# Iframe search
iframe_elements = await page.observe(
    instruction="find the form fields inside the iframe",
    iframes=True
)

Integration Patterns

// Observe → Act workflow
const actions = await page.observe("find checkout elements");
for (const action of actions) {
  await page.act(action);
  await page.waitForTimeout(1000);
}

// Observe → Extract workflow
const tables = await page.observe("find data tables");
if (tables.length > 0) {
  const data = await page.extract({
    instruction: "extract the table data",
    selector: tables[0].selector,
    schema: DataSchema
  });
}

// Element validation
const requiredElements = await page.observe("find the login form");
if (requiredElements.length === 0) {
  throw new Error("Login form not found");
}

# Observe → Act workflow  
actions = await page.observe("find checkout elements")
for action in actions:
    await page.act(action)
    await page.wait_for_timeout(1000)

# Observe → Extract workflow
tables = await page.observe("find data tables")
if tables:
    data = await page.extract(
        instruction="extract the table data",
        selector=tables[0].selector,
        schema=DataSchema
    )

# Element validation
required_elements = await page.observe("find login form")
if not required_elements:
    raise Exception("Login form not found")

First Steps

The Basics

Configuration

Best Practices

Integrations

Reference

Observe

Method Signatures

Parameters

Response

Code Examples

Integration Patterns

Observe

​Method Signatures

​Parameters

​Response

​Code Examples

​Integration Patterns

Method Signatures

Parameters

Response

Code Examples

Integration Patterns