WebMCP Data Extraction Agent: Replacing Fragile Scrapers with Chrome's Tool Protocol

1// Website registers tools in its own JavaScript
2// (runs in the browser page context)
3navigator.modelContext.registerTool('searchFlights', {
4  description: 'Search for available flights between two airports',
5  parameters: {
6    origin: { type: 'string', description: 'IATA airport code (e.g., OTP)' },
7    destination: { type: 'string', description: 'IATA airport code (e.g., LHR)' },
8    date: { type: 'string', description: 'Departure date (YYYY-MM-DD)' },
9    passengers: { type: 'number', default: 1 },
10  },
11  execute: async (params) => {
12    // Actual flight search logic — native to the website
13    const results = await flightSearchAPI.query(params)
14    return { flights: results }
15  }
16})
17
18// The AI agent discovers this via:
19const tools = await navigator.modelContext.getTools()
20// Returns: [{ name: 'searchFlights', description: ..., parameters: ... }]

1import CDP from 'chrome-remote-interface'
2
3async function discoverTools(client: CDP.Client): Promise<Tool[]> {
4  const { result } = await client.Runtime.evaluate({
5    expression: 'JSON.stringify(await navigator.modelContext.getTools())',
6    awaitPromise: true,
7    returnByValue: true,
8  })
9  return JSON.parse(result.value) as Tool[]
10}
11
12async function callTool(
13  client: CDP.Client,
14  toolName: string,
15  params: Record<string, unknown>
16): Promise<unknown> {
17  const expression = `
18    JSON.stringify(
19      await navigator.modelContext.callTool(
20        '${toolName}',
21        ${JSON.stringify(params)}
22      )
23    )
24  `
25  const { result } = await client.Runtime.evaluate({
26    expression,
27    awaitPromise: true,
28    returnByValue: true,
29  })
30
31  if (result.type === 'undefined' || result.value === undefined) {
32    throw new Error(`Tool '${toolName}' returned no result`)
33  }
34
35  return JSON.parse(result.value)
36}

1async function planExtraction(
2  tools: Tool[],
3  userGoal: string,
4  outputSchema: object
5): Promise<ExecutionStep[]> {
6  const toolDescriptions = tools.map(t =>
7    `${t.name}: ${t.description}\nParameters: ${JSON.stringify(t.parameters)}`
8  ).join('\n\n')
9
10  const prompt = `
11You are an extraction planner. Given these available tools and a user's goal,
12create an execution plan.
13
14AVAILABLE TOOLS:
15${toolDescriptions}
16
17USER GOAL: ${userGoal}
18
19EXPECTED OUTPUT SCHEMA: ${JSON.stringify(outputSchema)}
20
21Return a JSON array of steps: [{ "tool": "toolName", "params": {...}, "reason": "..." }]
22Chain tools when needed: e.g., search → filter → list → extract.
23`
24
25  const response = await model.generateContent(prompt)
26  return JSON.parse(response.text) as ExecutionStep[]
27}
28
29// Example plan for "Extract direct flights BUC→LHR next week":
30// [
31//   { tool: "searchFlights", params: { origin: "OTP", destination: "LHR", date: "2025-06-15" }, reason: "Search available flights" },
32//   { tool: "setFilters", params: { stops: 0 }, reason: "Filter to direct only" },
33//   { tool: "listFlights", params: {}, reason: "Get final filtered results" }
34// ]

1import Ajv from 'ajv'
2const ajv = new Ajv()
3
4async function executeWithValidation(
5  client: CDP.Client,
6  plan: ExecutionStep[],
7  outputSchema: object,
8  maxRetries = 2
9): Promise<unknown> {
10  const validate = ajv.compile(outputSchema)
11  let lastResult: unknown
12
13  for (const step of plan) {
14    lastResult = await callTool(client, step.tool, step.params)
15  }
16
17  // Validate final output against expected schema
18  if (!validate(lastResult)) {
19    if (maxRetries > 0) {
20      // Ask Gemini to fix the plan based on the validation errors
21      const fixedPlan = await repairPlan(plan, validate.errors, outputSchema)
22      return executeWithValidation(client, fixedPlan, outputSchema, maxRetries - 1)
23    }
24    throw new Error(`Output validation failed: ${JSON.stringify(validate.errors)}`)
25  }
26
27  return lastResult
28}