Python Web Scraping for AI Agents

Every Python AI agent eventually needs to read the web. Instead of installing Playwright, managing browser binaries, and writing async scraper boilerplate, call denkbot.dog. One HTTP request, structured JSON back. Works with every Python agent framework that can call an HTTP endpoint.

Get API Key →Read the Docs

What you'd use this for

CrewAI agents with web research tasks, AutoGen conversations that need factual grounding, custom agent loops built with bare Python, any asyncio-based agent that needs non-blocking URL fetching.

How it works

example

import httpx
from dataclasses import dataclass

@dataclass
class PageContent:
    url: str
    title: str
    text: str
    links: list[dict]
    cached: bool

class DenkbotClient:
    BASE = "https://api.denkbot.dog"

    def __init__(self, api_key: str):
        self.headers = {"Authorization": f"Bearer {api_key}"}

    def scrape(self, url: str, render_js: bool = False) -> PageContent:
        r = httpx.post(f"{self.BASE}/scrape", headers=self.headers,
            json={"url": url, "renderJs": render_js, "format": "json"}, timeout=30)
        d = r.json()
        return PageContent(d["url"], d["title"], d["text"], d["links"], d["cached"])

    async def scrape_async(self, url: str) -> PageContent:
        async with httpx.AsyncClient() as client:
            r = await client.post(f"{self.BASE}/scrape", headers=self.headers,
                json={"url": url, "format": "json"}, timeout=30)
        d = r.json()
        return PageContent(d["url"], d["title"], d["text"], d["links"], d["cached"])