Web Automation

┌─────────────────────────────────────────────────────────────┐
│                     Layer 4: Services                        │
│  - WebAutomationService  - WebCrawlService  - WebSearchService│
└─────────────────────────────────────────────────────────────┘
                           ↓
┌─────────────────────────────────────────────────────────────┐
│                      Layer 3: Core                           │
│  - BrowserService  - PageAnalyzer  - ActionCoordinator      │
└─────────────────────────────────────────────────────────────┘
                           ↓
┌─────────────────────────────────────────────────────────────┐
│                     Layer 2: Engines                         │
│  - ExtractionEngine  - DetectionEngine  - ActionEngine      │
└─────────────────────────────────────────────────────────────┘
                           ↓
┌─────────────────────────────────────────────────────────────┐
│                    Layer 1: Strategies                       │
│  - Detection: interactive_extractor                          │
│  - Extraction: readability, jina_reader                      │
│  - Actions: click, type, navigate, wait                      │
│  - Search: brave_search, parallel_search                     │
└─────────────────────────────────────────────────────────────┘

POST /api/v1/crawl

{
  "url": "https://example.com",
  "output_format": "markdown",
  "extract_links": true,
  "extract_images": true
}

async def crawl_page(url: str):
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://localhost:8000/api/v1/crawl",
            json={
                "url": url,
                "output_format": "markdown",
                "extract_links": True
            }
        )
        return response.json()

POST /api/v1/search

{
  "query": "Python async programming",
  "freshness": "week",
  "count": 10,
  "goggle": "tech",
  "summarize": true
}

POST /api/v1/deep-search

{
  "query": "How to implement async generators in Python",
  "max_iterations": 3,
  "rag_mode": "auto"
}

POST /api/v1/automation/execute

{
  "url": "https://example.com",
  "task": "Search for 'Python tutorials' and click the first result",
  "mode": "intelligent"
}

async def automate_task(url: str, task: str):
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://localhost:8000/api/v1/automation/execute",
            json={
                "url": url,
                "task": task,
                "mode": "intelligent"
            }
        )
        return response.json()
 
# Example: Login flow
result = await automate_task(
    "https://example.com/login",
    "Fill in username 'test@example.com' and password 'password123', then click submit"
)

POST /api/v1/screenshot

{
  "url": "https://example.com",
  "full_page": true,
  "format": "png"
}

@mcp.tool()
async def web_crawl(url: str, format: str = "markdown"):
    """Crawl webpage and extract content"""
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://localhost:8000/api/v1/crawl",
            json={"url": url, "output_format": format}
        )
        return response.json()
 
@mcp.tool()
async def web_search(query: str, count: int = 5):
    """Search the web"""
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://localhost:8000/api/v1/search",
            json={"query": query, "count": count}
        )
        return response.json()
 
@mcp.tool()
async def browser_automate(url: str, task: str):
    """Automate browser task"""
    async with httpx.AsyncClient() as client:
        response = await client.post(
            "http://localhost:8000/api/v1/automation/execute",
            json={"url": url, "task": task}
        )
        return response.json()