python -m venv venv
source venv/bin/activate
pip install playwright beautifulsoup4 httpx lxml
playwright install chromium

# browser/session.py
from __future__ import annotations
import asyncio
from playwright.async_api import async_playwright, Browser, Page
from typing import Optional

class BrowserSession:
    def __init__(self, headless: bool = True):
        self.headless = headless
        self.playwright = None
        self.browser: Optional[Browser] = None
        self.page: Optional[Page] = None
        self.context = None

    async def start(self) -> None:
        self.playwright = await async_playwright().start()
        self.browser = await self.playwright.chromium.launch(
            headless=self.headless
        )
        self.context = await self.browser.new_context(
            viewport={"width": 1920, "height": 1080},
            user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        )
        self.page = await self.context.new_page()

    async def navigate(self, url: str, wait_until: str = "networkidle") -> str:
        try:
            await self.page.goto(url, wait_until=wait_until, timeout=30000)
            return await self.page.content()
        except Exception as e:
            return f"Error navigating to {url}: {str(e)}"

    async def click(self, selector: str) -> bool:
        try:
            await self.page.click(selector, timeout=5000)
            return True
        except:
            return False

    async def fill(self, selector: str, text: str) -> bool:
        try:
            await self.page.fill(selector, text, timeout=5000)
            return True
        except:
            return False

    async def get_text(self, selector: str) -> str:
        try:
            element = await self.page.query_selector(selector)
            if element:
                return await element.text_content()
        except:
            pass
        return ""

    async def screenshot(self, path: str = "screenshot.png") -> None:
        await self.page.screenshot(path=path)

    async def close(self) -> None:
        if self.browser:
            await self.browser.close()
        if self.playwright:
            await self.playwright.stop()

# browser/page_parser.py
from bs4 import BeautifulSoup
from typing import Dict, List
import re

class PageParser:
    def __init__(self):
        self.soup = None

    def parse(self, html: str) -> None:
        self.soup = BeautifulSoup(html, "lxml")

    def get_text(self) -> str:
        if not self.soup:
            return ""
        return self.soup.get_text(separator="\n", strip=True)

    def get_links(self) -> List[Dict[str, str]]:
        if not self.soup:
            return []
        links = []
        for a in self.soup.find_all("a", href=True):
            links.append({
                "text": a.get_text(strip=True),
                "href": a["href"],
            })
        return links

    def get_tables(self) -> List[List[List[str]]]:
        if not self.soup:
            return []
        tables = []
        for table in self.soup.find_all("table"):
            rows = []
            for tr in table.find_all("tr"):
                cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                rows.append(cells)
            tables.append(rows)
        return tables

    def get_meta(self) -> Dict[str, str]:
        if not self.soup:
            return {}
        meta = {}
        for tag in self.soup.find_all("meta"):
            name = tag.get("name") or tag.get("property", "")
            content = tag.get("content", "")
            if name and content:
                meta[name] = content
        return meta

    def get_structured_data(self) -> Dict:
        return {
            "title": self.soup.title.string if self.soup and self.soup.title else "",
            "meta": self.get_meta(),
            "text": self.get_text()[:5000],
            "links": self.get_links()[:50],
            "tables": self.get_tables(),
        }

# extraction/extractor.py
from openai import OpenAI
from typing import Dict, Any
import json

class LLMExtractor:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.client = OpenAI()
        self.model = model

    def extract(self, page_data: Dict, instruction: str) -> Any:
        prompt = f"""Extract information from this web page based on the instruction.

Page title: {page_data.get('title', 'N/A')}
Page text (excerpt):
{page_data.get('text', '')[:3000]}

Instruction: {instruction}

Return the extracted information as JSON. If the information is not found, return null."""
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": "Extract structured data from web content."},
                {"role": "user", "content": prompt},
            ],
            temperature=0.0,
        )
        try:
            return json.loads(response.choices[0].message.content)
        except json.JSONDecodeError:
            return response.choices[0].message.content

    def classify_page(self, page_data: Dict) -> str:
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": """Classify this webpage into one category:
                article, product, forum, documentation, social, news, other.
                Return ONLY the category name."""},
                {"role": "user", "content": page_data.get("text", "")[:2000]},
            ],
            temperature=0.0,
        )
        return response.choices[0].message.content.strip()

# navigation/agent.py
from openai import OpenAI
from browser.session import BrowserSession
from browser.page_parser import PageParser
from typing import List, Dict, Optional
import json

class NavigationAgent:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.client = OpenAI()
        self.model = model
        self.parser = PageParser()

    async def navigate_to_goal(
        self,
        session: BrowserSession,
        start_url: str,
        goal: str,
        max_steps: int = 10,
    ) -> Dict:
        current_url = start_url
        history = []

        for step in range(max_steps):
            html = await session.navigate(current_url)
            self.parser.parse(html)
            page_data = self.parser.get_structured_data()

            action = self._decide_action(page_data, goal, history)
            history.append({
                "url": current_url,
                "action": action,
                "step": step + 1,
            })

            if action.get("type") == "extract":
                return {
                    "success": True,
                    "url": current_url,
                    "data": page_data,
                    "history": history,
                }
            elif action.get("type") == "click":
                clicked = await session.click(action["selector"])
                if not clicked:
                    return {"success": False, "error": "Click failed", "history": history}
                await session.page.wait_for_load_state("networkidle")
                current_url = session.page.url
            elif action.get("type") == "url":
                current_url = action["url"]
            else:
                return {"success": False, "error": "Unknown action", "history": history}

        return {"success": False, "error": "Max steps reached", "history": history}

    def _decide_action(
        self, page_data: Dict, goal: str, history: List[Dict]
    ) -> Dict:
        history_text = "\n".join(
            f"Step {h['step']}: {h['action'].get('type', 'unknown')} at {h['url']}"
            for h in history[-5:]
        )
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": """You are a web navigation agent.
                Decide the next action to achieve the goal.

                Return JSON with:
                - type: "click", "url", or "extract"
                - selector: CSS selector (for click)
                - url: URL to navigate (for url)
                - reason: why you chose this action"""},
                {"role": "user", "content": f"""Goal: {goal}

Current page title: {page_data.get('title', 'N/A')}
Current URL links: {json.dumps(page_data.get('links', [])[:10], indent=2)}

Navigation history:
{history_text}

What should I do next?"""},
            ],
            temperature=0.0,
        )
        try:
            return json.loads(response.choices[0].message.content)
        except:
            return {"type": "extract", "reason": "Default extraction"}

# agent.py
from __future__ import annotations
import asyncio
from browser.session import BrowserSession
from browser.page_parser import PageParser
from navigation.agent import NavigationAgent
from extraction.extractor import LLMExtractor
from typing import Dict

class WebBrowsingAgent:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.nav_agent = NavigationAgent(model)
        self.extractor = LLMExtractor(model)
        self.parser = PageParser()

    async def browse_and_extract(
        self, url: str, instruction: str, headless: bool = True
    ) -> Dict:
        session = BrowserSession(headless=headless)
        try:
            await session.start()
            result = await self.nav_agent.navigate_to_goal(
                session, url, instruction
            )
            if result["success"]:
                extracted = self.extractor.extract(
                    result["data"], instruction
                )
                return {
                    "success": True,
                    "url": result["url"],
                    "extracted_data": extracted,
                    "page_title": result["data"].get("title", ""),
                }
            return result
        finally:
            await session.close()

    async def scrape_multiple(
        self, urls: list[str], instruction: str
    ) -> list[Dict]:
        results = []
        for url in urls:
            result = await self.browse_and_extract(url, instruction)
            results.append(result)
            await asyncio.sleep(1)
        return results

import pytest
import asyncio
from agent import WebBrowsingAgent

@pytest.mark.asyncio
async def test_browse():
    agent = WebBrowsingAgent()
    result = await agent.browse_and_extract(
        "https://example.com",
        "Extract the main heading"
    )
    assert result["success"]

def test_parser():
    parser = PageParser()
    parser.parse("<html><body><h1>Test</h1></body></html>")
    assert "Test" in parser.get_text()

# main.py
import asyncio
from agent import WebBrowsingAgent

async def main():
    agent = WebBrowsingAgent()
    result = await agent.browse_and_extract(
        "https://news.ycombinator.com",
        "Extract the top 10 story titles and scores"
    )
    print(f"Success: {result['success']}")
    if result["success"]:
        print(f"Data: {result['extracted_data']}")

if __name__ == "__main__":
    asyncio.run(main())