python -m venv venv
source venv/bin/activate
pip install playwright beautifulsoup4 openai lxml
playwright install chromium
export OPENAI_API_KEY="sk-your-key"

# browser.py
"""Playwright-based browser controller with anti-bot stealth measures."""
import logging
import random
import time
from typing import Dict, List, Optional
from dataclasses import dataclass
from playwright.sync_api import sync_playwright, Page, Browser, BrowserContext

logger = logging.getLogger(__name__)

USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/120.0.0.0 Safari/537.36",
]


@dataclass
class BrowserConfig:
    headless: bool = True
    timeout: int = 30000
    viewport_width: int = 1280
    viewport_height: int = 720
    user_agent: str = USER_AGENTS[0]
    stealth_mode: bool = True
    request_delay: float = 1.0  # seconds between requests


class BrowserController:
    """Manages Playwright browser with stealth and error recovery.

    Args:
        config: Browser configuration options.
    """

    def __init__(self, config: BrowserConfig = None):
        self.config = config or BrowserConfig()
        self.playwright = None
        self.browser: Optional[Browser] = None
        self.context: Optional[BrowserContext] = None
        self.page: Optional[Page] = None

    def start(self) -> None:
        """Launch browser with stealth configurations."""
        self.playwright = sync_playwright().start()
        self.browser = self.playwright.chromium.launch(
            headless=self.config.headless,
            args=[
                "--disable-blink-features=AutomationControlled",
                "--disable-dev-shm-usage",
            ],
        )
        ua = random.choice(USER_AGENTS) if self.config.stealth_mode else self.config.user_agent
        self.context = self.browser.new_context(
            viewport={
                "width": self.config.viewport_width,
                "height": self.config.viewport_height,
            },
            user_agent=ua,
            java_script_enabled=True,
        )
        # Stealth: override navigator.webdriver
        self.context.add_init_script("""
            Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
        """)
        self.page = self.context.new_page()
        self.page.set_default_timeout(self.config.timeout)
        logger.info("Browser started with stealth mode")

    def stop(self) -> None:
        """Clean up browser resources."""
        if self.browser:
            self.browser.close()
        if self.playwright:
            self.playwright.stop()
        logger.info("Browser stopped")

    def navigate(self, url: str) -> Dict:
        """Navigate to URL with error handling and anti-bot delays."""
        try:
            time.sleep(random.uniform(0.5, self.config.request_delay))
            response = self.page.goto(url, wait_until="domcontentloaded")
            self.page.wait_for_load_state("networkidle")
            return {
                "success": True,
                "url": self.page.url,
                "title": self.page.title(),
                "status": response.status if response else None,
            }
        except Exception as e:
            logger.error(f"Navigation failed: {e}")
            return {"success": False, "error": str(e)}

    def click(self, selector: str) -> Dict:
        """Click an element with wait for navigation."""
        try:
            self.page.click(selector)
            self.page.wait_for_load_state("networkidle")
            return {"success": True, "url": self.page.url}
        except Exception as e:
            logger.error(f"Click failed: {e}")
            return {"success": False, "error": str(e)}

    def fill(self, selector: str, value: str) -> Dict:
        """Fill a form field."""
        try:
            self.page.fill(selector, value)
            return {"success": True}
        except Exception as e:
            logger.error(f"Fill failed: {e}")
            return {"success": False, "error": str(e)}

    def scroll_to_bottom(self) -> None:
        """Auto-scroll to load lazy content."""
        self.page.evaluate("window.scrollTo(0, document.body.scrollHeight)")
        time.sleep(1)

    def get_content(self) -> str:
        return self.page.content()

    def get_text(self) -> str:
        return self.page.inner_text("body")

    def screenshot(self, path: str) -> None:
        self.page.screenshot(path=path, full_page=True)

# parser.py
"""DOM parser with noise removal, content scoring, and structured extraction."""
import re
import logging
from typing import Dict, List, Optional
from bs4 import BeautifulSoup

logger = logging.getLogger(__name__)


class DOMParser:
    """Parse HTML with noise removal and content scoring.

    Removes: scripts, styles, nav, footer, header, ads, sidebars.
    Extracts: text, links, forms, tables, metadata.
    """

    NOISE_TAGS = ["script", "style", "nav", "footer", "header", "aside", "noscript"]
    NOISE_CLASSES = re.compile(r"(sidebar|ad|popup|modal|cookie|banner)", re.I)

    def parse(self, html: str) -> Dict:
        """Parse HTML and extract structured content."""
        soup = BeautifulSoup(html, "lxml")
        self._remove_noise(soup)

        return {
            "title": soup.title.string if soup.title else "",
            "text": self._extract_text(soup),
            "links": self._extract_links(soup),
            "forms": self._extract_forms(soup),
            "tables": self._extract_tables(soup),
            "structured": self._extract_structured(soup),
            "meta": self._extract_meta(soup),
        }

    def _remove_noise(self, soup: BeautifulSoup) -> None:
        """Remove non-content elements."""
        for tag in soup.find_all(self.NOISE_TAGS):
            tag.decompose()
        for tag in soup.find_all(class_=self.NOISE_CLASSES):
            tag.decompose()
        for tag in soup.find_all(id=self.NOISE_CLASSES):
            tag.decompose()

    def _extract_links(self, soup: BeautifulSoup) -> List[Dict]:
        links = []
        for a in soup.find_all("a", href=True):
            text = a.get_text(strip=True)
            if text and len(text) > 2:
                links.append({"text": text[:100], "href": a["href"]})
        return links[:50]

    def _extract_forms(self, soup: BeautifulSoup) -> List[Dict]:
        forms = []
        for form in soup.find_all("form"):
            inputs = []
            for inp in form.find_all(["input", "select", "textarea"]):
                inputs.append({
                    "type": inp.get("type", "text"),
                    "name": inp.get("name", ""),
                    "placeholder": inp.get("placeholder", ""),
                })
            forms.append({
                "action": form.get("action", ""),
                "method": form.get("method", "get"),
                "inputs": inputs,
            })
        return forms

    def _extract_text(self, soup: BeautifulSoup) -> str:
        text = soup.get_text(separator="\n", strip=True)
        lines = [line.strip() for line in text.splitlines() if line.strip()]
        return "\n".join(lines[:200])

    def _extract_tables(self, soup: BeautifulSoup) -> List[List[List[str]]]:
        result = []
        for table in soup.find_all("table")[:5]:
            rows = []
            for tr in table.find_all("tr"):
                cells = [td.get_text(strip=True) for td in tr.find_all(["td", "th"])]
                rows.append(cells)
            result.append(rows)
        return result

    def _extract_structured(self, soup: BeautifulSoup) -> Dict:
        data = {}
        lists = soup.find_all(["ul", "ol"])
        if lists:
            data["lists"] = [
                [li.get_text(strip=True) for li in lst.find_all("li")]
                for lst in lists[:10]
            ]
        return data

    def _extract_meta(self, soup: BeautifulSoup) -> Dict:
        meta = {}
        for tag in soup.find_all("meta"):
            name = tag.get("name") or tag.get("property", "")
            content = tag.get("content", "")
            if name and content:
                meta[name] = content
        return meta

# agent.py
"""Web browsing agent with LLM-driven decision making and error recovery."""
import json
import re
import logging
from typing import Dict, List
from openai import OpenAI
from browser import BrowserController, BrowserConfig
from parser import DOMParser

logger = logging.getLogger(__name__)

SYSTEM_PROMPT = """You are a web browsing agent. You navigate websites to accomplish user goals.

Available actions:
- navigate(url): Go to a URL
- click(selector): Click an element (CSS selector)
- fill(selector, value): Fill a form field
- extract(): Get page content
- done(answer): Task complete

Analyze the page content and decide the next action. Always explain your reasoning.
If stuck after 3 attempts at the same page, try a different approach or give up."""


class WebBrowsingAgent:
    """LLM-driven web browsing agent with error recovery.

    Args:
        model: OpenAI model for decision making.
        max_steps: Maximum navigation steps per task.
    """

    def __init__(
        self,
        model: str = "gpt-4-turbo-preview",
        max_steps: int = 15,
    ):
        self.client = OpenAI()
        self.model = model
        self.max_steps = max_steps
        self.browser = BrowserController()
        self.parser = DOMParser()
        self.history: List[Dict] = []

    def start(self) -> None:
        self.browser.start()

    def stop(self) -> None:
        self.browser.stop()

    def run(self, goal: str) -> Dict:
        """Execute a web browsing task from goal to completion.

        Args:
            goal: Natural language task description.

        Returns:
            Dict with success status, answer, steps taken, and history.
        """
        self.history = []
        repeated_pages = 0

        for step in range(self.max_steps):
            current_page = self.browser.get_content()
            parsed = self.parser.parse(current_page)

            # Detect stuck loops
            if self._is_loop(parsed["title"]):
                repeated_pages += 1
                if repeated_pages >= 3:
                    return {
                        "success": False,
                        "error": "Agent stuck in loop",
                        "steps": step + 1,
                        "history": self.history,
                    }
            else:
                repeated_pages = 0

            context = self._build_context(goal, parsed, step)

            try:
                response = self.client.chat.completions.create(
                    model=self.model,
                    messages=[
                        {"role": "system", "content": SYSTEM_PROMPT},
                        {"role": "user", "content": context},
                    ],
                    temperature=0.0,
                )
                action = self._parse_action(response.choices[0].message.content)
            except Exception as e:
                logger.error(f"LLM call failed: {e}")
                return {
                    "success": False,
                    "error": f"LLM error: {e}",
                    "steps": step + 1,
                    "history": self.history,
                }

            self.history.append({
                "step": step + 1,
                "action": action,
                "page_title": parsed["title"],
            })

            if action["type"] == "done":
                return {
                    "success": True,
                    "answer": action.get("answer", ""),
                    "steps": step + 1,
                    "history": self.history,
                }

            result = self._execute_action(action)
            if not result["success"]:
                logger.warning(f"Action failed: {result.get('error')}")
                self.history[-1]["error"] = result.get("error")

        return {
            "success": False,
            "error": "Max steps reached",
            "steps": self.max_steps,
            "history": self.history,
        }

    def _build_context(self, goal: str, parsed: Dict, step: int) -> str:
        text_preview = parsed["text"][:2000]
        links_preview = json.dumps(parsed["links"][:10], indent=2)
        forms_preview = json.dumps(parsed["forms"][:3], indent=2)

        return f"""Goal: {goal}
Current page: {parsed['title']}
URL: {self.browser.page.url}
Step: {step + 1} of {self.max_steps}

Page content:
{text_preview}

Available links:
{links_preview}

Available forms:
{forms_preview}

What action should I take next? Respond with one of:
- navigate(url): Go to a URL
- click(selector): Click an element
- fill(selector, value): Fill a form field
- extract(): Get page content
- done(answer): Task complete with answer"""

    def _parse_action(self, response: str) -> Dict:
        for action_type in ["navigate", "click", "fill", "extract", "done"]:
            if f"{action_type}(" in response:
                match = re.search(rf"{action_type}\(([^)]*)\)", response)
                if match:
                    args = match.group(1).split(",")
                    return {
                        "type": action_type,
                        "args": [a.strip().strip("\"'") for a in args],
                    }
        return {"type": "done", "args": ["Could not determine action"]}

    def _execute_action(self, action: Dict) -> Dict:
        if action["type"] == "navigate":
            return self.browser.navigate(action["args"][0])
        elif action["type"] == "click":
            return self.browser.click(action["args"][0])
        elif action["type"] == "fill":
            return self.browser.fill(action["args"][0], action["args"][1])
        elif action["type"] == "extract":
            return {"success": True, "content": self.browser.get_text()}
        elif action["type"] == "done":
            return {"success": True}
        return {"success": False, "error": "Unknown action"}

    def _is_loop(self, page_title: str) -> bool:
        if len(self.history) < 2:
            return False
        return self.history[-1]["page_title"] == page_title