python -m venv venv
source venv/bin/activate
pip install httpx openai arxiv pydantic
export OPENAI_API_KEY="sk-your-key"

# search/arxiv_client.py
import arxiv
from typing import List, Dict

class ArxivClient:
    def search(
        self, query: str, max_results: int = 20, sort_by: str = "relevance"
    ) -> List[Dict]:
        sort_map = {
            "relevance": arxiv.SortCriterion.Relevance,
            "date": arxiv.SortCriterion.SubmittedDate,
            "citations": arxiv.SortCriterion.Relevance,
        }
        client = arxiv.Client()
        search = arxiv.Search(
            query=query,
            max_results=max_results,
            sort_by=sort_map.get(sort_by, arxiv.SortCriterion.Relevance),
        )
        papers = []
        for result in client.results(search):
            papers.append({
                "id": result.entry_id,
                "title": result.title,
                "authors": [a.name for a in result.authors],
                "abstract": result.summary,
                "published": result.published.isoformat(),
                "updated": result.updated.isoformat(),
                "pdf_url": result.pdf_url,
                "categories": result.categories,
                "primary_category": result.primary_category,
                "source": "arxiv",
            })
        return papers

# search/semantic_scholar.py
import httpx
from typing import List, Dict

class SemanticScholarClient:
    BASE_URL = "https://api.semanticscholar.org/graph/v1"

    def __init__(self, api_key: str = None):
        self.headers = {}
        if api_key:
            self.headers["x-api-key"] = api_key

    def search(self, query: str, limit: int = 20) -> List[Dict]:
        response = httpx.get(
            f"{self.BASE_URL}/paper/search",
            params={"query": query, "limit": limit, "fields": "title,authors,abstract,year,citationCount,url"},
            headers=self.headers,
        )
        data = response.json()
        return [
            {
                "id": p.get("paperId", ""),
                "title": p.get("title", ""),
                "authors": [a.get("name", "") for a in p.get("authors", [])],
                "abstract": p.get("abstract", ""),
                "year": p.get("year"),
                "citations": p.get("citationCount", 0),
                "url": p.get("url", ""),
                "source": "semantic_scholar",
            }
            for p in data.get("data", [])
        ]

    def get_paper(self, paper_id: str) -> Dict:
        response = httpx.get(
            f"{self.BASE_URL}/paper/{paper_id}",
            params={"fields": "title,authors,abstract,year,citationCount,references,tldr"},
            headers=self.headers,
        )
        return response.json()

# extraction/paper_extractor.py
from openai import OpenAI
from typing import Dict, List
import json

class PaperExtractor:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.client = OpenAI()
        self.model = model

    def extract(self, paper: Dict) -> Dict:
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": """Extract structured information from this research paper.
                Return JSON:
                {
                    "research_question": "main question addressed",
                    "methodology": "approach used",
                    "key_findings": ["list of findings"],
                    "contributions": ["list of contributions"],
                    "limitations": ["list of limitations"],
                    "future_work": ["suggested future directions"],
                    "key_metrics": {"metric_name": "value"},
                    "dataset_used": "dataset name or description"
                }"""},
                {"role": "user", "content": f"Title: {paper['title']}\n\nAbstract: {paper.get('abstract', '')}"},
            ],
            temperature=0.0,
        )
        try:
            return json.loads(response.choices[0].message.content)
        except:
            return {"research_question": "Unknown", "key_findings": []}

    def extract_batch(self, papers: List[Dict]) -> List[Dict]:
        return [{**paper, "extracted": self.extract(paper)} for paper in papers]

# synthesis/synthesizer.py
from openai import OpenAI
from typing import Dict, List

class PaperSynthesizer:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.client = OpenAI()
        self.model = model

    def synthesize(self, papers: List[Dict], topic: str) -> Dict:
        summaries = []
        for p in papers:
            ext = p.get("extracted", {})
            summaries.append(f"Paper: {p['title']} ({p.get('year', 'N/A')})")
            summaries.append(f"  Method: {ext.get('methodology', 'N/A')}")
            summaries.append(f"  Findings: {', '.join(ext.get('key_findings', [])[:3])}")
        papers_text = "\n".join(summaries)
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": """Synthesize findings across multiple papers.
                Identify: consensus, contradictions, methodological trends, and overall narrative.
                Use academic writing style with citations (Author, Year)."""},
                {"role": "user", "content": f"Topic: {topic}\n\nPapers:\n{papers_text}\n\nSynthesize the literature:"},
            ],
            temperature=0.3,
        )
        return {
            "topic": topic,
            "num_papers": len(papers),
            "synthesis": response.choices[0].message.content,
        }

# synthesis/gap_analyzer.py
from openai import OpenAI
from typing import Dict, List

class GapAnalyzer:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.client = OpenAI()
        self.model = model

    def analyze_gaps(self, papers: List[Dict], topic: str) -> Dict:
        findings = []
        for p in papers:
            ext = p.get("extracted", {})
            findings.append(f"- {p['title']}: {', '.join(ext.get('key_findings', [])[:2])}")
        findings_text = "\n".join(findings)
        response = self.client.chat.completions.create(
            model=self.model,
            messages=[
                {"role": "system", "content": """Analyze research gaps in this literature.
                Return JSON:
                {
                    "gaps": [{"gap": "description", "importance": "high|medium|low", "suggested_approach": "how to address"}],
                    "underexplored_areas": ["list of areas needing more research"],
                    "methodological_gaps": ["methods not yet applied"],
                    "future_directions": ["promising research directions"]
                }"""},
                {"role": "user", "content": f"Topic: {topic}\n\nFindings:\n{findings_text}"},
            ],
            temperature=0.3,
        )
        try:
            return json.loads(response.choices[0].message.content)
        except:
            return {"gaps": [], "future_directions": []}

# agent.py
from search.arxiv_client import ArxivClient
from search.semantic_scholar import SemanticScholarClient
from extraction.paper_extractor import PaperExtractor
from synthesis.synthesizer import PaperSynthesizer
from synthesis.gap_analyzer import GapAnalyzer
from typing import Dict, List

class LiteratureReviewAgent:
    def __init__(self, model: str = "gpt-4-turbo-preview"):
        self.arxiv = ArxivClient()
        self.s2 = SemanticScholarClient()
        self.extractor = PaperExtractor(model)
        self.synthesizer = PaperSynthesizer(model)
        self.gap_analyzer = GapAnalyzer(model)

    def review(self, topic: str, max_papers: int = 15) -> Dict:
        arxiv_papers = self.arxiv.search(topic, max_results=max_papers)
        s2_papers = self.s2.search(topic, limit=max_papers)
        all_papers = self._deduplicate(arxiv_papers + s2_papers)
        extracted = self.extractor.extract_batch(all_papers[:max_papers])
        synthesis = self.synthesizer.synthesize(extracted, topic)
        gaps = self.gap_analyzer.analyze_gaps(extracted, topic)
        return {
            "topic": topic,
            "total_papers": len(extracted),
            "papers": extracted,
            "synthesis": synthesis,
            "research_gaps": gaps,
        }

    def _deduplicate(self, papers: List[Dict]) -> List[Dict]:
        seen = set()
        unique = []
        for p in papers:
            title = p.get("title", "").lower().strip()
            if title not in seen:
                seen.add(title)
                unique.append(p)
        return unique

    def generate_review_report(self, review_data: Dict) -> str:
        report = f"# Literature Review: {review_data['topic']}\n\n"
        report += f"## Overview\nReviewed {review_data['total_papers']} papers\n\n"
        report += f"## Synthesis\n{review_data['synthesis']['synthesis']}\n\n"
        report += "## Key Papers\n"
        for p in review_data["papers"][:10]:
            report += f"- {p['title']} ({p.get('year', 'N/A')})\n"
        report += f"\n## Research Gaps\n"
        for gap in review_data.get("research_gaps", {}).get("gaps", []):
            report += f"- {gap.get('gap', '')} (Importance: {gap.get('importance', 'medium')})\n"
        return report

import pytest
from search.arxiv_client import ArxivClient

def test_arxiv_search():
    client = ArxivClient()
    papers = client.search("transformer attention mechanism", max_results=5)
    assert len(papers) > 0
    assert "title" in papers[0]

def test_extraction():
    extractor = PaperExtractor()
    result = extractor.extract({"title": "Test", "abstract": "This paper proposes..."})
    assert "research_question" in result

# main.py
from agent import LiteratureReviewAgent

def main():
    agent = LiteratureReviewAgent()
    topic = input("Research topic: ").strip()
    print(f"\nSearching for papers on: {topic}\n")
    review = agent.review(topic)
    report = agent.generate_review_report(review)
    print(report)
    with open("review.md", "w") as f:
        f.write(report)
    print("\nReview saved to review.md")

if __name__ == "__main__":
    main()