#!/usr/bin/env python3
"""
Company profile extractor for advertising intelligence.

Usage:
  export LLM_API_KEY="your_key_here"
  export LLM_MODEL="gpt-4.1-mini"
  python3 company_profile_extractor.py "https://www.completetrailersco.com/" --output profile.json
"""

from __future__ import annotations

import argparse
import json
import os
import re
import sys
import xml.etree.ElementTree as ET
from collections import deque
from dataclasses import dataclass
from typing import Dict, List, Set
from urllib.parse import urljoin, urlparse

import requests
from bs4 import BeautifulSoup


DEFAULT_BASE_URL = "https://api.openai.com/v1"
DEFAULT_MODEL = "gpt-4.1-mini"
USER_AGENT = "Mozilla/5.0 (compatible; CompanyProfileExtractor/1.0)"
MAX_TEXT_CHARS_PER_PAGE = 7000
DEFAULT_MAX_PAGES = 6
MAX_COMPETITOR_SEARCH_RESULTS = 8
COMPETITOR_SEARCH_URL = "https://www.bing.com/search"

EXCLUDED_COMPETITOR_DOMAINS = {
    "facebook.com",
    "instagram.com",
    "linkedin.com",
    "twitter.com",
    "x.com",
    "youtube.com",
    "wikipedia.org",
    "yelp.com",
}

PRIORITY_HINTS = (
    "about",
    "company",
    "about-us",
    "service",
    "services",
    "product",
    "inventory",
    "solutions",
    "industries",
    "market",
    "contact",
    "locations",
    "investor",
    "financial",
    "press",
    "news",
    "our-story",
)


@dataclass
class PageContent:
    url: str
    title: str
    text: str


def normalize_url(url: str) -> str:
    parsed = urlparse(url)
    scheme = parsed.scheme or "https"
    netloc = parsed.netloc
    path = parsed.path or "/"
    if not netloc and parsed.path:
        # Handle inputs like "example.com"
        netloc = parsed.path
        path = "/"
    return f"{scheme}://{netloc}{path}"


def same_domain(url_a: str, url_b: str) -> bool:
    return urlparse(url_a).netloc.lower() == urlparse(url_b).netloc.lower()


def host_of(url: str) -> str:
    return urlparse(url).netloc.lower().replace("www.", "")


def infer_company_name(start_url: str, pages: List[PageContent]) -> str:
    for p in pages:
        title = (p.title or "").strip()
        if not title or title.lower() == "unknown":
            continue
        # Strip common title suffix patterns.
        for sep in ("|", "-", " - ", " | ", " :: "):
            if sep in title:
                title = title.split(sep)[0].strip()
                break
        if title:
            return title

    hostname = host_of(start_url)
    if not hostname:
        return "Unknown"
    first_label = hostname.split(".")[0]
    return first_label.replace("-", " ").strip().title()


def fetch_html(url: str, timeout: int = 20) -> str:
    resp = requests.get(url, timeout=timeout, headers={"User-Agent": USER_AGENT})
    resp.raise_for_status()
    return resp.text


def extract_text(html: str) -> str:
    soup = BeautifulSoup(html, "html.parser")
    for tag in soup(["script", "style", "noscript", "svg"]):
        tag.decompose()
    text = soup.get_text(" ", strip=True)
    text = re.sub(r"\s+", " ", text).strip()
    return text[:MAX_TEXT_CHARS_PER_PAGE]


def extract_title(html: str) -> str:
    soup = BeautifulSoup(html, "html.parser")
    if soup.title and soup.title.string:
        return soup.title.string.strip()
    return "Unknown"


def score_url(url: str) -> int:
    lower = url.lower()
    score = 0
    for hint in PRIORITY_HINTS:
        if hint in lower:
            score += 2
    if lower.endswith((".pdf", ".jpg", ".jpeg", ".png", ".gif", ".zip")):
        score -= 10
    return score


def collect_internal_links(base_url: str, html: str) -> List[str]:
    soup = BeautifulSoup(html, "html.parser")
    links: Set[str] = set()
    for a in soup.select("a[href]"):
        href = a.get("href", "").strip()
        if not href or href.startswith("#") or href.startswith("mailto:") or href.startswith("tel:"):
            continue
        full = urljoin(base_url, href)
        if same_domain(base_url, full):
            links.add(full.split("#")[0])
    return sorted(links, key=score_url, reverse=True)


def crawl_site(start_url: str, max_pages: int) -> List[PageContent]:
    visited: Set[str] = set()
    queue: deque[str] = deque([start_url])
    pages: List[PageContent] = []

    while queue and len(pages) < max_pages:
        current = queue.popleft()
        if current in visited:
            continue
        visited.add(current)

        try:
            html = fetch_html(current)
            text = extract_text(html)
            if not text:
                continue
            pages.append(PageContent(url=current, title=extract_title(html), text=text))

            # Add scored links to queue
            for link in collect_internal_links(start_url, html):
                if link not in visited and len(queue) < 50:
                    queue.append(link)
        except Exception as exc:  # noqa: BLE001
            print(f"[warn] Skipping {current}: {exc}", file=sys.stderr)

    return pages


def search_competitors(company_name: str, target_url: str, max_results: int = MAX_COMPETITOR_SEARCH_RESULTS) -> List[Dict[str, str]]:
    query = f'"{company_name}" competitors alternatives'
    params = {"q": query, "format": "rss"}
    resp = requests.get(COMPETITOR_SEARCH_URL, params=params, timeout=20, headers={"User-Agent": USER_AGENT})
    resp.raise_for_status()

    results: List[Dict[str, str]] = []
    target_host = host_of(target_url)

    root = ET.fromstring(resp.text)
    for item in root.findall(".//item"):
        title = (item.findtext("title") or "").strip()
        link = (item.findtext("link") or "").strip()
        snippet = (item.findtext("description") or "").strip()
        if not link:
            continue
        try:
            parsed = urlparse(link)
            if parsed.scheme not in ("http", "https"):
                continue
        except Exception:  # noqa: BLE001
            continue

        result_host = host_of(link)
        if not result_host or result_host == target_host:
            continue
        if any(result_host == d or result_host.endswith(f".{d}") for d in EXCLUDED_COMPETITOR_DOMAINS):
            continue

        results.append(
            {
                "title": title or "Unknown",
                "url": link,
                "snippet": re.sub(r"\s+", " ", snippet).strip(),
            }
        )
        if len(results) >= max_results:
            break

    return results


def build_prompt(target_url: str, pages: List[PageContent], competitor_results: List[Dict[str, str]]) -> str:
    source_payload: List[Dict[str, str]] = []
    for p in pages:
        source_payload.append(
            {
                "url": p.url,
                "title": p.title,
                "text": p.text,
            }
        )

    schema = {
        "company_name": "string",
        "company_profile_overview": "string",
        "estimated_company_size_range": {
            "range": "string",
            "confidence": "Low|Medium|High",
            "rationale": "string",
        },
        "annual_revenue_estimate": {
            "range": "string",
            "confidence": "Low|Medium|High",
            "rationale": "string",
        },
        "main_competitors": [
            {
                "company_name": "string",
                "website": "string",
                "reason_competitor": "string",
                "confidence": "Low|Medium|High",
                "evidence": ["string"],
            }
        ],
        "locations": ["string"],
        "products_services": ["string"],
        "target_market": ["string"],
        "target_audiences": ["string"],
        "demographic_geographic_profile": {
            "geographic_focus": ["string"],
            "buyer_demographic_signals": ["string"],
        },
        "psychographic_profile": {
            "core_motivations": ["string"],
            "purchase_triggers": ["string"],
            "brand_preferences": ["string"],
        },
        "confidence_notes": ["string"],
        "evidence_quotes": [{"quote": "string", "source_url": "string"}],
        "source_urls": ["string"],
    }

    instructions = (
        "You are a B2B advertising intelligence analyst.\n"
        "Create an accurate company profile from provided website data.\n"
        "Rules:\n"
        "1) Use only provided sources.\n"
        "2) If unknown, return 'Unknown' and explain briefly in confidence_notes.\n"
        "3) Do not hallucinate company size, customer count, locations, exact revenue values, or competitors.\n"
        "4) For annual revenue, return a broad range only when supported by explicit signals in the provided text; otherwise set annual_revenue_estimate.range to 'Unknown'.\n"
        "5) Include revenue-specific rationale and confidence in annual_revenue_estimate.\n"
        "6) For main_competitors, use competitor search results plus site context; include only likely true direct competitors and set confidence conservatively.\n"
        "7) Return strict JSON only with the exact schema.\n\n"
        f"Schema:\n{json.dumps(schema, indent=2)}\n\n"
        f"Target URL: {target_url}\n"
        f"Crawled Sources JSON:\n{json.dumps(source_payload, indent=2)}\n\n"
        f"Competitor Search Results JSON:\n{json.dumps(competitor_results, indent=2)}"
    )
    return instructions


def call_model(prompt: str, model: str, api_key: str, base_url: str) -> str:
    endpoint = f"{base_url.rstrip('/')}/responses"
    payload = {
        "model": model,
        "input": prompt,
    }
    headers = {
        "Authorization": f"Bearer {api_key}",
        "Content-Type": "application/json",
    }
    response = requests.post(endpoint, json=payload, headers=headers, timeout=120)
    response.raise_for_status()
    data = response.json()

    if isinstance(data.get("output_text"), str) and data["output_text"].strip():
        return data["output_text"].strip()

    # Fallback parser for providers that do not include output_text
    chunks: List[str] = []
    for item in data.get("output", []):
        for content in item.get("content", []):
            text = content.get("text")
            if text:
                chunks.append(text)
    return "\n".join(chunks).strip()


def parse_json_output(raw_text: str) -> Dict:
    try:
        return json.loads(raw_text)
    except json.JSONDecodeError:
        # Attempt to salvage JSON block from noisy output
        match = re.search(r"\{.*\}", raw_text, flags=re.DOTALL)
        if not match:
            raise
        return json.loads(match.group(0))


def main() -> int:
    parser = argparse.ArgumentParser(description="Extract company profile from public website pages.")
    parser.add_argument("url", help="Company website URL")
    parser.add_argument("--max-pages", type=int, default=DEFAULT_MAX_PAGES, help="Maximum pages to crawl")
    parser.add_argument("--output", default="company-profile.json", help="Path to output JSON file")
    parser.add_argument("--model", default=os.getenv("LLM_MODEL", DEFAULT_MODEL), help="LLM model name")
    parser.add_argument("--base-url", default=os.getenv("LLM_BASE_URL", DEFAULT_BASE_URL), help="LLM API base URL")
    parser.add_argument("--api-key", default=os.getenv("LLM_API_KEY") or os.getenv("OPENAI_API_KEY"), help="LLM API key")
    args = parser.parse_args()

    if not args.api_key:
        print(
            "Missing API key. Set LLM_API_KEY (or OPENAI_API_KEY) and retry.",
            file=sys.stderr,
        )
        return 1

    target_url = normalize_url(args.url)
    print(f"[info] Crawling: {target_url}", file=sys.stderr)
    pages = crawl_site(target_url, max_pages=args.max_pages)
    if not pages:
        print("No pages were crawled. Check URL/network access.", file=sys.stderr)
        return 1

    company_name = infer_company_name(target_url, pages)
    competitor_results: List[Dict[str, str]] = []
    try:
        competitor_results = search_competitors(company_name=company_name, target_url=target_url)
    except Exception as exc:  # noqa: BLE001
        print(f"[warn] Competitor search failed: {exc}", file=sys.stderr)

    print(
        f"[info] Collected {len(pages)} pages and {len(competitor_results)} competitor search results",
        file=sys.stderr,
    )
    prompt = build_prompt(target_url, pages, competitor_results)
    raw = call_model(prompt, model=args.model, api_key=args.api_key, base_url=args.base_url)
    profile = parse_json_output(raw)

    with open(args.output, "w", encoding="utf-8") as fh:
        json.dump(profile, fh, indent=2, ensure_ascii=True)

    print(f"[ok] Wrote profile to {args.output}", file=sys.stderr)
    print(json.dumps(profile, indent=2, ensure_ascii=True))
    return 0


if __name__ == "__main__":
    raise SystemExit(main())