#!/usr/bin/env python3
"""Mine rich personality, social graph, and life context from AIM archive logs."""

from __future__ import annotations

import argparse
import gzip
import html
import json
import random
import re
import subprocess
from collections import Counter, defaultdict
from pathlib import Path

DB = {
    "host": "127.0.0.1",
    "user": "convo",
    "password": "b9db74bfda4d4e73",
    "database": "convo",
}

CONAN_ID = 10
PARTNER_KEY = "conan0fthenight"
PERSONAS_ROOT = Path("/opt/aim-bots/personas")
MANIFEST = Path("/opt/aim-bots/side_chicks_manifest.json")

SPEAKER_RE = re.compile(
    r"(?:Auto response from )?([A-Za-z0-9][A-Za-z0-9 ]{0,40})<!--\s*\([^)]+\)\s*-->",
    re.I,
)

KNOWN_CREW = {
    "malachi", "andrew", "lydia", "amy", "keefe", "charles", "dag", "joy", "spencer",
    "erica", "gioia", "michelle", "cindy", "katelyn", "taylor", "katie", "steph",
    "marissa", "alex", "alexis", "bekah", "mary", "tori", "leo", "eric", "mal",
}

TRAIT_PATTERNS = {
    "flirtatious": r"\b(hot|sexy|cute|pic|n00d|kiss|babe|baby|flirt|handsome|gorgeous)\b",
    "lazy": r"\b(bored|tired|lazy|sleepy|dont wanna|don't wanna|nothing to do)\b",
    "dramatic": r"\b(om+g|!!|hate her|hate him|so mad|pissed|furious|devastated)\b",
    "sweet": r"\b(<3|aww|sweet|nice of you|thank you|thanks)\b",
    "sarcastic": r"\b(yeah right|whatever|sureee|oh wow|nice one|smooth)\b",
    "emo": r"\b(sad|depressed|alone|cry|crying|miserable|empty)\b",
    "religious": r"\b(church|god|youth group|bible|christian|jesus|pray)\b",
    "nerdy": r"\b(wow|warcraft|guild|computer|program|code|firefox|linux)\b",
    "artistic": r"\b(design|draw|art|tattoo|piercing|photo|music|band|song)\b",
    "graphic": r"\b(dick|sex|naked|nude|horny|hard)\b",
    "petty": r"\b(bitch|stupid|annoying|drama|jealous|ugh)\b",
    "goofy": r"\b(haha|rotfl|x[dD]|=p|=P|lol|heyy+|yeahhh+)\b",
    "supportive": r"\b(i got you|anytime|here for you|good luck|proud|sorry to hear)\b",
    "planner": r"\b(ride|pick you up|hang out|come over|see you tomorrow|call me)\b",
}

TOPIC_PATTERNS = {
    "music": r"\b(music|band|song|listen|cd|itunes|concert|show|guitar|relient)\b",
    "school": r"\b(school|class|homework|teacher|test|exam|study hall|diploma)\b",
    "work": r"\b(work|job|shift|boss|hire|pay|hour)\b",
    "myspace": r"\b(myspace|layout|bulletin|comment|top 8|top8|profile)\b",
    "church": r"\b(church|youth|god|bible|christian)\b",
    "wow": r"\b(wow|warcraft|guild|paladin|raid|horde|alliance)\b",
    "flirting": r"\b(hot|sexy|cute|pic|n00d|kiss|babe|flirt|handsome)\b",
    "family": r"\b(mom|dad|brother|sister|cousin|parent|family)\b",
    "food": r"\b(hungry|eat|food|pizza|starbucks|poptart|tater tot)\b",
    "drama": r"\b(drama|hate|fight|broke up|dating|boyfriend|girlfriend|ex)\b",
    "rides": r"\b(ride|drive|pick up|car|gas)\b",
    "pics": r"\b(pic|photo|picture|webcam|cam)\b",
    "tattoos": r"\b(tattoo|piercing|ink|pierced)\b",
    "shows": r"\b(maylene|chariot|haste|norma|emery|metal|hardcore)\b",
    "4chan": r"\b(/b/|4chan|imagechan|ebaums|owned)\b",
}

LIFE_PATTERNS = [
    (r"\bi(?:'m| am) (\d{1,2})\b", "age_mention"),
    (r"\bmy (?:name is|names) (\w+)\b", "name"),
    (r"\bmy (cousin|friend|sister|brother|mom|dad|best friend) (\w+)", "relative"),
    (r"\bi (?:work|worked) (?:at|for) ([^.?!]{5,60})", "job"),
    (r"\bi go to ([^.?!]{5,50}) school", "school_name"),
    (r"\bi (?:study|studied|major in) ([^.?!]{5,60})", "studies"),
    (r"\bi (?:live|lived) (?:in|near) ([^.?!]{5,40})", "location"),
]

REAL_NAMES = {}
if MANIFEST.exists():
    REAL_NAMES = json.loads(MANIFEST.read_text()).get("real_names_known", {})
REAL_NAMES.update({"magus5311": "Spencer", "soewaslike": "Erica", "dudesy7777": "Lydia"})

ACCOUNT_ALIASES = {
    "magus5311": ("magus5311", "magusishiding"),
    "magusishiding": ("magus5311", "magusishiding"),
    "soewaslike": ("soewaslike",),
}


def account_key(name: str) -> str:
    """Normalize screenname for DB account lookup — do NOT merge aliases here."""
    return re.sub(r"\s+", "", name.lower())


def norm(name: str) -> str:
    n = account_key(name)
    if n == "magusishiding":
        return "magus5311"
    return n


def strip_html(text: str) -> str:
    text = re.sub(r"<!--.*?-->", "", text, flags=re.S)
    text = re.sub(r"<[^>]+>", " ", text)
    text = html.unescape(text)
    text = re.sub(r"^[:\s>]+", "", text)
    text = re.sub(r"\s+", " ", text).strip()
    if re.search(r"signed (on|off)|is away|returned at|is idle at", text, re.I):
        return ""
    return text


def mysql_query(sql: str) -> str:
    return subprocess.check_output(
        ["mysql", "-u", DB["user"], f"-p{DB['password']}", DB["database"], "-N", "-B", "-e", sql],
        text=True,
        errors="replace",
    )


def mysql_rows(sql: str) -> list[str]:
    out = mysql_query(sql)
    return [line for line in out.splitlines() if line.strip()]


def load_accounts() -> tuple[dict[int, str], dict[str, int]]:
    rows = mysql_rows("SELECT id, screenname FROM accounts")
    id_to_name: dict[int, str] = {}
    name_to_id: dict[str, int] = {}
    for row in rows:
        aid, sn = row.split("\t", 1)
        id_to_name[int(aid)] = sn
        name_to_id[account_key(sn)] = int(aid)
    return id_to_name, name_to_id


def parse_conversation(content: str) -> list[tuple[str, str]]:
    entries = []
    matches = list(SPEAKER_RE.finditer(content))
    for i, match in enumerate(matches):
        speaker = norm(match.group(1))
        start = match.end()
        end = matches[i + 1].start() if i + 1 < len(matches) else len(content)
        text = strip_html(content[start:end])
        if text:
            entries.append((speaker, text))
    return entries


def is_good_message(text: str) -> bool:
    if len(text) < 2 or len(text) > 180:
        return False
    if text.startswith("http") or "http://" in text or "www." in text:
        return False
    if re.search(r"[\x00-\x08\x0b\x0c\x0e-\x1f]", text):
        return False
    return True


def build_pairs(entries: list[tuple[str, str]], persona_key: str) -> list[dict]:
    pairs = []
    for i, (speaker, text) in enumerate(entries):
        if speaker == persona_key and i > 0 and entries[i - 1][0] == PARTNER_KEY:
            pairs.append({"user": entries[i - 1][1], "assistant": text})
    return pairs


def score_traits(msgs: list[str]) -> list[tuple[str, float]]:
    scores: Counter = Counter()
    for msg in msgs:
        for trait, pat in TRAIT_PATTERNS.items():
            if re.search(pat, msg, re.I):
                scores[trait] += 1
    total = max(len(msgs), 1)
    ranked = [(t, round(c / total, 4)) for t, c in scores.most_common()]
    return [(t, s) for t, s in ranked if s >= 0.005][:8]


def extract_topics(msgs: list[str]) -> dict[str, list[str]]:
    hits: dict[str, list[str]] = defaultdict(list)
    for msg in msgs:
        for topic, pat in TOPIC_PATTERNS.items():
            if re.search(pat, msg, re.I) and len(msg) >= 8:
                if len(hits[topic]) < 4:
                    hits[topic].append(msg[:140])
    return dict(hits)


def extract_life_details(msgs: list[str]) -> list[str]:
    details: list[str] = []
    seen: set[str] = set()
    for msg in msgs:
        if len(msg.split()) < 5:
            continue
        for pat, _kind in LIFE_PATTERNS:
            m = re.search(pat, msg, re.I)
            if m:
                detail = msg[:160].strip()
                key = detail.lower()[:60]
                if key not in seen:
                    seen.add(key)
                    details.append(detail)
        if len(details) >= 20:
            break
    return details[:15]


def extract_people(msgs: list[str], name_to_id: dict[str, int], persona_key: str) -> list[dict]:
    mention_counts: Counter = Counter()
    mention_samples: dict[str, str] = {}

    for msg in msgs:
        low = msg.lower()
        for nkey, aid in name_to_id.items():
            if nkey in (persona_key, PARTNER_KEY) or len(nkey) < 5:
                continue
            if re.search(rf"\b{re.escape(nkey)}\b", low.replace(" ", "")):
                mention_counts[nkey] += 1
                if nkey not in mention_samples:
                    mention_samples[nkey] = msg[:120]
        for crew in KNOWN_CREW:
            if re.search(rf"\b{crew}\b", low):
                mention_counts[crew] += 1
                if crew not in mention_samples:
                    mention_samples[crew] = msg[:120]

    people = []
    for name, count in mention_counts.most_common(20):
        if count < 1:
            continue
        display = name
        if name in name_to_id:
            # recover original casing from id lookup would need id_to_name reverse
            pass
        people.append({
            "name": display.title() if name in KNOWN_CREW else display,
            "mentions": count,
            "sample": mention_samples.get(name, ""),
        })
    return people


def extract_social_roles(msgs: list[str]) -> dict[str, str]:
    roles: dict[str, str] = {}
    role_patterns = [
        (r"\bmy cousin\b", "cousin", "close cousin — often around"),
        (r"\bmy best friend\b", "best_friend", "best friend"),
        (r"\bmy (?:mom|mother)\b", "mom", "mom"),
        (r"\bmy (?:dad|father)\b", "dad", "dad"),
        (r"\bmy (?:sister|brother)\b", "sibling", "sibling"),
        (r"\bjoy\b", "joy", "Joy — best friend, carpool, sleepovers"),
        (r"\bamy\b", "amy", "Amy — Spencer's girlfriend, in the crew"),
        (r"\bmalachi\b", "malachi", "Malachi — show crew"),
        (r"\bandrew\b", "andrew", "Andrew — show crew"),
        (r"\blydia\b", "lydia", "Lydia — in the friend group"),
        (r"\bspencer\b", "spencer", "Spencer (magus5311) — Caleb's best friend"),
        (r"\berica\b", "erica", "Erica (soewaslike) — casual history with Caleb"),
    ]
    for msg in msgs:
        low = msg.lower()
        for pat, key, desc in role_patterns:
            if key not in roles and re.search(pat, low):
                roles[key] = desc
    return roles


def relationship_with_caleb(msgs: list[str], pairs: list[dict], persona_kind: str, screen: str) -> dict:
    caleb_mentions = sum(1 for m in msgs if re.search(r"\bcaleb\b", m, re.I))
    flirt_hits = sum(1 for m in msgs if re.search(TRAIT_PATTERNS["flirtatious"], m, re.I))
    name_elong = sum(1 for m in msgs if re.search(r"caleb{3,}", m, re.I))
    avg_words = round(sum(len(m.split()) for m in msgs) / max(len(msgs), 1), 2)

    if persona_kind == "wife" or screen == "dudesy7777":
        dynamic = "Future wife energy — warm, present, best-friend intimacy becoming partnership"
    elif screen == "magus5311":
        dynamic = "Best friend since age 3 — zero filter, sarcasm is love, never say his name much"
    elif screen == "soewaslike":
        dynamic = "Casual hookup/flirty history — never official boyfriend/girlfriend, bored AIM friend"
    elif persona_kind == "side_chick":
        dynamic = "Casual side flirt — random hits, playful not clingy, NOT his girlfriend"
    else:
        dynamic = "AIM friend — keep it casual and in-character from your logs"

    notes = []
    if caleb_mentions > 10:
        notes.append(f"says 'caleb' naturally ({caleb_mentions} times in archive)")
    if name_elong > 2:
        notes.append("elongates his name cute: calebbbb")
    if flirt_hits / max(len(msgs), 1) > 0.05:
        notes.append("flirty banter is normal in your logs")
    if avg_words > 6:
        notes.append("tends toward longer thoughtful replies than typical AIM")
    elif avg_words < 3:
        notes.append("mostly ultra-short AIM bursts")

    return {
        "dynamic": dynamic,
        "caleb_name_usage": caleb_mentions,
        "flirt_ratio": round(flirt_hits / max(len(msgs), 1), 4),
        "avg_words": avg_words,
        "notes": notes,
    }


def personality_summary(traits: list[tuple[str, float]], topics: dict, persona_kind: str) -> str:
    top = [t for t, _ in traits[:4]]
    if not top:
        top = ["casual", "bored AIM energy"]
    topic_names = list(topics.keys())[:4]
    bits = [", ".join(top)]
    if topic_names:
        bits.append("into " + ", ".join(topic_names))
    if persona_kind == "side_chick":
        bits.append("myspace-era random side piece")
    elif persona_kind == "wife":
        bits.append("real emotional depth")
    return " — ".join(bits)


def curate_rich_examples(pairs: list[dict], limit: int = 25) -> list[dict]:
    scored = []
    boring = re.compile(r"^(x[dD]|haha|lol|yeah|ok+|mhm|sup|hey|yo)$", re.I)
    for p in pairs:
        u, a = p["user"], p["assistant"]
        if not is_good_message(u) or not is_good_message(a):
            continue
        if boring.match(a.strip()):
            continue
        score = 0
        if 2 <= len(a.split()) <= 12:
            score += 2
        if re.search(r"[.!?…]|,|caleb|my |i'm|i am|love|hate|school|work|church|joy|amy", a, re.I):
            score += 2
        if len(u.split()) >= 3:
            score += 1
        scored.append((score, p))
    scored.sort(key=lambda x: (-x[0], len(x[1]["assistant"])))
    seen: set[str] = set()
    out = []
    for _, p in scored:
        key = p["assistant"].lower()[:50]
        if key in seen:
            continue
        seen.add(key)
        out.append(p)
        if len(out) >= limit:
            break
    return out


def extract_scene_samples(entries: list[tuple[str, str]], persona_key: str, limit: int = 6) -> list[dict]:
    scenes = []
    i = 0
    while i < len(entries) - 2 and len(scenes) < limit:
        if entries[i][0] == PARTNER_KEY:
            burst_user = [entries[i][1]]
            j = i + 1
            assistant_burst = []
            while j < len(entries) and entries[j][0] == persona_key:
                assistant_burst.append(entries[j][1])
                j += 1
            if assistant_burst and any(len(m.split()) >= 4 for m in assistant_burst):
                scenes.append({
                    "user": burst_user[-1][:120],
                    "assistant": [m[:120] for m in assistant_burst[:4]],
                })
            i = j
        else:
            i += 1
    return scenes


def synthesize_backstory(
    screen: str,
    real_name: str,
    display: str,
    persona_kind: str,
    traits: list[tuple[str, float]],
    topics: dict,
    people: list[dict],
    life_details: list[str],
    social_roles: dict[str, str],
    relationship: dict,
) -> str:
    rn = real_name or display
    parts = [f"Your name is {rn} (AIM: {display})."]

    if persona_kind == "wife":
        parts.append(
            "You and Caleb are becoming each other's person — this AIM friendship is the foundation "
            "of a marriage. Warm, goofy, honest. Joy is your best friend. NEVER side-chick energy."
        )
    elif persona_kind == "side_chick":
        parts.append(
            "You and Caleb flirt on AIM — not boyfriend/girlfriend, casual side thing. "
            "Random bored hits, myspace era, playful not clingy."
        )
    elif screen == "magus5311":
        parts.append(
            "You are Spencer, Caleb's best friend since age 3. Zero filter, /b/ humor, metal shows with the crew. "
            "Almost never say his name."
        )
    elif screen == "soewaslike":
        parts.append(
            "You are Erica — casual hookup/flirty history with Caleb, never official. "
            "Bored AIM, simpsons, petty sometimes."
        )

    if traits:
        parts.append(f"Personality: {', '.join(t for t, _ in traits[:5])}.")

    if social_roles:
        role_bits = [f"{k}: {v}" for k, v in list(social_roles.items())[:6]]
        parts.append("People in your world: " + "; ".join(role_bits) + ".")

    if life_details:
        parts.append("Real life from your logs: " + " | ".join(life_details[:4]) + ".")

    if topics:
        parts.append("You actually talk about: " + ", ".join(list(topics.keys())[:8]) + ".")

    notes = relationship.get("notes", [])
    if notes:
        parts.append("With Caleb: " + "; ".join(notes[:3]) + ".")

    return " ".join(parts)


def synthesize_profile_blurb(traits: list[tuple[str, float]], topics: dict, real_name: str) -> str:
    name = real_name.lower() if real_name else ""
    topic_bits = list(topics.keys())[:3]
    trait_bits = [t for t, _ in traits[:2]]
    chunks = [c for c in [name] + topic_bits + trait_bits if c]
    if not chunks:
        return "bored . msg me :]"
    return " . ".join(chunks[:5]) + " . msg me"


def load_persona_kind(persona_dir: Path) -> str:
    persona_path = persona_dir / "persona.json"
    if persona_path.exists():
        return json.loads(persona_path.read_text()).get("persona_kind", "")
    voice_path = persona_dir / "voice.json"
    if voice_path.exists():
        return json.loads(voice_path.read_text()).get("persona_kind", "")
    return ""


def extract_for_persona(persona_key: str, name_to_id: dict[str, int], preserve_manual: bool = True) -> None:
    persona_dir = PERSONAS_ROOT / persona_key
    if not persona_dir.exists():
        raise SystemExit(f"No persona dir: {persona_dir}")

    alias_keys = ACCOUNT_ALIASES.get(persona_key, (persona_key,))
    account_ids = [name_to_id[k] for k in alias_keys if k in name_to_id]
    if not account_ids:
        raise SystemExit(f"Account not found: {persona_key}")

    id_clause = " OR ".join(
        f"(from_account={aid} AND to_account={CONAN_ID}) OR (from_account={CONAN_ID} AND to_account={aid})"
        for aid in account_ids
    )
    conv_ids = mysql_rows(f"SELECT id FROM conversations WHERE {id_clause} ORDER BY id")

    all_entries: list[tuple[str, str]] = []
    for cid in conv_ids:
        content = mysql_query(f"SELECT content FROM conversations WHERE id={cid}")
        if content:
            all_entries.extend(parse_conversation(content))

    persona_msgs = [t for s, t in all_entries if s == persona_key]
    pairs = build_pairs(all_entries, persona_key)
    persona_kind = load_persona_kind(persona_dir) or (
        "wife" if persona_key == "dudesy7777" else
        "side_chick" if persona_key not in {"magus5311", "soewaslike"} else "friend"
    )

    traits = score_traits(persona_msgs)
    topics = extract_topics(persona_msgs)
    life_details = extract_life_details(persona_msgs)
    people = extract_people(persona_msgs, name_to_id, persona_key)
    social_roles = extract_social_roles(persona_msgs)
    relationship = relationship_with_caleb(persona_msgs, pairs, persona_kind, persona_key)
    scenes = extract_scene_samples(all_entries, persona_key)
    rich_examples = curate_rich_examples(pairs)

    real_name = REAL_NAMES.get(persona_key, "")
    display = persona_key
    persona_path = persona_dir / "persona.json"
    voice_path = persona_dir / "voice.json"
    persona = json.loads(persona_path.read_text()) if persona_path.exists() else {}
    voice = json.loads(voice_path.read_text()) if voice_path.exists() else {}
    display = persona.get("display_name") or voice.get("display_name") or display
    real_name = voice.get("real_name") or persona.get("real_name") or real_name

    deep = {
        "personality_summary": personality_summary(traits, topics, persona_kind),
        "personality_traits": [{"trait": t, "weight": s} for t, s in traits],
        "topics_with_quotes": topics,
        "life_details": life_details,
        "people_mentioned": people,
        "social_roles": social_roles,
        "relationship_with_caleb": relationship,
        "scene_samples": scenes,
        "conv_count": len(conv_ids),
        "message_count": len(persona_msgs),
    }

    backstory = synthesize_backstory(
        persona_key, real_name, display, persona_kind,
        traits, topics, people, life_details, social_roles, relationship,
    )

    people_dict = {p["name"]: p["sample"][:80] for p in people[:8] if p.get("sample")}

    persona["deep_context"] = deep
    if rich_examples:
        persona["examples"] = rich_examples
    persona_path.write_text(json.dumps(persona, indent=2) + "\n")

    if preserve_manual and persona_key in {"magus5311", "dudesy7777"}:
        # Hand-tuned core personas — enrich without replacing core backstory
        voice.setdefault("topical_refs", [])
        for t in list(topics.keys())[:6]:
            if t not in voice["topical_refs"]:
                voice["topical_refs"].append(t)
        voice["people_from_archive"] = people_dict
        voice["deep_personality"] = deep["personality_summary"]
        voice["life_details_from_archive"] = life_details[:10]
        if scenes:
            voice["scene_samples"] = scenes[:4]
    else:
        voice["backstory"] = backstory
        voice["relationship_with_caleb"] = relationship.get("dynamic", "")
        if traits:
            voice["personality_traits"] = [t for t, _ in traits[:6]]
        if topics:
            voice["topics"] = list(dict.fromkeys(list(voice.get("topics", [])) + list(topics.keys())))[:12]
        if people_dict:
            voice["people"] = people_dict
        if life_details:
            voice["life_details"] = life_details[:10]
        if scenes:
            voice["scene_samples"] = scenes[:4]
        if not voice.get("profile_blurb") or persona_kind == "side_chick":
            voice["profile_blurb"] = synthesize_profile_blurb(traits, topics, real_name)

    voice_path.write_text(json.dumps(voice, indent=2) + "\n")
    print(
        f"{persona_key}: {len(persona_msgs)} msgs, traits={[t for t,_ in traits[:3]]}, "
        f"people={len(people)}, topics={list(topics.keys())[:5]}, examples={len(rich_examples)}"
    )


def main() -> None:
    parser = argparse.ArgumentParser(description="Extract deep personality context from archive")
    parser.add_argument("target", nargs="?", default="all", help="screenname or 'all'")
    args = parser.parse_args()

    _, name_to_id = load_accounts()

    if args.target == "all":
        keys = sorted(p.name for p in PERSONAS_ROOT.iterdir() if p.is_dir() and (p / "persona.json").exists())
    else:
        keys = [norm(args.target)]

    for key in keys:
        try:
            extract_for_persona(key, name_to_id)
        except SystemExit as exc:
            print(f"SKIP {key}: {exc}")


if __name__ == "__main__":
    main()