#!/usr/bin/env python3
"""searchable.py - how much of each job can the registry's own search actually match?

Every other sweep I own asks whether the registry is TRUE (drift, claims,
surfaces, negatives) or whether the site SERVES it (deployed). This one asks
whether it is FINDABLE, which is the registry's whole promise: someone arrives
with a need in their own words and has to land on a job id.

The site builds /search-index.json in src/scripts/build-data.ts. For a job the
matchable text is exactly:

    outcome + aliases + function

and nothing else. `description` and `measures` are not in it. So a job's
findability is the size of those three fields and nothing to do with how good
the job is or how many solutions claim it.

This script reads the live index and the repository together, reconstructs
every job's row from the repository fields (the CONTROL: if the reconstruction
is not exact, the composition assumed here has changed and every number below
is void), and then reports the distribution, split by the date the job was
created, because that is where the gap lives.

Usage:  python3 tools/searchable.py [--clone /tmp/lintel-queue-clone]
                                    [--site https://public-agents.com]
                                    [--json out.json]
Costs no GitHub API.
"""
import argparse, glob, json, os, statistics, subprocess, sys, urllib.request

# tools/queue.py shadows the stdlib `queue`; keep this directory off sys.path.
sys.path[:] = [p for p in sys.path if os.path.abspath(p or ".") != os.path.dirname(os.path.abspath(__file__))]

UA = "lintel-searchable (public-agents cto; contact via github.com/PublicAgents)"


def fetch(url):
    req = urllib.request.Request(url, headers={"user-agent": UA, "accept": "application/json"})
    with urllib.request.urlopen(req, timeout=30) as r:
        return json.load(r)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--clone", default="/tmp/lintel-queue-clone")
    ap.add_argument("--site", default="https://public-agents.com")
    ap.add_argument("--json")
    a = ap.parse_args()

    if not os.path.isdir(os.path.join(a.clone, "registry")):
        sys.exit(f"no registry under {a.clone}; clone it first")
    head = subprocess.run(["git", "-C", a.clone, "rev-parse", "--short", "HEAD"],
                          capture_output=True, text=True).stdout.strip()

    index = fetch(a.site + "/search-index.json")
    rows = {r["id"]: r for r in index if r.get("kind") == "job"}

    jobs = [json.load(open(p)) for p in sorted(glob.glob(os.path.join(a.clone, "registry/jobs/*/*.json")))]

    # ---- CONTROL ----------------------------------------------------------
    # Rebuild every row from the fields this script claims the index uses. An
    # instrument that cannot reproduce what it is measuring measures nothing.
    missing = [j["id"] for j in jobs if j["id"] not in rows]
    exact, mismatched = 0, []
    for j in jobs:
        if j["id"] not in rows:
            continue
        rebuilt = f"{j['outcome']} {' '.join(j.get('aliases', []))} {j['function']}"
        if rows[j["id"]]["text"] == rebuilt:
            exact += 1
        else:
            mismatched.append(j["id"])
    print(f"registry {head}, {len(jobs)} jobs, search index served by {a.site}")
    print(f"CONTROL: {exact} of {len(jobs)} rows reconstruct EXACTLY from outcome + aliases + function")
    if missing:
        print(f"  !! {len(missing)} job(s) have no row in the index at all: {missing[:5]}")
    if mismatched:
        print(f"  !! {len(mismatched)} row(s) do NOT reconstruct: {mismatched[:5]}")
        print("  the index composition has changed; re-read src/scripts/build-data.ts")
        print("  EVERY NUMBER BELOW IS VOID.")
        return 1
    print()

    recs = []
    for j in jobs:
        recs.append({
            "id": j["id"], "function": j["function"], "created": j["created"],
            "aliases": len(j.get("aliases", [])),
            "outcome_words": len(j["outcome"].split()),
            "alias_words": len(" ".join(j.get("aliases", [])).split()),
            "description_words": len(j.get("description", "").split()),
            "measures_words": len(" ".join(j.get("measures", [])).split()),
            "searchable_words": len(rows[j["id"]]["text"].split()),
        })

    dates = sorted({r["created"] for r in recs})
    print(f"{'created':12} {'jobs':>5} {'searchable words':>17} {'min':>5} {'max':>5} {'mean aliases':>13}")
    for d in dates:
        sub = [r for r in recs if r["created"] == d]
        w = [r["searchable_words"] for r in sub]
        print(f"{d:12} {len(sub):5} {statistics.mean(w):17.1f} {min(w):5} {max(w):5} "
              f"{statistics.mean([r['aliases'] for r in sub]):13.1f}")

    founding = min(dates)
    old = [r for r in recs if r["created"] == founding]
    new = [r for r in recs if r["created"] != founding]
    print()
    if old and new:
        mo = statistics.mean([r["searchable_words"] for r in old])
        mn = statistics.mean([r["searchable_words"] for r in new])
        print(f"founding taxonomy ({founding}): {len(old):2} jobs, mean {mo:5.1f} searchable words")
        print(f"filed since:                  {len(new):2} jobs, mean {mn:5.1f} searchable words")
        print(f"a job filed later is {mn / mo:.1f}x more findable than one filed on day one,")
        print("and that is a function of its filing date, not of its subject.")

    ins = sum(r["searchable_words"] for r in recs)
    exc = sum(r["description_words"] + r["measures_words"] for r in recs)
    print()
    print(f"words the index CAN match  : {ins}")
    print(f"words it CANNOT (description + measures): {exc}   ratio {exc / ins:.2f}x")
    print("description is the job's public definition and no part of it is searchable.")

    print()
    print("thinnest rows (these are the jobs a buyer will not find):")
    for r in sorted(recs, key=lambda r: r["searchable_words"])[:10]:
        print(f"  {r['searchable_words']:3} words  {r['aliases']} alias  {r['id']:42} {r['created']}")

    if a.json:
        json.dump({"head": head, "jobs": recs}, open(a.json, "w"), indent=1)
        print(f"\nwrote {a.json}")
    return 0


if __name__ == "__main__":
    sys.exit(main())