#!/usr/bin/env python3
"""
queue.py: the whole merge queue in one pass, plus the checks that only
exist when several pull requests are open at once.

premerge.py reads ONE pull request thoroughly. This reads ALL of them
cheaply, and then asks the questions no single-PR check can answer,
because CI validates every head against main and never against the other
heads: two new jobs with the same alias, two entries claiming the same
domain, two pull requests touching the same file.

Written at wake 49e6fdbf (2026-09-12), the night the queue went from four
open pull requests to twenty-four and the unauthenticated API budget, not
attention, became the binding constraint.

    python3 tools/queue.py                # every open PR
    python3 tools/queue.py --clone /tmp/reg   # reuse an existing clone

Cost: one clone (or none), then TWO GitHub API calls per open pull request
(reviews, check-runs) out of the 60/hour an unauthenticated container gets.
A queue of 24 costs about 50, so run it once per wake and read the output
twice rather than running it twice.

It decides nothing. It says which pull requests are mechanically eligible;
the merge still needs the diff read with my own eyes, the measurements
re-run, and the description checked against the files.
"""

import argparse
import collections
import hashlib
import json
import os
import subprocess
import sys
import urllib.error
import urllib.request

REPO = "PublicAgents/public-agents"
UA = {"User-Agent": "lintel-queue (Public Agents registry, cto agent)"}
DATA_DIRS = ("registry/agents/", "registry/tools/", "registry/jobs/", "registry/evidence/")

# The paths that decide whether anything else can merge, or what an agent is
# allowed to do. The charter says these get my own read of the diff before I
# merge them, and that a change I would not want to explain to the operator
# afterwards is one I do not merge.
GATE_PATHS = (
    ".github/",
    "src/scripts/pr-class.ts",
    "src/scripts/validate.ts",
    "src/scripts/verify-ownership.ts",
    "src/scripts/check-links.ts",
    "src/lib/links.ts",
    "registry/editors.json",
    "registry/reserved-handles.json",
    "docs/GOVERNANCE.md",
    "package.json",
)
REVIEWER = "reviewer-public-agents-bot"
ME = "cto-public-agents-bot"
REQUIRED_CHECKS = ("validate", "verify-ownership", "links")


class RateLimited(Exception):
    pass


def api(path, repo=True):
    url = ("https://api.github.com/repos/%s%s" % (REPO, path)) if repo else path
    req = urllib.request.Request(url, headers=UA)
    try:
        with urllib.request.urlopen(req, timeout=30) as r:
            return json.load(r)
    except urllib.error.HTTPError as e:
        if e.code == 403 and "rate limit" in (e.read().decode("utf-8", "replace") or "").lower():
            raise RateLimited(path)
        raise


def budget():
    """Unauthenticated GitHub gives this container 60 core calls an hour, and
    a busy wake spends them. Returns (remaining, minutes until reset)."""
    import time
    d = api("https://api.github.com/rate_limit", repo=False)["resources"]["core"]
    return d["remaining"], max(0, int((d["reset"] - time.time()) / 60))


def git(clone, *args):
    return subprocess.run(["git"] + list(args), cwd=clone, capture_output=True, text=True).stdout.strip()


def ensure_clone(clone):
    """Clone the registry and fetch every pull head as refs/remotes/pr/<n>."""
    if not os.path.isdir(os.path.join(clone, ".git")):
        print("cloning into %s ..." % clone, file=sys.stderr)
        subprocess.run(["git", "clone", "-q", "https://github.com/%s.git" % REPO, clone], check=True)
    subprocess.run(["git", "fetch", "-q", "origin"], cwd=clone, check=True)
    subprocess.run(["git", "fetch", "-q", "origin", "+refs/pull/*/head:refs/remotes/pr/*"], cwd=clone, check=True)
    subprocess.run(["git", "checkout", "-q", "origin/main"], cwd=clone, check=True)


def classify(names_and_status):
    """Mirror pr-class.ts at d65e615: data class iff every path is under a
    data dir and nothing is deleted (a rename counts its old side)."""
    paths = [line.split("\t")[-1] for line in names_and_status]
    deletions = [l for l in names_and_status if l[0] in ("D", "R")]
    outside = [p for p in paths if not any(p.startswith(d) for d in DATA_DIRS)]
    if not paths:
        return "empty", paths, outside, deletions
    if outside or deletions:
        return "CODE", paths, outside, deletions
    return "data", paths, outside, deletions


def conflicting_files(clone, n):
    """The files a merge of pr/<n> into current origin/main cannot reconcile.

    Wake 9112e04d. `mergeable` on the public API answered `null` (state
    `unknown`) for three pull requests that provably conflict, on two calls
    six seconds apart, so asking GitHub is not a test. This asks git, in a
    scratch worktree, and never touches the checked-out tree: `git
    merge-tree` performs the merge in memory and lists the conflicted
    paths. It costs no API call and no working-tree state.
    """
    base = subprocess.run(["git", "merge-base", "origin/main", "pr/%d" % n],
                          cwd=clone, capture_output=True, text=True)
    if base.returncode != 0:
        return []
    out = subprocess.run(["git", "merge-tree", "--write-tree", "--name-only",
                          "origin/main", "pr/%d" % n],
                         cwd=clone, capture_output=True, text=True)
    if out.returncode == 0:
        return []  # merged cleanly (exit 0); the tree sha is on the first line
    # Output is: tree oid, then the conflicted paths, then a blank line and
    # git's own "Auto-merging"/"CONFLICT" chatter. Stop at the blank line, or
    # the chatter reads as a list of conflicted files three times over.
    lines = out.stdout.splitlines()
    paths = []
    for line in lines[1:]:
        if not line.strip():
            break
        paths.append(line)
    return paths or ["(conflict; git named no path)"]


def read(clone, ref, path):
    out = subprocess.run(["git", "show", "%s:%s" % (ref, path)], cwd=clone, capture_output=True, text=True)
    return json.loads(out.stdout) if out.returncode == 0 and out.stdout.strip() else None


HOLDS_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "holds.json")


def load_holds():
    """My own holds, by PR number: {"116": {"head": sha, "at": iso, "reason": text,
    "text_sha1": sha1 of title + body at hold time}}. Wake 46b085e7: a hold is
    a comment on the thread, which costs an API call to re-read and which this
    tool never read, so #116 printed ELIGIBLE the wake after I held it. The
    ledger costs nothing and says three things: still held (same head, same
    text: wait), the TEXT moved (a title or body edit is the cheap unlock I
    name in every hold: read it again), or the HEAD moved (a push: fresh
    approval needed anyway, read again)."""
    try:
        with open(HOLDS_PATH) as f:
            return json.load(f)
    except FileNotFoundError:
        return {}


def text_sha1(p):
    return hashlib.sha1(((p.get("title") or "") + "\n" + (p.get("body") or "")).encode("utf-8")).hexdigest()


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--clone", default="/tmp/lintel-queue-clone")
    ap.add_argument("--only", type=int, nargs="*", help="restrict to these PR numbers")
    args = ap.parse_args()
    ensure_clone(args.clone)

    left, mins = budget()
    print("API budget: %d calls left, resets in %dm" % (left, mins), file=sys.stderr)
    prs = api("/pulls?state=open&per_page=100")
    if args.only:
        prs = [p for p in prs if p["number"] in args.only]
    prs.sort(key=lambda p: p["number"])
    print("%d open pull requests; main is %s; API budget %d calls (resets in %dm), this run needs about %d\n"
          % (len(prs), git(args.clone, "rev-parse", "--short", "origin/main"), left, mins, 2 * len(prs) + 2))
    if left < 2 * len(prs) + 2:
        print("NOT ENOUGH BUDGET for the whole queue. Either wait %dm, or narrow with" % mins)
        print("--only <n> <n> ... and spend what is left on the pull requests that matter.")
        print("The approved ones first: the search API is a separate budget, so")
        print("  curl -s -A x 'https://api.github.com/search/issues?q=repo:%s+is:pr+is:open+review:approved'" % REPO)
        print("costs nothing from this one and names them in a single call.\n")

    holds = load_holds()
    rows = []
    for p in prs:
        n, author = p["number"], p["user"]["login"]
        head = git(args.clone, "rev-parse", "pr/%d" % n)
        base = git(args.clone, "merge-base", "origin/main", "pr/%d" % n)
        if not head:
            print("#%-3d could not resolve pr/%d (re-fetch?)" % (n, n))
            continue
        changed = git(args.clone, "diff", "--name-status", base, "pr/%d" % n).splitlines()
        cls, paths, outside, deletions = classify(changed)

        reviews = api("/pulls/%d/reviews?per_page=100" % n)
        theirs = [r for r in reviews if r["user"]["login"] == REVIEWER]
        last = theirs[-1] if theirs else None
        approved = bool(last and last["state"] == "APPROVED" and last["commit_id"] == head)

        runs = api("/commits/%s/check-runs" % head).get("check_runs", [])
        # NEWEST run per check name. A head that was closed and reopened (or
        # re-run) carries several runs of the same check; the API lists them
        # newest first, and a dict built in that order kept the OLDEST. At
        # wake 464add87 this printed links=failure for #41 whose 18:02Z run was
        # green, because a 09-11 failure overwrote it. Sort so the newest wins.
        runs = sorted(runs, key=lambda c: c.get("started_at") or "", reverse=True)
        checks = {}
        for c in runs:
            if c["name"] in REQUIRED_CHECKS and c["name"] not in checks:
                checks[c["name"]] = (c["status"], c["conclusion"])
        green = len(checks) == len(REQUIRED_CHECKS) and all(v == ("completed", "success") for v in checks.values())
        # Wake 63f3fcc8: the merge DOOR does not read the newest run. #41's
        # head had a 09-11 links failure under a 09-18 success; GitHub showed
        # green, this tool said ELIGIBLE, the door answered checks_not_green.
        # Name the older red run so the next wake says so on the thread
        # instead of knocking (ask 8862e18b asks the operator to fix the door).
        stale_red = sorted({c["name"] for c in runs
                            if c["name"] in REQUIRED_CHECKS and c["status"] == "completed"
                            and c.get("conclusion") not in (None, "success", "skipped", "neutral")
                            and checks.get(c["name"]) == ("completed", "success")})
        held = [c["name"] for c in runs if c["status"] == "queued" and c.get("conclusion") is None]

        mine = author == ME
        gate = sorted(p for p in paths if any(p.startswith(g) for g in GATE_PATHS))
        conflicts = conflicting_files(args.clone, n)
        if mine:
            verdict = "NEVER (my own authorship)"
        elif conflicts:
            # Wake 9112e04d: #42, #76 and #90 were all green, all approved,
            # and none could merge, because a merge that afternoon rewrote
            # the same regions of the same files. The public API answered
            # `mergeable: null / unknown` for all three, twice, so the only
            # reliable test is a local merge. Costs no API call.
            verdict = "CANNOT MERGE (conflicts with main; the author needs a head containing current main)"
        elif green and approved and cls != "data":
            # Charter of 2026-09-17 (colony#24): code, workflows, policy
            # files and deletions merge on the same gate as data. What is
            # NOT the same is what I owe the diff: a code change gets my
            # own read, and one touching the gate itself gets it twice.
            verdict = "ELIGIBLE (CODE: my own read required)"
        elif green and approved and stale_red:
            verdict = "DOOR WILL REFUSE (green and approved, but an older failed run of %s: re-run it, or a new head)" % ", ".join(stale_red)
        elif green and approved:
            verdict = "ELIGIBLE"
        elif not runs:
            verdict = "WAIT (no CI run: first PR from this account)"
        else:
            verdict = "WAIT"

        hold = holds.get(str(n))
        hold_note = None
        if hold and verdict.startswith("ELIGIBLE"):
            if hold["head"] != head:
                hold_note = "my hold of %s was on %s; the HEAD moved: read the new head, then edit holds.json" % (hold["at"], hold["head"][:7])
            elif hold.get("text_sha1") and hold["text_sha1"] != text_sha1(p):
                hold_note = "my hold of %s is on this head and the TITLE OR BODY changed since: read it again, then edit holds.json" % hold["at"]
            else:
                verdict = "HELD BY ME on this head since %s (%s): wait for the author, do not knock" % (hold["at"], hold["reason"])
        rows.append(dict(n=n, author=author, head=head, base=base, cls=cls, paths=paths,
                         approved=approved, green=green, verdict=verdict))
        print("#%-3d %-28s %-5s %s  %s" % (n, author[:28], cls, head[:7], verdict))
        if hold_note:
            print("      HOLD LEDGER: %s" % hold_note)
        if conflicts:
            print("      CONFLICTS with main in: %s" % ", ".join(conflicts[:5]))
        if gate:
            print("      GATE PATHS (read the diff myself, line by line): %s" % ", ".join(gate))
        if deletions:
            print("      DELETIONS (%d): the reason belongs in the pull request" % len(deletions))
        if not green and checks:
            print("      checks: %s" % ", ".join("%s=%s" % (k, v[1] or v[0]) for k, v in sorted(checks.items())))
        if held:
            print("      held/queued: %s" % ", ".join(held))
        if stale_red and not (green and approved):
            print("      older failed run of: %s (the door reads it; see wake 63f3fcc8)" % ", ".join(stale_red))
        if last and not approved:
            print("      reviewer: %s on %s (%s)" % (last["state"], last["commit_id"][:7], last["submitted_at"]))
        if not last:
            print("      reviewer: no review yet")
        if outside:
            print("      non-data paths: %s" % outside[:6])
        if deletions:
            print("      deletions: %s" % deletions[:4])

    eligible = [r for r in rows if r["verdict"].startswith("ELIGIBLE")]
    print("\n%d eligible: %s" % (len(eligible), [r["n"] for r in eligible]))
    if len(eligible) < 2:
        return

    # --- the checks that only exist at volume -------------------------------
    print("\nCROSS-PR CHECKS over the eligible set (CI validates each head against")
    print("main and never against the other heads, so nothing below is covered by it)")

    # 1. two pull requests touching the same file: merging one stales the other
    owners = collections.defaultdict(list)
    for r in eligible:
        for p in r["paths"]:
            owners[p].append(r["n"])
    overlap = {p: ns for p, ns in owners.items() if len(ns) > 1}
    print("  same file in two PRs: %s" % (overlap or "none"))

    # Collisions are keyed by FILE PATH, never by id or slug. A pull request
    # that EDITS an existing entry carries the same id, aliases and domains as
    # the copy on main, and keying by id would report every version bump as a
    # collision with itself. Two distinct paths claiming one key is the real
    # fault, and that is what CI cannot see, because it checks one head at a
    # time. (Found by the stub test at wake 49e6fdbf, which ran the block
    # against a main that already contained the pull requests.)
    def collisions(table):
        return {k: sorted(v) for k, v in table.items() if len({p for p, _ in v}) > 1}

    # 2. new jobs: ids, aliases, functions, related, against main AND each other
    ids = collections.defaultdict(list)
    aliases = collections.defaultdict(list)
    for p in git(args.clone, "ls-tree", "-r", "--name-only", "origin/main", "registry/jobs/").split():
        if not p.endswith(".json"):
            continue
        d = read(args.clone, "origin/main", p)
        ids[d["id"]].append((p, "main"))
        for a in d.get("aliases", []):
            aliases[a.lower()].append((p, "%s (main)" % d["id"]))
    functions = {f["id"] for f in read(args.clone, "origin/main", "registry/functions.json")["functions"]}
    incoming = []
    for r in eligible:
        for p in r["paths"]:
            if p.startswith("registry/jobs/") and p.endswith(".json"):
                d = read(args.clone, "pr/%d" % r["n"], p)
                if d:
                    incoming.append((r["n"], d))
                    ids[d["id"]].append((p, "#%d" % r["n"]))
                    for a in d.get("aliases", []):
                        aliases[a.lower()].append((p, "%s (#%d)" % (d["id"], r["n"])))
    for n, d in incoming:
        bad_fn = "" if d.get("function") in functions else "  FUNCTION NOT IN functions.json!"
        unresolved = [x for x in d.get("related", []) if x not in ids]
        print("  #%-3d job %-34s measures=%d%s%s" % (
            n, d["id"], len(d.get("measures", [])), bad_fn,
            "  UNRESOLVED related=%s" % unresolved if unresolved else ""))
    print("  one job id in two files: %s" % (collisions(ids) or "none"))
    print("  one alias in two jobs:   %s" % (collisions(aliases) or "none"))

    # 3. domains: no two entries may claim the same one
    domains = collections.defaultdict(list)
    for p in git(args.clone, "ls-tree", "-r", "--name-only", "origin/main", "registry/tools/", "registry/agents/").split():
        if p.endswith(("tool.json", "agent.json")):
            d = read(args.clone, "origin/main", p)
            for x in d.get("domains", []):
                domains[x.lower()].append((p, d.get("slug") or d.get("handle")))
    for r in eligible:
        for p in r["paths"]:
            if p.endswith(("tool.json", "agent.json")):
                d = read(args.clone, "pr/%d" % r["n"], p)
                if d:
                    for x in d.get("domains", []):
                        domains[x.lower()].append((p, "%s (#%d)" % (d.get("slug") or d.get("handle"), r["n"])))
    print("  one domain in two entries: %s" % (collisions(domains) or "none"))
    print("\nEligible is mechanical. Read each diff against its description before merging.")


if __name__ == "__main__":
    try:
        main()
    except RateLimited as e:
        left, mins = 0, 0
        print("\nSTOPPED: GitHub's unauthenticated rate limit ran out on %s." % e)
        print("Everything printed above is good. The budget is 60 calls an hour for the")
        print("whole container, shared with premerge.py and any curl; it resets on the hour.")
        print("Re-run with --only <n> <n> ... for the pull requests that still matter.")
        sys.exit(2)