#!/usr/bin/env python3
"""reach.py: which claims in this registry can any instrument re-run at all?

Written at wake d17bbbfe. Every sweep in this toolbox reports a number of the
form "N of N", and each of those N is the set the instrument could REACH, not
the set the registry makes claims about. `drift.py` prints "0 of 31 disagree
with their entry" on a registry of 35 tools, and the four it never mentions are
not agreeing: they are invisible to it, because it takes `surfaces.mcp` and
skips every entry without one. The gap that prompted this was Bitrefill's, at
wake ec0e3ab2: a `machinePayable: true` claim that is TRUE (measured by hand)
and that no checker in this repository can re-run, because the only payable
address anywhere in its files is the pattern `api.bitrefill.com/x402/*`, and a
wildcard is not an endpoint. That entry sat nine days without anyone noticing,
because every instrument here reads URLs and a URL that does not exist produces
no row anywhere.

So this instrument measures the DENOMINATORS the others do not print. For every
machine-checkable claim cell in the registry it asks one question: does the
entry name a concrete address at which this claim could be re-run, and by whom?
It makes no network request of its own and it judges no claim's truth. NOT
RE-RUNNABLE here is never "false" and never "wrong": it is "this registry is
asking to be trusted on this one", which is the single thing this registry
exists not to do.

The answer on the first registry it was pointed at (33591d2, 35 tools) was not
the one it was built to find, and that is worth stating at the top rather than
burying: NOTHING is unreachable to a person. All eight access claims drift.py
cannot see name an endpoint in a probe record filed about them. What the
sweep found instead is a gap between two things that had been read as one, so
they now have two names: CONTINUOUS (the nightly instrument re-runs this claim
by itself) and ON RECORD (a human reading the entry can re-run it; the
instrument never asks). Eight of thirty-five access claims are ON RECORD, and
an ON-RECORD claim decays silently, because the sweep that would notice is
pointed at a field that entry does not fill.

    python3 tools/drift.py --clone /tmp/reg --json /tmp/drift.json
    python3 tools/reach.py --clone /tmp/reg --drift-json /tmp/drift.json

The two claim cells it models, and the instrument each belongs to:

- `agentAccess` (`noAccountNeeded`, `auth`) -> drift.py, which needs
  `surfaces.mcp`. Modelled exactly as drift.py selects, and the CONTROL below
  asserts the two agree; if drift.py's selection ever changes and this model
  does not, the control fails rather than the number quietly drifting. With
  `--drift-json` the sweep also folds in drift.py's MEASURED verdicts, which
  is the difference between 31 and 27 on this registry: four of the entries
  that carry an `mcp` surface carry a DOCUMENTATION PAGE there, drift.py says
  so by measurement ("NOT AN ENDPOINT"), and an address that is not the thing
  is no more re-runnable than no address at all. Offline, without that file,
  this sweep will happily count all 31; that limit is the reason the flag
  exists and the reason the control prints both numbers.
- `payments` -> payments.py, GRADED BY PROVENANCE: PROBE (a probe record about
  payment names a GET-able endpoint: a stranger re-runs the exact
  measurement), PROSE (only documentation URLs near a payment word, and a docs
  page never answers 402), PATTERN (a wildcard, which no checker can call),
  NONE. Only PROBE counts as re-runnable. The first draft of this file graded
  on "is there any URL" and therefore called Bitrefill's payments claim
  re-runnable on the strength of an ACCESS probe's MCP endpoint, erasing the
  finding the file was written to generalise. An instrument that cannot
  reproduce the case that motivated it is measuring something else.

A cell this instrument does not model is reported as UNMODELLED in the summary
rather than passed over, because the whole point of the thing is that a silent
skip is how a gap survives nine days.
"""
import argparse
import glob
import json
import os
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import corpus  # noqa: E402
import payments  # noqa: E402

# the cells this instrument knows how to ask about; anything else is UNMODELLED
MODELLED = ("agentAccess", "payments")


def tool_rows(clone, drift=None):
    rows = []
    for f in sorted(glob.glob(os.path.join(clone, "registry/tools/*/tool.json"))):
        d = json.load(open(f, encoding="utf-8"))
        slug = d["slug"]
        aa = d.get("agentAccess") or {}
        pay = d.get("payments") or {}
        mcp = (d.get("surfaces") or {}).get("mcp")

        if aa.get("noAccountNeeded") is not None or aa.get("auth"):
            measured = (drift or {}).get(slug)
            # ON RECORD is the grade this instrument gained after its first run.
            # Its first draft called the eight entries drift.py cannot read
            # unreachable, and all eight name an endpoint in a probe record
            # filed about them. The claim is re-runnable BY A PERSON reading the
            # entry; what it is not is re-runnable by the instrument that
            # re-reads this cell every wake, because drift.py reads exactly one
            # field. "Nobody can check this" and "the nightly check never asks"
            # are different failures and they get different names.
            onrec = next((c for c in payments.candidates(clone, slug)
                          if c.get("url") and str(c.get("from", "")).startswith("probe:")),
                         None)
            if mcp and not (measured or "").startswith("NOT AN ENDPOINT"):
                grade, addr, why = "CONTINUOUS", mcp, None
            elif onrec:
                grade, addr = "ON RECORD", onrec["url"]
                why = ("%s, so drift.py never asks; a probe record names %s %s"
                       % ("surfaces.mcp is a documentation page" if mcp
                          else "no surfaces.mcp",
                          onrec.get("method", "GET"), onrec["url"]))
            else:
                grade, addr, why = "NOTHING", None, (
                    "no mcp surface and no probe record names an address")
            rows.append({
                "slug": slug, "cell": "agentAccess", "grade": grade,
                "claim": "noAccountNeeded=%s auth=%s" % (
                    aa.get("noAccountNeeded"), aa.get("auth")),
                "address": addr, "why": why, "instrument": "drift.py", "mcp": mcp,
            })

        if pay:
            cands = payments.candidates(clone, slug)
            # GRADED BY PROVENANCE, not by "is there a URL here". The first draft of
            # this file asked only whether some URL existed and called Bitrefill's
            # payments claim reachable on the strength of an ACCESS probe's MCP
            # endpoint, which is the wrong door: it erased the very finding this
            # instrument was written to generalise. An address is only the payment
            # claim's address if a probe record about PAYMENT names it.
            probe_get = next((c for c in cands
                              if c.get("url") and c.get("method") == "GET"
                              and str(c.get("from", "")).startswith("probe:")
                              and "pay" in str(c.get("from", "")).lower()), None)
            pats = [c["pattern"] for c in cands if c.get("pattern")]
            prose = [c for c in cands if c.get("from") == "prose"]
            if probe_get:
                grade, addr, why = "PROBE", probe_get["url"], None
            elif pats:
                grade, addr, why = "PATTERN", None, (
                    "only a wildcard pattern (%s): no checker can call it"
                    % ", ".join(pats))
            elif prose:
                grade, addr, why = "PROSE", None, (
                    "%d profile URL(s) near a payment word, all documentation: a docs "
                    "page never answers 402" % len(prose))
            else:
                grade, addr, why = "NONE", None, (
                    "no payment address in any probe record or profile sentence")
            rows.append({
                "slug": slug, "cell": "payments", "grade": grade,
                "claim": "machinePayable=%s protocols=%s" % (
                    pay.get("machinePayable"), ",".join(pay.get("protocols") or []) or "-"),
                "address": addr, "why": why, "instrument": "payments.py",
                "machine_payable": bool(pay.get("machinePayable")),
            })
    return rows


def unmodelled(clone):
    """Claim-bearing cells in tool.json that this instrument does not ask about.

    Printed, not skipped: the failure this whole file is about is a cell that
    nobody counts. A name here is a to-do for a future wake, not a finding.
    """
    seen = set()
    for f in sorted(glob.glob(os.path.join(clone, "registry/tools/*/tool.json"))):
        d = json.load(open(f, encoding="utf-8"))
        for k, v in d.items():
            if k in MODELLED or v in (None, "", [], {}):
                continue
            if isinstance(v, (dict, list)):
                seen.add(k)
    return sorted(seen)


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--clone", default="/tmp/reg")
    ap.add_argument("--ref", default="origin/main",
                    help="the tree this sweep claims to measure")
    ap.add_argument("--json", default=None)
    ap.add_argument("--drift-json", default=None,
                    help="a drift.py --json file: folds its MEASURED verdicts in, so "
                         "an mcp surface that is really a documentation page counts "
                         "as no address rather than as one")
    a = ap.parse_args()

    head = corpus.require_ref(a.clone, a.ref, "the reach sweep")
    drift = None
    if a.drift_json:
        d = __import__("json").load(open(a.drift_json))
        drift = {r["slug"]: r.get("verdict", "") for r in d.get("rows", [])}
        print("folding in drift.py's measured verdicts for %d tools, taken at %s\n"
              % (len(drift), d.get("measured")))
    rows = tool_rows(a.clone, drift)
    corpus.require_rows(len(rows), "claim cells", a.clone)

    tools = len(glob.glob(os.path.join(a.clone, "registry/tools/*/tool.json")))
    print("registry %s, %d tools, %d modelled claim cells\n" % (head, tools, len(rows)))

    for r in sorted(rows, key=lambda r: (r["cell"], r["grade"] not in ("CONTINUOUS", "PROBE"), r["slug"])):
        if r["grade"] in ("CONTINUOUS", "PROBE"):
            print("  RE-RUNNABLE   %-12s %-12s %s"
                  % (r["cell"], r["slug"], (r["address"] or "")[:56]))
        else:
            print("! NOT RE-RUNNABLE %-10s %-12s %s" % (r["cell"], r["slug"], r["why"]))
            print("                  claim: %s (%s cannot re-run it)"
                  % (r["claim"], r["instrument"]))

    for cell in MODELLED:
        cs = [r for r in rows if r["cell"] == cell]
        ok = [r for r in cs if r["grade"] in ("CONTINUOUS", "PROBE")]
        print("\n%-12s %d of %d claim cells are re-runnable by the instrument that watches them"
              % (cell, len(ok), len(cs)))
        for g in sorted({r["grade"] for r in cs} - {"CONTINUOUS", "PROBE"}):
            ws = [r["slug"] for r in cs if r["grade"] == g]
            print("             %-16s %d: %s" % (g, len(ws), ", ".join(ws)))

    claims_true = [r for r in rows if r["cell"] == "payments" and r.get("machine_payable")]
    if claims_true:
        runnable = [r["slug"] for r in claims_true if r["grade"] == "PROBE"]
        print("\nof the %d entries claiming an agent can pay without a human, %d name "
              "the address the claim was measured at: %s"
              % (len(claims_true), len(runnable), ", ".join(runnable) or "none"))
        for r in claims_true:
            if r["grade"] != "PROBE":
                print("  ! %s claims machinePayable and %s" % (r["slug"], r["why"]))

    # CONTROL: this instrument's model of drift.py's selection, against drift.py's
    # own rule. Two numbers that must be equal, stated out loud, so that a change
    # in either file fails here instead of shifting a denominator in silence.
    # what drift.py WOULD select (an mcp surface, endpoint or not), which is the
    # only thing this control is about: the selection model, not the verdicts
    drift_reach = len([r for r in rows if r["cell"] == "agentAccess" and r["mcp"]])
    drift_rule = len([f for f in glob.glob(os.path.join(a.clone, "registry/tools/*/tool.json"))
                      if (json.load(open(f, encoding="utf-8")).get("surfaces") or {}).get("mcp")])
    if drift_reach != drift_rule:
        print("\nCONTROL FAILED: reach models %d access cells as drift-checkable, "
              "drift.py's own rule selects %d. One of the two files moved; read both "
              "before believing any number above." % (drift_reach, drift_rule))
    else:
        docs = len([r for r in rows if r["cell"] == "agentAccess"
                    and r["grade"] == "ON RECORD" and (r["why"] or "").startswith("surfaces.mcp")])
        print("\ncontrol: %d tools carry surfaces.mcp, which is exactly the set "
              "drift.py selects and exactly what this sweep models." % drift_rule)
        if drift:
            print("         %d of the %d are documentation pages by drift.py's own "
                  "measurement, which is why the access count above is %d and not %d."
                  % (docs, drift_rule, drift_rule - docs, drift_rule))
        else:
            print("         No --drift-json was given, so the access count above is an "
                  "UPPER BOUND: an mcp surface that is really a documentation page is "
                  "counted here as an address. Run drift.py --json and pass it to "
                  "lower this number to what is actually re-runnable.")

    um = unmodelled(a.clone)
    if um:
        print("UNMODELLED cells present in tool.json and not asked about here: %s"
              % ", ".join(um))

    if a.json:
        json.dump({"head": head, "rows": rows}, open(a.json, "w"), indent=1)


if __name__ == "__main__":
    main()