#!/usr/bin/env python3
"""artifacts.py: does a probe record's prose agree with the artifact it cites?

Wake 6a552ca3. #220 (an absent header that the artifact shows present) and
#221 (two documents called byte-identical that the artifact shows 743 and 831
bytes) were both caught by hand at the merge seat, by the same move: read the
record's `decoded` prose, then read the artifact section it cites. This is the
mechanisable half of that move, over every probe record that names an
artifactsUrl.

Two claim shapes are checked, both CANDIDATES for a human read, never findings:
  COUNT     "N bytes" / "content-length N" in the record: does N occur in the
            artifact's relevant sections as a size (content-length, "N bytes",
            or an etag whose hex equals N)?
  NEGATIVE  "no X header" / "without an X header" / "X header absent" in the
            record: does header X appear in a response of the relevant sections?
  TIME      "at HH:MM:SSZ" / "at HH:MMZ" in the record (and its `at` field):
            does that clock occur in the relevant sections, as a `sent at`
            line, an ISO stamp or a date: header? Wake ad61b97b, after three
            records in a week (#220's cousin on #232: "89,430 bytes at 12:19Z"
            with no 12:19Z exchange of that file anywhere; the DNS "at 12:25Z"
            on the same head; the Mintlify (K4) second) cited a reading their
            artifact does not carry. A time with seconds must match to the
            second; a time to the minute matches any clock in that minute.
            A clock from another DAY in the same minute also passes: this is
            a candidate generator, not a proof.
Relevant sections = artifact blocks (split on lines starting with ##, ###, ===)
that mention the record's surface host or any host its prose names; the whole
artifact if none does.

Usage: artifacts.py --clone <dir> [--only <id-substring>]   |   --selftest
"""
import argparse, json, os, re, sys, urllib.request
sys.path.insert(0, os.path.dirname(__file__))
import corpus

UA = "lintel-artifacts/1 (+https://lintel.public-agents.ai)"
_cache = {}

def fetch(url):
    if url in _cache: return _cache[url]
    try:
        with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": UA}), timeout=60) as r:
            t = r.read().decode("utf-8", "replace"); st = r.status
    except Exception as e:
        t, st = "", getattr(e, "code", None) or str(e)
    _cache[url] = (st, t); return _cache[url]

def blocks(text):
    out, cur = [], []
    for line in text.splitlines():
        if re.match(r"^(##+ |=== )", line) and cur:
            out.append("\n".join(cur)); cur = []
        cur.append(line)
    if cur: out.append("\n".join(cur))
    return out

LABEL_RE = re.compile(r"\bsections?\s+([A-Z]|\d{1,2})\b")
# Plumb's appendices since late September label every request in parentheses,
# "(Z6)", "(S4)", "(V3)", "(T)", "(U10)", "(S0b)", and head each block
# "## (Z6) GET https://...". Wake 16cb3dc9: the zendesk oauth-metadata record
# cites its invented-host control as "section (Z6): 200, 607 bytes"; the block
# carries "bytes 607" in its own header line and names a host the record's
# prose never spells, so host scoping dropped it and the old label form
# ("section B") never matched a parenthesised label. 607 read as a CANDIDATE
# in every full run since 2026-10-11.
PAREN_LABEL_RE = re.compile(r"\(([A-Z]{1,2}\d{0,2}[a-z]?)\)")
HEAD_PAREN_RE = re.compile(r"^(?:#+|-{3,}|={3,})\s*\(([A-Z]{1,2}\d{0,2}[a-z]?)\)")

def relevant(text, hosts, labels=(), neighbors=False):
    """Blocks a record's prose points at: by HOST, and by the SECTION LABEL the
    prose names. Wake a709d73e: the statsig record says "(artifact section B)"
    and that block is a bare `### B. Console API GET, keyless` carrying no host
    string, so host scoping alone dropped the very section cited and the
    content-length: 12 inside it."""
    hosts = [h for h in hosts if h]
    bs, all_blocks = [], blocks(text)
    hits = set()
    for i, b in enumerate(all_blocks):
        head = b.splitlines()[0] if b.splitlines() else ""
        hit = any(h in b for h in hosts)
        if not hit and labels:
            m = re.match(r"^(?:#+|-{3,}|={3,})\s*([A-Za-z]|\d{1,2})[.)]", head)
            hit = bool(m) and m.group(1) in labels
            if not hit:
                m = HEAD_PAREN_RE.match(head)
                hit = bool(m) and m.group(1) in labels
        if hit: hits.add(i)
    # neighbors: a clock often sits in the block AFTER the request heading
    # ("## 1. MCP initialize, POST https://host/mcp" then "## at 2026-10-05T06:07:26Z"
    # as its own heading), so host scoping alone drops the very line a TIME
    # claim must be read against (wake ad61b97b: semgrep and grafana both
    # read as candidates with their clocks one block away)
    # and BEFORE it too: Plumb's per-request appendices write "## 01-initialize",
    # then "## at <iso>", then "## curl https://host/...", so the clock is the
    # block before the one that names the host (grafana, same wake)
    if neighbors: hits |= {j for i in list(hits) for j in (i - 1, i + 1) if 0 <= j < len(all_blocks)}
    # a `## ` heading that names the host OWNS the `### ` sub-blocks under it,
    # up to the next heading of its own level or higher (wake aea2eadf: the
    # deepwiki appendix of 2026-10-09 is one `## Appended ... https://deepwiki.com/`
    # heading over four `### n.` requests whose own lines name no host; the
    # neighbor rule reached the first and the clock of the second, 12:08:55Z,
    # read as a candidate though the artifact carries it)
    def level(b):
        m = re.match(r"^(#+) ", b.splitlines()[0] if b.splitlines() else "")
        return len(m.group(1)) if m else 99
    for i in sorted(hits):
        lv = level(all_blocks[i])
        if lv >= 99: continue
        for j in range(i + 1, len(all_blocks)):
            if level(all_blocks[j]) <= lv: break
            hits.add(j)
    bs = [all_blocks[i] for i in sorted(hits)]
    return "\n".join(bs) if bs else text

# a field whose KEY says it is quoting or correcting is not making the claim it
# contains: wake 6a552ca3 read exa's `corrected` field as a www-authenticate
# claim and it was the sentence being corrected (wake a709d73e makes it a rule)
QUOTING_KEY = re.compile(r"correct|revision|superseded|withdrawn|retract|was-", re.I)

def prose_fields(rec):
    """[(key, text)] over the record's decoded prose and its finding."""
    out = []
    def walk(k, v):
        if isinstance(v, dict): [walk(kk, vv) for kk, vv in v.items()]
        elif isinstance(v, list): [walk(k, x) for x in v]
        elif isinstance(v, str): out.append((k, v))
    walk("decoded", rec.get("observed", {}).get("decoded", {}))
    walk("finding", rec.get("finding", ""))
    walk("at", rec.get("at", "") or "")
    return out

def prose(rec):
    return "\n".join(t for _, t in prose_fields(rec))

REQ_LINE = re.compile(r"^(?:\$\s*curl\b.*?(?:-X\s+([A-Z]+)\s+)?(https?://\S+)|([A-Z]{3,7})\s+(https?://\S+))", re.M)

def exchanges(text):
    """Split a transcript into (method, url, segment) at each request line, so a
    claim about ONE response is read against that response and no other. Wake
    a709d73e: 'no RateLimit header on any MCP response' was a candidate only
    because the same host's discovery and REST exchanges carry those headers."""
    hits = list(REQ_LINE.finditer(text))
    out = []
    for i, m in enumerate(hits):
        end = hits[i + 1].start() if i + 1 < len(hits) else len(text)
        meth = (m.group(1) or m.group(3) or "").upper()
        url = m.group(2) or m.group(4) or ""
        out.append((meth, url.rstrip("'\"),;"), text[m.start():end]))
    return out

def exchange_text(text, method, surface):
    """The segments whose request line is this record's own request."""
    if not surface: return ""
    segs = [s for me, u, s in exchanges(text) if surface in u and (not method or not me or me == method.upper())]
    return "\n".join(segs)

COUNT_RE = re.compile(r"\b(\d{1,3}(?:,\d{3})+|\d{2,9})[- ]bytes?\b|content-length(?:\s*(?:of|is|:))?\s*(\d{1,9})", re.I)
NEG_RE = re.compile(r"\b(?:no|without an?|lacks?(?: an?)?)\s+`?([A-Za-z][A-Za-z0-9-]{2,40})`?\s+header|`?([A-Za-z][A-Za-z0-9-]{2,40})`?\s+header\s+(?:is\s+)?(?:absent|missing)", re.I)

def sizes_in(text):
    found = set()
    for m in re.finditer(r"content-length:\s*(\d+)", text, re.I): found.add(int(m.group(1)))
    for m in re.finditer(r"\b(\d{1,3}(?:,\d{3})+|\d{2,9})\s*bytes?\b", text, re.I): found.add(int(m.group(1).replace(",", "")))
    # the artifacts write the size AFTER the word as often as before it
    # ("body bytes: 208", "bytes 386"): wake a709d73e, where nine of the
    # twelve count candidates were this convention and nothing else.
    for m in re.finditer(r"\bbytes:?\s+(\d{1,3}(?:,\d{3})+|\d{1,9})\b", text, re.I): found.add(int(m.group(1).replace(",", "")))
    # and the summary lines the probe scripts print: "byte count: 11205",
    # "textbytes 2341". Same wake, same cause: the number follows the word.
    for m in re.finditer(r"\b(?:byte count|textbytes|size)[:= ]\s*(\d{1,3}(?:,\d{3})+|\d{1,9})\b", text, re.I):
        found.add(int(m.group(1).replace(",", "")))
    for m in re.finditer(r'etag:\s*W?/?"([0-9a-f]{1,8})(?:-[^"]*)?"', text, re.I):
        try: found.add(int(m.group(1), 16))
        except ValueError: pass
    return found

# lookarounds and not \b: the T of an ISO stamp is a word character, so
# \b never fires between T and 06 and "T06:05:44Z" would read as "05:44"
TIME_RE = re.compile(r"(?<![\d:])(\d{2}):(\d{2})(?::(\d{2}))?Z(?![\w])")

def times_in(text):
    """Every clock the artifact carries, as (HH:MM:SS set, HH:MM set): ISO
    stamps ('sent at 2026-10-09T06:11:29Z', 'sent 2026-10-09T06:05:44Z'),
    bare 'HH:MMZ' mentions, and RFC 7231 date: headers ('06:11:35 GMT', which
    is the same clock as Z)."""
    secs, mins = set(), set()
    for m in re.finditer(r"(?<![\d:])(\d{2}):(\d{2}):(\d{2})(?!\d)", text):
        secs.add(m.group(0)); mins.add(m.group(1) + ":" + m.group(2))
    for m in re.finditer(r"(?<![\d:])(\d{2}):(\d{2})Z(?!\w)", text):
        mins.add(m.group(1) + ":" + m.group(2))
    return secs, mins

# the minute of a record's `at` must be a STAMP in the artifact (an HH:MM:SS
# clock on a sent line, a date: header or a bare stamp, or a wildcard heading
# such as "18:2xZ"), never a mention: wake c6705c54, the optimizely record's
# `at` 12:28Z was the artifact's PUBLICATION minute, and the transcript's own
# section 5 says "filed at 12:28Z" in prose, which is why the mention-level
# read above let it through while Plumb's minute check (#238) did not
WILD_RE = re.compile(r"(?<![\d:])(\d{2}):(\d)x(?:Z|\b)")

RANGE_RE = re.compile(r"(?<![\d:])(\d{2}):(\d{2})Z\s+(?:to|-|\u2013|through)\s+(\d{2}):(\d{2})Z")

def stamped_minutes(text):
    """(minutes carried by an HH:MM:SS stamp or inside a pass range the
    transcript's header states, '06:31Z to 06:32Z'; wildcard prefixes 'HH:M'
    from headings like '18:2xZ')."""
    mins = {m.group(1) + ":" + m.group(2) for m in re.finditer(r"(?<![\d:])(\d{2}):(\d{2}):(\d{2})(?!\d)", text)}
    for m in RANGE_RE.finditer(text):
        a = int(m.group(1)) * 60 + int(m.group(2)); b = int(m.group(3)) * 60 + int(m.group(4))
        if 0 <= b - a <= 180: mins |= {"%02d:%02d" % divmod(x, 60) for x in range(a, b + 1)}
    wild = {m.group(1) + ":" + m.group(2) for m in WILD_RE.finditer(text)}
    return mins, wild

def headers_in(text):
    return {m.group(1).lower() for m in re.finditer(r"^([A-Za-z][A-Za-z0-9-]+):\s", text, re.M)}

SIBLING_RE = re.compile(r"\b([A-Za-z0-9][A-Za-z0-9._-]{2,80}\.(?:txt|json|log))\b")

def siblings(rec, p):
    """Filenames the record's prose names, resolved in the artifact's own
    directory. Wake a709d73e: the 09-18 docusign record's 47,958-byte tool
    catalogue is NOT in the probe transcript it cites, and the record says so
    in the same sentence ("transcribed beside the artifact as
    tools-catalog-1212Z.txt"). A record's evidence is a FAMILY of files; one
    artifactsUrl is the family's entry point, not its whole extent."""
    base = rec.get("reproducibility", {}).get("artifactsUrl", "")
    if "/" not in base: return []
    d = base.rsplit("/", 1)[0] + "/"
    own = base.rsplit("/", 1)[1]
    out = []
    # a FULL URL the prose names is resolved as written, not in the artifact's
    # directory: wake ad61b97b, the context7 counter record names its
    # 2026-09-30 transcript by full URL and this resolver looked for the bare
    # filename under 2026-09-29, got a 404, and reported three clocks that
    # the named file carries as candidates
    full = [m.group(0).rstrip(".,;)") for m in re.finditer(r"https?://\S+\.(?:txt|json|log)\b", p)]
    # a document the sentence names by full URL is where a FOLDED file lives:
    # wake 6a851d4e, the intercom 2026-09-17 record (v2, #243) says its
    # 18:16Z transcript "is published whole in the artifact directory's
    # README at https://.../README.md under the heading Folded files", and
    # this resolver tried the bare name under five dates, reported five
    # misses, and never opened the README the same sentence hands it
    full += [m.group(0).rstrip(".,;)") for m in re.finditer(r"https?://\S+\.md\b", p)]
    full = [u for u in full if u != base]
    named = {u.rsplit("/", 1)[1] for u in full}
    for m in SIBLING_RE.finditer(p):
        n = m.group(1)
        if n == own or n.endswith(".sh") or n in out or n in named: continue
        out.append(n)
    # "beside this record's artifact" is sometimes beside the DATE the sentence
    # names instead: wake c6705c54, both zapier ownership records of 2026-10-08
    # name dns-txt-0605Z.txt "beside this record's artifact" and it is published
    # under 2026-10-09, the day of the re-query the same sentence dates. Try the
    # artifact's directory first, then the same directory re-dated to each
    # date the prose names.
    dates = sorted({m.group(0) for m in re.finditer(r"\b20\d{2}-\d{2}-\d{2}\b", p)})
    redated = []
    dm = re.search(r"/(20\d{2}-\d{2}-\d{2})/$", d)
    if dm:
        for dt in dates:
            if dt != dm.group(1): redated += [d[:dm.start(1)] + dt + "/" + n for n in out]
    seen, res = set(), []
    for u in full + [d + n for n in out] + redated:
        if u not in seen: seen.add(u); res.append(u)
    return res[:8]

def still_missed(missed, got):
    """The misses that stay misses once the answered documents are read: a
    name that answered under one of its tried directories is found, not
    missed (the re-dated tries must not inflate the miss count: 34 where 12
    were real, wake c6705c54), and so is a name that an answered document
    CARRIES in its text, which is what a folded file looks like from
    outside (wake 6a851d4e, the intercom README)."""
    found = {n for n, _ in got}
    out = []
    for x in missed:
        n = x[2].rsplit("/", 1)[1]
        if n in found or any(n in t for _, t in got): continue
        out.append(x)
    return out

DATE_RE = re.compile(r"\b(20\d{2}-\d{2}-\d{2})\b")

def attributed(text, m, own_date):
    """True when the 120 characters around a clock name a DIFFERENT date than
    the record's own, or a probe record id, or a transcript URL: the clock
    belongs to the pass the sentence names, not to this artifact."""
    w = text[max(0, m.start() - 120):m.end() + 120]
    if re.search(r"\bp-\d{8}-[a-z0-9-]+", w) or re.search(r"https?://\S+\.(?:txt|log)\b", w): return True
    return any(d != own_date for d in DATE_RE.findall(w))

def check(rec, art):
    host = re.sub(r"^https?://([^/]+).*$", r"\1", rec.get("surface", "") or "")
    own_date = (rec.get("at") or "")[:10]
    p = prose(rec)
    # every host the record's own prose names counts as cited (wake 6a552ca3:
    # #221's issuer sections name login.semgrep.dev, not the surface host)
    # ... with or without a scheme: wake ad61b97b, the samsara record cites
    # "developers.samsara.com/docs/mcp.md (updatedAt 2026-09-16T22:49:45Z)"
    # bare, and the block carrying that stamp names only that host
    named = {m.group(1) for m in re.finditer(r"https?://([A-Za-z0-9.-]+)", p)}
    named |= {m.group(1).lower() for m in re.finditer(r"(?<![\w@/.-])((?:[A-Za-z0-9-]+\.)+(?:com|org|net|io|dev|ai|app|so|build|sh|wiki|cloud|co|me|xyz|aws))(?=[/\s,;:)]|$)", p)}
    hosts = [host] + sorted(named - {host})
    labels = {m.group(1) for m in LABEL_RE.finditer(p)} | {m.group(1) for m in PAREN_LABEL_RE.finditer(p)}
    # the two rules want windows of OPPOSITE width, and conflating them made the
    # negative side noisier the moment labels were added (wake a709d73e): a
    # POSITIVE claim ("N bytes") is satisfied by finding N anywhere the record
    # points, so widen by section label; a NEGATIVE claim ("no X header") is
    # falsified only by the exchange it describes, so narrow it to that request
    # and fall back to the host window only when no exchange matches.
    sizes = sizes_in(relevant(art, hosts, labels))
    byhost = relevant(art, hosts)
    own = exchange_text(byhost, (rec.get("request") or {}).get("method", ""), rec.get("surface", ""))
    hdrs = headers_in(own or byhost)
    window = relevant(art, hosts, labels, neighbors=True)
    secs, mins = times_in(window)
    # the record's `at` is the minute of the whole pass, so it is read over the
    # whole artifact and not the host window: the firecrawl 2026-09-29 record's
    # 12:16:49Z sits in the transcript's index table, which names no host
    stamped, wild = stamped_minutes(art)
    _, allmins = times_in(art)
    rows = []
    for key, p1 in prose_fields(rec):
        quoting = bool(QUOTING_KEY.search(key))
        for m in COUNT_RE.finditer(p1):
            n = int((m.group(1) or m.group(2)).replace(",", ""))
            if n < 10 or quoting: continue
            # "roughly 234 bytes per result" is arithmetic over a size, not a
            # size: the hedge is the author saying so, and reading it as a wire
            # size is how this sweep manufactured a candidate (wake a709d73e)
            if re.search(r"\b(?:roughly|about|approximately|around|some|average|avg)\s*$", p1[max(0, m.start() - 24):m.start()], re.I):
                continue
            rows.append(("COUNT", n, "ok" if n in sizes else "CANDIDATE: %d not a size in the cited sections" % n))
        for m in NEG_RE.finditer(p1):
            h = (m.group(1) or m.group(2)).lower()
            if h in ("response", "request", "the", "a", "any", "such", "that", "this", "other", "custom", "session"): continue
            if quoting: continue
            # "a tools/list SENT with no mcp-session-id header" describes what
            # left the container, not what came back; the response is innocent
            before = p1[max(0, m.start() - 40):m.start()].lower()
            if re.search(r"\b(?:sent|send|sending|asked|request(?:ed)?|carried|carrying|posted)\b[^.]{0,20}$", before):
                rows.append(("NEGATIVE", h, "ok")); continue
            rows.append(("NEGATIVE", h, "CANDIDATE: '%s' header present in %s" % (h, "this record's own exchange" if own else "the cited sections") if h in hdrs else "ok"))
        for m in TIME_RE.finditer(p1):
            if quoting: continue
            hh, mm, ss = m.groups()
            hit = ("%s:%s:%s" % (hh, mm, ss) in secs) if ss else ("%s:%s" % (hh, mm) in mins)
            if key == "at" and not ss:
                # the record's own minute: a mention is not a measurement
                mn = "%s:%s" % (hh, mm)
                if mn in stamped or "%s:%s" % (hh, mm[0]) in wild: rows.append(("AT", m.group(0), "ok"))
                elif mn in allmins: rows.append(("AT", m.group(0), "CANDIDATE: `at` minute %s is in the artifact only as a mention, not as a stamped response or sent line" % mn))
                else: rows.append(("AT", m.group(0), "CANDIDATE: no %s clock in the cited sections" % m.group(0)))
                continue
            if not hit and attributed(p1, m, own_date):
                # the sentence hands the clock to another dated pass or a
                # named record ("re-measured 2026-09-22 12:06Z", "at 06:31Z
                # on 2026-09-28 (p-20260928-allium-mcp-gate-closed)"): the
                # record is naming its source, which is what #234 asks for;
                # wake c6705c54 read eleven of these by hand and every one
                # was of this shape
                rows.append(("TIME", m.group(0), "ok (attributed to another dated pass or record)")); continue
            rows.append(("AT" if key == "at" else "TIME", m.group(0), "ok" if hit else "CANDIDATE: no %s clock in the cited sections" % m.group(0)))
    return host, rows

def run(clone, only):
    corpus.require_clone(clone, "artifacts.py"); corpus.require_ref(clone, "origin/main", "artifacts.py")
    root = os.path.join(clone, "registry", "evidence", "probes")
    recs = []
    for f in sorted(os.listdir(root)):
        if not f.endswith(".json"): continue
        r = json.load(open(os.path.join(root, f)))
        if r.get("reproducibility", {}).get("artifactsUrl") and (not only or only in r.get("id", f)): recs.append(r)
    corpus.require_rows(len(recs), "probe records with an artifactsUrl", clone)
    tot = cand = dead = resolved = 0; ncount = nneg = ntime = 0; sibdead = []
    for r in recs:
        url = r["reproducibility"]["artifactsUrl"]; st, art = fetch(url)
        if st != 200 or not art:
            dead += 1; print("DEAD-ARTIFACT %-60s %s %s" % (r["id"], st, url)); continue
        host, rows = check(r, art)
        ncount += sum(1 for k, _, _ in rows if k == "COUNT"); nneg += sum(1 for k, _, _ in rows if k == "NEGATIVE"); ntime += sum(1 for k, _, _ in rows if k in ("TIME", "AT"))
        bad = [x for x in rows if x[2] != "ok"]
        note = ""
        # a COUNT the cited transcript lacks may be in a sibling file the
        # record's own prose names; only then is it worth a fetch
        # a TIME the cited transcript lacks may be a sibling's too (the DNS
        # file beside the Zapier records carries its own `sent` lines)
        if any(k in ("COUNT", "TIME", "AT") for k, _, w in bad):
            sibs = siblings(r, prose(r))
            got = []
            missed = []
            for u in sibs:
                st2, t2 = fetch(u)
                if st2 == 200 and t2: got.append((u.rsplit("/", 1)[1], t2))
                else: missed.append((r["id"], st2, u))
            # a name that answered under one of its tried directories is
            # found, not missed: the re-dated tries above must not inflate
            # the miss count (34 where 12 were real, wake c6705c54)
            sibdead += still_missed(missed, got)
            if got:
                host, rows2 = check(r, art + "\n" + "\n".join(t for _, t in got))
                bad2 = [x for x in rows2 if x[2] != "ok"]
                if len(bad2) < len(bad):
                    resolved += len(bad) - len(bad2)
                    note = "  (resolved in sibling: %s)" % ", ".join(n for n, _ in got)
                bad = bad2
        tot += 1; cand += bool(bad)
        if bad:
            print("%-62s host %s%s" % (r["id"], host, note))
            for k, v, why in bad: print("    %-8s %-28s %s" % (k, v, why))
    print("\n%d records with a live artifact (%d dead artifact URLs); %d byte-count claims, %d header negatives and %d cited clock times read;" % (tot, dead, ncount, nneg, ntime))
    print("%d records carry at least one CANDIDATE. A candidate is a sentence to read beside its section, not a finding." % cand)
    print("%d count or time claim(s) resolved in a SIBLING file the record names in the same sentence; %d named sibling(s) did not answer." % (resolved, len(sibdead)))
    for rid, st, u in sibdead: print("    SIBLING-MISS %-56s %s %s" % (rid, st, u))
    print("control: a count that is in the artifact reads ok; '743 bytes' beside an etag W/\"33f\" document reads ok for 743 and would flag 831 only if the record stated it, so an OMITTED number is still invisible here.")

ART = """## section 8  GET https://x.example/.well-known/a
HTTP/2 200
content-length: 743

## section 9  GET https://y.example/.well-known/a
HTTP/2 200
etag: W/"33f-abc"

## other  POST https://z.example/mcp
HTTP/2 401
www-authenticate: Bearer realm=x
content-length: 121
"""
def selftest():
    ok = True
    def t(name, cond):
        nonlocal ok; ok &= bool(cond); print(("ok   " if cond else "FAIL ") + name)
    rec = {"surface": "https://x.example/mcp", "observed": {"decoded": {"a": "answered 200 with 743 bytes"}}, "finding": ""}
    t("count present in cited block reads ok", check(rec, ART)[1][0][2] == "ok")
    rec["observed"]["decoded"]["a"] = "answered 200, 744 bytes"
    t("count absent reads CANDIDATE", "CANDIDATE" in check(rec, ART)[1][0][2])
    rec = {"surface": "https://y.example/mcp", "observed": {"decoded": {"a": "the issuer's copy is 831 bytes"}}, "finding": ""}
    t("etag hex 33f resolves to 831", check(rec, ART)[1][0][2] == "ok")
    rec = {"surface": "https://z.example/mcp", "observed": {"decoded": {"a": "No WWW-Authenticate header and a 121-byte body"}}, "finding": ""}
    rows = check(rec, ART)[1]
    t("#220 shape: absent header that the section carries reads CANDIDATE", any(k == "NEGATIVE" and "CANDIDATE" in w for k, _, w in rows))
    t("121-byte body reads ok beside content-length 121", any(k == "COUNT" and w == "ok" for k, _, w in rows))
    rec["observed"]["decoded"]["a"] = "no retry-after header"
    t("absent header the section lacks reads ok", check(rec, ART)[1][0][2] == "ok")
    rec = {"surface": "https://q.example/", "observed": {"decoded": {"a": "121 bytes"}}, "finding": ""}
    t("unknown host falls back to the whole artifact", check(rec, ART)[1][0][2] == "ok")
    t("sizes_in reads 1,234 with a comma", 1234 in sizes_in("body of 1,234 bytes"))
    t("sizes_in reads 'body bytes: 208'", 208 in sizes_in("body bytes: 208  sha256: 4e62"))
    t("sizes_in reads 'bytes: 1,151'", 1151 in sizes_in("bytes: 1,151"))
    t("sizes_in does not read a bare 'bytes' with no number", sizes_in("body bytes: n/a") == set())
    sibrec = {"reproducibility": {"artifactsUrl": "https://h.example/evidence/x/2026-09-18/probes-1204Z.txt"}}
    sibs = siblings(sibrec, "transcribed beside the artifact as tools-catalog-1212Z.txt (the 06:08Z read, tools-catalog-0608Z.txt, listed the same names); see probes-1204Z.txt")
    t("siblings() resolves named files in the artifact's directory",
      sibs == ["https://h.example/evidence/x/2026-09-18/tools-catalog-1212Z.txt",
               "https://h.example/evidence/x/2026-09-18/tools-catalog-0608Z.txt"])
    t("siblings() never names the artifact itself",
      all(not s.endswith("probes-1204Z.txt") for s in sibs))
    t("siblings() ignores a record with no artifactsUrl", siblings({}, "a.txt") == [])
    t("siblings() resolves a full URL as written and not in the artifact's directory",
      siblings(sibrec, "Transcript: https://h.example/evidence/x/2026-09-19/probes-0603Z.txt (section 4)")
      == ["https://h.example/evidence/x/2026-09-19/probes-0603Z.txt"])
    t("siblings() never returns the artifact's own URL",
      siblings(sibrec, "see https://h.example/evidence/x/2026-09-18/probes-1204Z.txt") == [])
    t("sizes_in reads 'byte count: 11205'", 11205 in sizes_in("# whole-response byte count: 11205; x"))
    t("sizes_in reads 'textbytes 2341'", 2341 in sizes_in("http 200 textbytes 2341 results 10"))
    LART = """### A. first, https://a.example/mcp
HTTP/2 200
server: nginx

### B. Console API GET, keyless
content-length: 12

Unauthorized
"""
    rec = {"surface": "https://a.example/mcp", "observed": {"decoded": {"a": "answers 401 with the 12-byte body Unauthorized (artifact section B)"}}, "finding": ""}
    t("a section named by letter is cited even with no host in it",
      [w for k, _, w in check(rec, LART) [1] if k == "COUNT"] == ["ok"])
    rec2 = {"surface": "https://a.example/mcp", "observed": {"decoded": {"a": "answers 401 with a 12-byte body"}}, "finding": ""}
    t("without the label, the same count is still a CANDIDATE",
      any("CANDIDATE" in w for k, _, w in check(rec2, LART)[1] if k == "COUNT"))
    rec3 = {"surface": "https://a.example/mcp", "observed": {"decoded": {"a": "each result is roughly 234 bytes"}}, "finding": ""}
    t("a hedged count is not read as a wire size", not check(rec3, LART)[1])
    rec4 = {"surface": "https://a.example/mcp", "observed": {"decoded": {"a": "no content-length header on the body (artifact section B)"}}, "finding": ""}
    t("a section label widens the COUNT window and not the NEGATIVE one",
      all(w == "ok" for k, _, w in check(rec4, LART)[1] if k == "NEGATIVE"))
    ZART = """## (Z5) GET https://mcp.example.com/.well-known/oauth-authorization-server/api/mcp/oauth
# sent 2026-10-11T12:04:58Z  status 200  bytes 611

## (Z6) GET https://zzz-invented-4q2.example.com/.well-known/oauth-authorization-server/api/mcp/oauth
# sent 2026-10-11T12:04:59Z  status 200  bytes 607
# response headers:
HTTP/2 200
content-type: application/json

## (Z7) GET https://zzz-invented-4q2.example.com/api/mcp
# sent 2026-10-11T12:05:01Z  status 401  bytes 83
"""
    rec = {"surface": "https://mcp.example.com/.well-known/oauth-authorization-server/api/mcp/oauth", "observed": {"decoded": {"control": "the same path on an invented account host, section (Z6): 200, 607 bytes, the same nine fields"}}, "finding": ""}
    t("wake 16cb3dc9: a parenthesised label (Z6) names a block whose host the prose never spells, and its 607 reads ok",
      [w for k, v, w in check(rec, ZART)[1] if k == "COUNT" and v == 607] == ["ok"])
    rec = {"surface": "https://mcp.example.com/.well-known/oauth-authorization-server/api/mcp/oauth", "observed": {"decoded": {"control": "the same path on an invented account host: 200, 607 bytes, the same nine fields"}}, "finding": ""}
    t("control: without the label, the invented host's 607 is still a CANDIDATE",
      any("CANDIDATE" in w for k, v, w in check(rec, ZART)[1] if k == "COUNT" and v == 607))
    rec = {"surface": "https://mcp.example.com/.well-known/oauth-authorization-server/api/mcp/oauth", "observed": {"decoded": {"control": "section (Z6) answered 200 with 83 bytes"}}, "finding": ""}
    t("control: a label does not widen to its neighbours; (Z7)'s 83 under a (Z6) citation is a CANDIDATE",
      any("CANDIDATE" in w for k, v, w in check(rec, ZART)[1] if k == "COUNT" and v == 83))
    TART = """--- (N1) GET https://n.example/.well-known/public-agents.json
  sent at 2026-10-09T06:11:29Z
  date: Fri, 09 Oct 2026 06:11:29 GMT
  body: 46345 bytes

--- (N3) GET https://m.example/.well-known/public-agents.json
  sent at 2026-10-09T06:11:34Z
  date: Fri, 09 Oct 2026 06:11:35 GMT
"""
    def times(rec, art): return [(v, w) for k, v, w in check(rec, art)[1] if k in ("TIME", "AT")]
    rec = {"surface": "https://n.example/.well-known/public-agents.json", "observed": {"decoded": {"a": "read at 06:11:29Z, 46,345 bytes"}}, "finding": "", "at": "2026-10-09T06:11Z"}
    t("a cited second that is a sent line reads ok, and so does the record's own `at` minute",
      times(rec, TART) == [("06:11:29Z", "ok"), ("06:11Z", "ok")])
    rec["observed"]["decoded"]["a"] = "89,430 bytes at 12:19Z the same day as well"
    t("#232 shape: a cited minute with no clock in the cited sections reads CANDIDATE",
      any("CANDIDATE" in w for v, w in times(rec, TART) if v == "12:19Z"))
    rec["observed"]["decoded"]["a"] = "sent at 18:06:14Z"
    t("the (K4) shape: a second that differs by one from the sent line reads CANDIDATE",
      times(rec, TART)[0][1].startswith("CANDIDATE"))
    rec["observed"]["decoded"]["a"] = "the response's own date header says 06:11:35Z"
    t("a date: header clock counts as a clock the artifact carries",
      times(rec, TART)[0][1] == "ok")
    rec["observed"]["decoded"] = {"corrected-from": "was 12:19Z"}
    t("a quoting key's time is not a claim", times(rec, TART) == [("06:11Z", "ok")])
    SART = "intro\n\n## Appended: https://s.example/ read three ways\n### 1. HEAD, sent 2026-10-09T12:08:52Z\n$ curl -I\n\n### 2. GET, sent 2026-10-09T12:08:55Z\n$ curl\n\n### 3. GET, sent 2026-10-09T12:08:57Z\n$ curl\n\n## Other host: https://t.example/\n### 1. sent 2026-10-09T13:00:00Z\n"
    rec = {"surface": "https://s.example/", "observed": {"decoded": {"a": "sent at 12:08:52Z, 12:08:55Z and 12:08:57Z"}}, "finding": "", "at": "2026-10-09T12:08Z"}
    t("a host-bearing ## heading owns its ### sub-blocks, beyond neighbor distance one",
      [w for _, w in times(rec, SART)] == ["ok", "ok", "ok", "ok"])
    rec["observed"]["decoded"]["a"] = "sent at 13:00:00Z"
    t("...but not the sub-blocks of the next ## heading", "CANDIDATE" in times(rec, SART)[0][1])
    SART = """## 1. MCP initialize, POST https://s.example/mcp
## at 2026-10-05T06:07:26Z
## curl https://s.example/mcp -d {}

HTTP/2 401
content-length: 74

## 2. GET https://other.example/
## at 2026-10-05T06:09:00Z
## 3. unrelated
## 4. unrelated
## at 2026-10-05T06:10:00Z
"""
    rec = {"surface": "https://s.example/mcp", "observed": {"decoded": {}}, "finding": "sent on 2026-10-05 at 06:07:26Z answered 401", "at": "2026-10-05T06:07Z"}
    t("a clock in the block AFTER the request heading is read (semgrep shape)",
      times(rec, SART) == [("06:07:26Z", "ok"), ("06:07Z", "ok")])
    rec["finding"] = "sent at 06:10:00Z"
    t("a clock three blocks away, under another host, is still a CANDIDATE",
      times(rec, SART)[0][1].startswith("CANDIDATE"))
    BART = """## GET https://api.s.example/fleet
HTTP/2 401

## docs, GET https://developers.s.example/docs/mcp.md
updatedAt: 2026-09-16T22:49:45.000Z
"""
    rec = {"surface": "https://api.s.example/fleet", "observed": {"decoded": {"d": "developers.s.example/docs/mcp.md (updatedAt 2026-09-16T22:49:45Z)"}}, "finding": ""}
    t("a host the prose names WITHOUT a scheme widens the window (samsara shape)",
      times(rec, BART) == [("22:49:45Z", "ok")])
    t("times_in reads 'sent 2026-10-09T06:05:44Z' and a bare 'HH:MMZ'",
      times_in("sent 2026-10-09T06:05:44Z; the 12:25Z pass") == ({"06:05:44"}, {"06:05", "12:25"}))

    XART = """## 1. the MCP endpoint
POST https://h.example/mcp  {"method":"initialize"}
HTTP/2 200
content-type: text/event-stream

## 2. the REST endpoint on the same host
GET https://h.example/api/whoami
HTTP/2 401
ratelimit: 300
"""
    xr = {"surface": "https://h.example/mcp", "request": {"method": "POST"},
          "observed": {"decoded": {"not-measured": "no RateLimit header on any MCP response"}}, "finding": ""}
    t("a negative is read against THIS record's exchange, not the whole host",
      [w for k, _, w in check(xr, XART)[1] if k == "NEGATIVE"] == ["ok"])
    xr2 = dict(xr); xr2["surface"] = "https://h.example/api/whoami"; xr2["request"] = {"method": "GET"}
    t("the same claim on the exchange that DOES carry it is a CANDIDATE",
      any("CANDIDATE" in w for k, _, w in check(xr2, XART)[1] if k == "NEGATIVE"))
    qr = {"surface": "https://h.example/api/whoami", "request": {"method": "GET"},
          "observed": {"decoded": {"corrected": "the sentence said: no RateLimit header, which was wrong"}}, "finding": ""}
    t("a field keyed `corrected` is quoting, not claiming", not check(qr, XART)[1])
    sr = {"surface": "https://h.example/api/whoami", "request": {"method": "GET"},
          "observed": {"decoded": {"a": "a GET sent with no RateLimit header answered 401"}}, "finding": ""}
    t("a negative about the REQUEST is not a claim about the response",
      [w for k, _, w in check(sr, XART)[1] if k == "NEGATIVE"] == ["ok"])
    AART = """## 1. GET https://o.example/api/search
HTTP/2 200
date: Thu, 24 Sep 2026 12:18:31 GMT

## 5. appended later: the value sent at 12:28Z was never recorded; the record filed at 12:28Z said 10000

## 13. the bracket repeated, 18:2xZ
GET https://o.example/api/search
HTTP/2 200
"""
    ar = {"surface": "https://o.example/api/search", "request": {"method": "GET"}, "observed": {"decoded": {}}, "finding": "", "at": "2026-09-24T12:28Z"}
    t("`at` minute present only as a prose mention reads CANDIDATE (the optimizely shape, #238)",
      [w for k, _, w in check(ar, AART)[1] if k == "AT"] == ["CANDIDATE: `at` minute 12:28 is in the artifact only as a mention, not as a stamped response or sent line"])
    ar["at"] = "2026-09-24T12:18Z"
    t("`at` minute carried by a date: header reads ok", [w for k, _, w in check(ar, AART)[1] if k == "AT"] == ["ok"])
    ar["at"] = "2026-09-24T18:27Z"
    t("`at` minute inside a wildcard heading 18:2xZ reads ok", [w for k, _, w in check(ar, AART)[1] if k == "AT"] == ["ok"])
    ar["at"] = "2026-09-24T12:09Z"
    t("`at` minute nowhere in the artifact reads CANDIDATE (the context7 18:09Z shape)",
      [w for k, _, w in check(ar, AART)[1] if k == "AT"] == ["CANDIDATE: no 12:09Z clock in the cited sections"])
    pr = {"surface": "https://o.example/api/search", "request": {"method": "GET"}, "observed": {"decoded": {"a": "re-run at 12:28Z with the default agent"}}, "finding": ""}
    t("control: a PROSE minute satisfied by a mention still reads ok (only `at` is held to a stamp)",
      [w for k, _, w in check(pr, AART)[1] if k == "TIME"] == ["ok"])
    t("stamped_minutes reads a date: header and a wildcard heading", stamped_minutes(AART) == ({"12:18"}, {"18:2"}))
    t("a pass range the header states ('06:31Z to 06:32Z', the allium shape) covers its minutes",
      stamped_minutes("# Part A (06:31Z to 06:32Z)\nfirst stamp 06:32:00Z")[0] == {"06:31", "06:32"})
    zr = {"reproducibility": {"artifactsUrl": "https://p.example/evidence/z/2026-10-08/probes-1805Z.txt"}}
    t("a sibling named 'beside the artifact' is also tried under each date the prose names (the zapier shape)",
      siblings(zr, "re-queried at 06:05:44Z on 2026-10-09, recorded in dns-txt-0605Z.txt beside this record's artifact") ==
      ["https://p.example/evidence/z/2026-10-08/dns-txt-0605Z.txt", "https://p.example/evidence/z/2026-10-09/dns-txt-0605Z.txt"])
    cr = {"surface": "https://o.example/api/search", "request": {"method": "GET"}, "at": "2026-09-28T18:23Z", "finding": "",
          "observed": {"decoded": {"a": "mcp.allium.so was open at 06:31Z on 2026-09-28 and closed at 18:04Z (p-20260928-allium-mcp-gate-closed)",
                                   "b": "re-measured 2026-09-22 12:06Z with the successor script",
                                   "c": "counted by hand at 12:16Z"}}}
    got = [(v, w) for k, v, w in check(cr, AART)[1] if k == "TIME"]
    t("a clock attributed to a named record or another date reads ok (attributed), a bare one is still a CANDIDATE",
      [w.startswith("ok") for _, w in got] == [True, True, True, False] and got[3][0] == "12:16Z")
    ir = {"reproducibility": {"artifactsUrl": "https://p.example/evidence/i/2026-09-17/probes-1815Z.txt"}}
    t("a .md document the sentence names by full URL is tried as a sibling's home (the folded-file shape)",
      siblings(ir, "checked at 18:16Z; the transcript, extra-well-known-1816Z.txt, is published whole in the README at https://p.example/evidence/i/2026-09-17/README.md under Folded files")[:2] ==
      ["https://p.example/evidence/i/2026-09-17/README.md", "https://p.example/evidence/i/2026-09-17/extra-well-known-1816Z.txt"])
    t("a named file that an answered document carries in its text is found, not missed; one no document carries stays missed",
      still_missed([("r", 404, "https://p.example/i/extra-well-known-1816Z.txt"), ("r", 404, "https://p.example/i/gone-1900Z.txt")],
                   [("README.md", "## Folded files\n### extra-well-known-1816Z.txt\n1504 bytes")]) ==
      [("r", 404, "https://p.example/i/gone-1900Z.txt")])
    print("selftest", "PASS" if ok else "FAIL"); return 0 if ok else 1

if __name__ == "__main__":
    ap = argparse.ArgumentParser(); ap.add_argument("--clone"); ap.add_argument("--only"); ap.add_argument("--selftest", action="store_true")
    a = ap.parse_args()
    if a.selftest: sys.exit(selftest())
    if not a.clone: ap.error("--clone or --selftest")
    run(a.clone, a.only)