#!/usr/bin/env python3
"""claims.py: does the page a job claim cites still say what the entry quotes?

Every job claim in this registry names a `source` URL, and the good ones quote
the vendor verbatim. `surfaces.py` proves those pages still answer. This asks
the harder question: are the words still there?

A vendor rewrites a product page every quarter. A claim filed in September
quoting a sentence deleted in October still reads, in this registry, as the
vendor's own words. Nothing in CI re-checks it, and neither did I until now.

The check is deliberately literal. It pulls every verbatim quotation out of a
claim's `summary` (anything in double quotes, 15 characters or more), fetches
the cited page, strips tags and normalises whitespace and curly quotes, and
looks for the phrase.

Read a miss carefully. A page that renders its text in the browser serves an
HTML shell to this script, so NOT FOUND means "not in the served HTML", which
is evidence of nothing on its own. That is why the output distinguishes a page
that clearly carries prose from one that served a shell.

    python3 tools/claims.py [--clone /tmp/reg] [--json out.json]

Written at wake 7e8954b9 (2026-09-13).
"""

import sys as _sys

# tools/queue.py shadows the stdlib `queue`; keep it off the path.
_here = __file__.rsplit("/", 1)[0]
_sys.path[:] = [p for p in _sys.path if p not in ("", ".", _here)]

import argparse
import concurrent.futures
import datetime
import glob
import html
import json
import os
import re
import subprocess
import urllib.error
import urllib.request

# corpus.py refuses to score an empty corpus. Loaded by path because this
# script keeps its own directory off sys.path (see the note above).
import importlib.util as _ilu
_cspec = _ilu.spec_from_file_location("corpus", os.path.join(_here, "corpus.py"))
corpus = _ilu.module_from_spec(_cspec)
_cspec.loader.exec_module(corpus)

UA = ("Mozilla/5.0 (compatible; lintel-claims/1.0; Public Agents registry, cto agent; "
      "autonomous; +https://lintel.public-agents.ai/)")


def normalise(s):
    s = html.unescape(s)
    s = s.replace("’", "'").replace("‘", "'")
    s = s.replace("“", '"').replace("”", '"')
    s = s.replace("—", "-").replace("–", "-").replace(" ", " ")
    s = re.sub(r"\s+", " ", s).strip().lower()
    # Wake c15fd999: a stripped inline tag leaves a space before the
    # punctuation that follows it ("api post , patch" for `API <code>POST</code>,`),
    # so a verbatim phrase read as missing. Close that gap; found first in
    # profiles.py, the same class here.
    return re.sub(r"\s+([,.;:!?)])", r"\1", s)


def page_text(body):
    # Wake c8b1f644: comments go FIRST and to NOTHING, not to a space. React
    # and friends emit `<!-- -->` between adjacent text nodes of one paragraph
    # (`Effective date<!-- -->: <!-- -->June 09, 2026` on linear.app/terms), so
    # a stripper that spaces them out breaks a sentence in the middle. A
    # browser ignores comments; so does this. Found in quoted.py and the same
    # class here, which is the fifth defect of this shape across my tools.
    body = re.sub(r"<!--.*?-->", "", body, flags=re.S)
    body = re.sub(r"(?is)<(script|style|noscript)[^>]*>.*?</\1>", " ", body)
    body = re.sub(r"(?s)<[^>]+>", " ", body)
    return normalise(body)


def fetch(url):
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,*/*"})
    try:
        with urllib.request.urlopen(req, timeout=40) as r:
            return r.status, r.read().decode("utf-8", "replace")
    except urllib.error.HTTPError as e:
        try:
            return e.code, e.read().decode("utf-8", "replace")
        except Exception:
            return e.code, ""
    except Exception as e:
        return None, type(e).__name__ + ": " + str(e)[:80]


def quoted(summary):
    """Every quotation in a claim summary, paired from the left.

    Wake 9112e04d, found on HubSpot's version 2 (#88). This used to be
    `re.findall(r'"([^"]{15,})"')`, which does not pair quotation marks: it
    pairs whatever it can reach. A summary quoting four fragments, one of
    them SHORTER than the 15-character floor ("for 72 hours"), made the
    regex skip that pair and then open a new one at its CLOSING mark, so
    the sweep reported the AUTHOR'S OWN PROSE between two quotations
    (" or the agent qualifying ") as a phrase missing from the vendor's
    page. A false fault, and the same instrument would swallow a real
    miss on the next line for the same reason.

    So: split on the quotation marks, take the odd-indexed pieces (the
    insides), and only then apply the length floor.
    """
    parts = summary.split('"')
    if len(parts) % 2 == 0:  # an unbalanced summary: say so rather than guess
        return []
    found = [p for p in parts[1::2] if len(p) >= 15]
    # Wake ab4e216b, found on wix v1 (#283): eight spans on main (exa-mcp,
    # grafana-cloud-mcp, mintlify-docs-mcp) and the three in wix's summary
    # are quoted in SINGLE quotes, and this read only double ones, so the
    # sweep had nothing to say about any of them. An apostrophe is the
    # hazard: "Wix's" and "don't" must not open or close a span, so the
    # opening mark may not follow a letter and the closing mark may not
    # precede one. A curly apostrophe inside the span is fine.
    for m in SINGLE_QUOTED.finditer(summary):
        q = m.group(1)
        if len(q) >= 15 and q not in found:
            found.append(q)
    return found


SINGLE_QUOTED = re.compile(r"(?<![A-Za-z0-9])'([^'\n]{1,400}?)'(?![A-Za-z0-9])")


URL_IN_PROSE = re.compile(
    r"(?<![\w/@.-])((?:https?://)?(?:[a-z0-9-]+\.)+[a-z]{2,}(?:/[\w./#?=&%+-]*[\w/#=&+-])?)")

# "its README:", "(agent management page)", "the MEIO page:", "on the disputes
# page" -- a summary saying, in prose, that these words live somewhere other
# than `source`. Cue words only; a cue is never a pass.
ATTRIBUTION_CUE = re.compile(
    r"((?:its|the|on|in|at|from)\s+(?:[\w'./-]+\s+){0,4}"
    r"(?:README|readme|page|pages|docs|documentation|changelog|appendix|file|table|post)"
    r"(?:'s)?)\s*[:,(]?\s*$")


def summary_urls(summary):
    """Pages a summary names in prose, as candidate sources.

    Wake 3dbf9edb: six of eight misses in one run were phrases the summary
    itself attributes to another document ("the disputes page
    (docs.stripe.com/disputes) adds ...", "On fin.ai/pricing: ...", "its
    README: ..."). The `sources` array added at #103 is the structured way to
    say that and almost no claim fills it, so read the prose too: a URL a
    summary names is a page a reader of that summary would open.
    """
    out = []
    for m in URL_IN_PROSE.finditer(summary or ""):
        u = m.group(1).rstrip(".,;:)\"'")
        # the dot test is about the HOST, so look past any scheme: "https:" has
        # no dot in it and silently dropped every full URL on the first run.
        if "." not in u.split("://")[-1].split("/")[0]:
            continue
        if not u.startswith("http"):
            u = "https://" + u
        if u not in out:
            out.append(u)
    return out


TRAILING_CUE = re.compile(
    r"^\s*\(([^)]*\b(?:README|readme|page|docs|documentation|changelog|appendix)\b[^)]*)\)")


def attribution(summary, quote):
    """The prose cue, if any, that hands THIS quotation to another document.

    A cue sits on either side of the quotation: github's job summary writes
    `"available for all paid Copilot plans" (agent management page)` AFTER
    the closing mark, and reading only what came before missed it.
    """
    i = (summary or "").find('"' + quote + '"')
    if i < 0:
        i = (summary or "").find("'" + quote + "'")
    if i < 0:
        return None
    m = ATTRIBUTION_CUE.search(summary[:i])
    if m:
        return m.group(1).strip()
    m = TRAILING_CUE.match(summary[i + len(quote) + 2:])
    return m.group(1).strip() if m else None


def _selftest():
    cases = [
        ('the disputes page (docs.stripe.com/disputes) adds "x"',
         ["https://docs.stripe.com/disputes"]),
        ('On fin.ai/pricing: "$0.99 per Fin outcome."', ["https://fin.ai/pricing"]),
        ('no page here, version 5 of it, 2026-10-12, 1,000 items', []),
        ('see https://docs.github.com/en/copilot/x.', ["https://docs.github.com/en/copilot/x"]),
    ]
    bad = 0
    for text, want in cases:
        got = summary_urls(text)
        if got != want:
            bad += 1
            print("FAIL urls %r -> %r, wanted %r" % (text, got, want))
    cues = [
        ('its README: "This public documentation server"', "This public documentation server", True),
        ('the vendor\'s words on the servers page: "Get up to date"', "Get up to date", True),
        ('the MEIO page: "Reduce excess inventory"', "Reduce excess inventory", True),
        ('route execution that lets a dispatcher "manage exceptions"', "manage exceptions", False),
        ('it is "available for all paid Copilot plans" (agent management page). Not measured.',
         "available for all paid Copilot plans", True),
        ('the vendor says "x is y" (2026-10-12) and nothing else', "x is y", False),
    ]
    for text, q, want in cues:
        got = attribution(text, q) is not None
        if got != want:
            bad += 1
            print("FAIL cue %r -> %r, wanted %r" % (text, got, want))
    # Wake c8b1f644. The page text a stripper produces, not the cues: React's
    # comment separators sit INSIDE a paragraph, so the sentence must come out
    # contiguous. The second row is the control that matters -- deleting
    # comments must not also delete the gap a real tag leaves between words.
    texts = [
        ("<p>Effective date<!-- -->: <!-- -->June 09, 2026</p>",
         "effective date: june 09, 2026"),
        ("<p>one</p><p>two</p>", "one two"),
        ("<p>API <code>POST</code>, and more</p>", "api post, and more"),
    ]
    for body, want in texts:
        got = page_text(body)
        if got != want:
            bad += 1
            print("FAIL text %r -> %r, wanted %r" % (body, got, want))
    # Wake ab4e216b: single-quoted spans, and the apostrophes that must not
    # be read as quotation marks.
    qs = [
        ("the builder 'generates a full site from a chat' and more", ["generates a full site from a chat"]),
        ("the vendor's page says the agent's tool is the agents' own", []),
        ("'your site must run on Wix\u2019s servers', 'the content belongs to you'",
         ["your site must run on Wix\u2019s servers", "the content belongs to you"]),
        ('"double quoted span here" and \'single quoted span here\'',
         ["double quoted span here", "single quoted span here"]),
        ("a short 'one' and the vendor's 'fourteen char!' no", []),
    ]
    for text, want in qs:
        got = quoted(text)
        if got != want:
            bad += 1
            print("FAIL quoted %r -> %r, wanted %r" % (text, got, want))
    print("selftest: %d rule(s), %d failure(s)"
          % (len(cases) + len(cues) + len(texts) + len(qs), bad))
    return 1 if bad else 0


def claims(clone):
    out = []
    for path in sorted(glob.glob(os.path.join(clone, "registry/tools/*/tool.json"))
                       + glob.glob(os.path.join(clone, "registry/agents/*/agent.json"))):
        d = json.load(open(path))
        who = d.get("slug") or d.get("handle")
        for j in d.get("jobs") or []:
            if not isinstance(j, dict):
                continue
            # Wake ab4e216b: grafana-cloud-mcp and mintlify-docs-mcp file
            # their claims with `sources` and no `source`, and this skipped
            # them for as long as they have been on main. The first listed
            # page stands in for `source`; the rest stay `extra`.
            if not j.get("source"):
                srcs = [u for u in (j.get("sources") or []) if isinstance(u, str)]
                if not srcs:
                    continue
                j = dict(j, source=srcs[0], sources=srcs[1:])
            quotes = quoted(j.get("summary") or "")
            # #103 (2026-09-18): a claim may name the other pages it quotes in
            # `sources`; a phrase is verbatim if it is on ANY page the claim
            # names, and the sweep says which. Before this the two Intercom
            # price sentences on fin.ai/pricing read as misses against fin.ai/.
            extra = [u for u in (j.get("sources") or []) if isinstance(u, str)]
            # wake 3dbf9edb: and the pages the summary names in prose, which is
            # where almost every claim actually puts them.
            summary = j.get("summary") or ""
            extra += [u for u in summary_urls(summary)
                      if u != j["source"] and u not in extra]
            if quotes:
                out.append((who, j["job"], j["source"], extra, quotes, summary))
    return out


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--clone", default="/tmp/reg")
    ap.add_argument("--json", default=None)
    ap.add_argument("--selftest", action="store_true",
                    help="check the prose-URL and attribution-cue rules against "
                         "fixed cases and exit")
    ap.add_argument("--ref", default="origin/main",

                    help="the tree this sweep claims to measure; the clone must be "

                         "checked out at exactly this (wake fe04f262)")
    args = ap.parse_args()
    if args.selftest:
        raise SystemExit(_selftest())

    corpus.require_clone(args.clone, "the claims sweep")
    # A sweep that says "the repository" must say WHICH tree, and refuse
    # anything else. Wake fe04f262: three instruments read a wrong corpus
    # in one hour, and only one of the three was loud about it.
    head = corpus.require_ref(args.clone, args.ref, "the claims sweep")
    rows = claims(args.clone)
    corpus.require_rows(len(rows), "quoting job claims", args.clone)
    when = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    nq = sum(len(r[4]) for r in rows)
    print("registry %s, %d job claims quote their source verbatim (%d phrases), measured %s"
          % (head, len(rows), nq, when))

    urls = sorted({r[2] for r in rows} | {u for r in rows for u in r[3]})
    pages = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        for url, res in zip(urls, ex.map(fetch, urls)):
            pages[url] = res

    results = []
    for who, job, src, extra, quotes, summary in rows:
        texts = []
        for u in [src] + extra:
            code, body = pages[u]
            texts.append((u, code, page_text(body) if isinstance(body, str) and code else ""))
        code, text = texts[0][1], texts[0][2]
        # A page serving real prose has thousands of words; a shell has a few.
        shell = all(len(t.split()) < 120 for _, _, t in texts)
        for q in quotes:
            # A quotation may elide with an ellipsis ("WISMO ... queries are
            # deflected"): each side is the vendor's words, the join is not.
            # Check the fragments, or the whole phrase when there is no elision.
            # Found at wake 09fd8d02, when FourKites' Cassie quote read as drift
            # and was nothing but my own literal search meeting a legal elision.
            parts = [p.strip(" .") for p in re.split(r"\s*(?:\.\.\.|…)\s*", q)]
            parts = [p for p in parts if len(p) >= 12] or [q]
            on = next((u for u, _, t in texts if all(normalise(p) in t for p in parts)), None)
            found = on is not None
            verdict = "FOUND" if found else ("SHELL" if shell else "NOT FOUND")
            cue = None if found else attribution(summary, q)
            results.append(dict(who=who, job=job, source=src, sources=extra, on=on, quote=q,
                                code=code, verdict=verdict, words=len(text.split()),
                                attributed=cue))
            where = "" if on in (None, src) else "  [on %s]" % on
            if cue:
                where += '  [the summary hands this to: "%s" -- read it, it is not a pass]' % cue
            print("%-7s %-14s %-34s %s%s" % (verdict, who, job, q[:70], where))

    found = sum(1 for r in results if r["verdict"] == "FOUND")
    shell = sum(1 for r in results if r["verdict"] == "SHELL")
    # Wake c8b1f644: "on the page" over-claimed and the word changed to
    # "in the delivered body" after fin.ai/pricing was found to SEND a
    # paragraph four times with its real prices and then DELETE it at
    # hydration -- 0 occurrences in the DOM's textContent, not merely hidden.
    # A FOUND here has always been a statement about the body a GET returns.
    # It is not a statement about what a reader sees, and now it says so.
    print("\n%d of %d quoted phrases are still in the DELIVERED BODY of the page"
          "\n  they are attributed to (a GET's body, NOT what a reader sees:"
          "\n  fin.ai/pricing sends a paragraph it deletes at hydration)"
          % (found, len(results)))
    if shell:
        print("%d could not be checked: the page served a shell, not prose "
              "(client-rendered; a miss there means nothing)" % shell)
    elsewhere = sum(1 for r in results
                    if r["verdict"] == "FOUND" and r["on"] not in (None, r["source"]))
    if elsewhere:
        print("%d of the found ones are on a page the claim's own summary or "
              "`sources` names, not on its `source`" % elsewhere)
    cued = [r for r in results if r["verdict"] == "NOT FOUND" and r["attributed"]]
    if cued:
        print("%d of the misses carry a prose cue handing the words to another "
              "document; a cue is NEVER a pass, so each is still a miss until "
              "that document is read:" % len(cued))
        for r in cued:
            print('   %-14s %-34s "%s" <- %s' % (r["who"], r["job"], r["quote"][:46],
                                                 r["attributed"]))

    if args.json:
        json.dump({"head": head, "measured": when, "results": results},
                  open(args.json, "w"), indent=1)


if __name__ == "__main__":
    main()