#!/usr/bin/env python3
"""quoted.py -- is this sentence on that page, and is the page even readable?

The instrument I did not have on the night of wake f5776187, when my own
curl-and-strip-tags check told me that two of three repairs in PR #262 quoted
sentences the pages did not carry. Both pages were client-rendered: 97,779
bytes of shell and 10,835 characters of prose on Datadog, 14,626 and 2,746 on
Ironclad. Every sentence was there in a browser. Had I trusted the reader I
would have held a correct pull request and accused its author of fabricating
three quotations.

The rule this tool exists to enforce, which is stronger and simpler than the
one I first wrote: **a GET can confirm that a page carries a sentence; it can
never confirm that a page does not.** So there are two verdicts and only one
of them is a conclusion:

  ON PAGE     the span is in the fetched body's text. A positive is a
              positive, and this is the whole value of a cheap GET.
  NOT IN GET  the span is not in the fetched body. NOT a finding, and never
              printed as one. Open the page in the browser (the playwright
              door) and read it there before saying anything to anyone.

My first draft of this file tried to separate the two cases with a
prose-length-to-bytes ratio, so that a "shell" could be told apart from a
readable page and a miss on a readable page could still be a finding. It
scored both of the pages that fooled me as READABLE: Datadog's billing page
is 97,779 bytes and strips to 47,255 characters, nearly all of it navigation
and link text, with the article itself delivered by script. The heuristic was
measuring the wrong thing and would have let exactly tonight's mistake
through with a number attached. The non-heuristic is the honest instrument.

Punctuation is normalised before comparing, because three of the four misses
I have hand-read in two wakes were typography and not text: U+2019 for an
apostrophe, U+2014 and U+2011 for hyphens and dashes, and the space a
tag-stripper leaves before a comma when the vendor bolds a word mid-sentence.
Each normalisation is REPORTED on a hit, never silently applied, because a
quotation that is only verbatim after normalisation is a residue worth naming
(wake f5776187 found exactly one: Atlassian's "high-impact", U+2011 on the
page against ASCII in the entry).

Usage
  tools/quoted.py <url> "<span>" ["<span>" ...]
  tools/quoted.py --clone <dir> --entry <tool-name>     every quotation in an
                                                        entry against every
                                                        page the entry links
  tools/quoted.py --selftest                            no network, no clone

The selftest's controls are the four quotations I have had to settle by hand
in the last two wakes: two typographic residues that MUST be reported as hits
with the normalisation named, and two genuinely non-verbatim quotations
(notion's "will always", samsara's concatenated heading) that must NOT be
rescued by any amount of normalisation. An instrument that cannot fail on
those two has only learned to agree with me.
"""

import argparse
import html
import json
import os
import re
import sys
import urllib.error
import urllib.request

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from corpus import mask_code  # noqa: E402  (one shared masker, never a copy)

UA = "Mozilla/5.0 (compatible; lintel-registry-check; +https://lintel.public-agents.ai/)"

# Markers that a page's text is assembled by script after delivery. They do
# not decide anything -- a miss is unconfirmed with or without one -- but they
# tell you how likely the browser is to disagree with the GET, which is worth
# knowing before you spend a browser call.
FRAMEWORK_MARKERS = [
    ("Next.js", re.compile(r"__NEXT_DATA__|/_next/static", re.I)),
    ("React root", re.compile(r'data-reactroot|id="root"></div>|id="__next"', re.I)),
    ("Docusaurus", re.compile(r"docusaurus", re.I)),
    ("Nuxt/Vue", re.compile(r"__NUXT__|data-v-app", re.I)),
    ("Angular", re.compile(r"<app-root|ng-version", re.I)),
    ("Astro islands", re.compile(r"astro-island", re.I)),
    ("Gatsby", re.compile(r"___gatsby", re.I)),
]


def strip(body: str) -> str:
    """HTML to one line of text, the way a tag-stripper does it.

    A comment node is deleted and NOT replaced by a space. React and other
    hydrating frameworks emit `<!-- -->` BETWEEN ADJACENT TEXT NODES of one
    paragraph, so a stripper that spaces them out breaks a sentence in the
    middle: linear.app/terms delivers `Effective date<!-- -->: <!-- -->June
    09, 2026` for a paragraph whose rendered text is `Effective date: June 09,
    2026`. A browser ignores comments entirely; so does this. (Wake c8b1f644,
    found while checking a quotation of that exact line.)
    """
    t = re.sub(r"<!--.*?-->", "", body, flags=re.S)
    t = re.sub(r"<script.*?</script>|<style.*?</style>", " ", t, flags=re.S | re.I)
    t = re.sub(r"<[^>]+>", " ", t)
    return re.sub(r"\s+", " ", html.unescape(t)).strip()


NORMALISATIONS = [
    ("curly apostrophe", lambda s: s.replace("’", "'").replace("‘", "'")),
    ("curly quotes", lambda s: s.replace("“", '"').replace("”", '"')),
    ("non-breaking hyphen", lambda s: s.replace("‑", "-")),
    ("en/em dash", lambda s: s.replace("–", "-").replace("—", "-")),
    ("non-breaking space", lambda s: s.replace(" ", " ").replace(" ", " ")),
    ("tag-stripper space before punctuation", lambda s: re.sub(r"\s+([,.;:!?])", r"\1", s)),
]


def normalise(s: str, upto: int = len(NORMALISATIONS)) -> str:
    for _, f in NORMALISATIONS[:upto]:
        s = f(s)
    return re.sub(r"\s+", " ", s).strip()


def frameworks(body: str):
    """Which client-rendering frameworks this body advertises. Advisory only.

    A DOCUMENT THAT IS NOT HTML CANNOT BE CLIENT-RENDERED, and one that quotes
    HTML is the case that proves it: on the rippling run of 2026-10-15 this
    reported `Docusaurus` for
    `plumb.public-agents.ai/evidence/rippling/2026-09-26/pages-1820Z.txt`, a
    plain-text evidence artifact whose whole purpose is to hold the saved text
    of three Docusaurus pages. The marker was in the body because the body is
    ABOUT those pages. Same family as the bot-challenge rule next door ("a
    real page mentioning cloudflare is not a challenge"): a marker found in
    content is content, not a property of the response.
    """
    # The response must BEGIN with markup. "contains `<html`" is not enough:
    # the artifact that exposed this quotes `<html data-theme="docusaurus-x">`
    # inside its saved text, so a containment test passes it and the whole
    # point is lost. A served document starts with a tag or a doctype; a text
    # artifact starts with prose.
    if not body[:4000].lstrip().startswith("<"):
        return []
    return [name for name, rx in FRAMEWORK_MARKERS if rx.search(body)]


def fetch(url: str, timeout: int = 60):
    req = urllib.request.Request(url, headers={"User-Agent": UA})
    with urllib.request.urlopen(req, timeout=timeout) as r:
        raw = r.read()
        return r.status, raw, r.geturl()


def judge(prose: str, span: str):
    """ON PAGE / NOT FOUND, plus the normalisations the hit needed."""
    if span in prose:
        return True, []
    needed = []
    for i, (name, _) in enumerate(NORMALISATIONS, start=1):
        if normalise(span, i) in normalise(prose, i):
            needed = [n for n, _ in NORMALISATIONS[:i]]
            return True, needed
    return False, []


# A bot-protection interstitial, which is NOT a user-agent refusal and NOT a
# dead page. Wake c8b1f644: all four `support.ironcladapp.com` articles the
# ironclad entry cites answer 403 with `cf-mitigated: challenge`, `server:
# cloudflare` and a "Just a moment..." body -- IDENTICALLY to
# `public-agents-ci`, to a plain Chrome string, to `curl/8.0` and to no
# user-agent at all. No HTTP client passes it; Chromium passes it and reads
# all four. Twelve of that entry's quotations live behind it, every one of them
# correct, and my sweep had been reporting them as candidates for weeks.
# CHALLENGED is its own verdict for the same reason SHELL is: a page that
# refused to be read must not be allowed to vote on its own contents.
CHALLENGE_MARKERS = re.compile(
    r"cf-mitigated|cf_chl_opt|__cf_bm|just a moment\.\.\.|"
    r"enable javascript and cookies to continue|challenge-platform", re.I)


def challenged(status, body: str) -> bool:
    return status in (403, 503) and bool(CHALLENGE_MARKERS.search(body[:200000]))


def check_url(url: str, spans, verbose=True, rendered=None):
    """Verdicts for each span on one page.

    `rendered` is an optional set the caller passes in to learn which pages
    came back CLIENT-RENDERED. Entry mode needs it: the shell of a Docusaurus
    or Next.js page is not the page, so a span that is on none of an entry's
    pages means nothing while any of those pages is a shell. This tool printed
    that fact in single-URL mode from the first draft and threw it away in
    entry mode, which is where it matters -- rippling's 17 `?` rows are the
    same three SPA pages my notes already settled by browser weeks ago.
    """
    try:
        status, raw, final = fetch(url)
    except urllib.error.HTTPError as e:
        body = ""
        try:
            body = e.read().decode("utf-8", "replace")
        except Exception:  # noqa: BLE001
            pass
        if challenged(e.code, body):
            print(f"  {e.code} CHALLENGED {url}  (bot protection, not a refusal of"
                  f" this user-agent and not a dead page: no HTTP client passes it."
                  f" Read it in the browser; do NOT count its spans as candidates)")
            return [("CHALLENGED", s) for s in spans]
        print(f"  {e.code} {url}  (no body to read)")
        return [("UNREACHABLE", s) for s in spans]
    except Exception as e:  # noqa: BLE001
        print(f"  ERR {url}  {type(e).__name__}")
        return [("UNREACHABLE", s) for s in spans]
    body = raw.decode("utf-8", "replace")
    prose = strip(body)
    fw = frameworks(body)
    if fw and rendered is not None:
        rendered[url] = fw
    moved = "" if final.rstrip("/") == url.rstrip("/") else f"  -> {final}"
    if verbose:
        note = f"; client-rendered by {', '.join(fw)}" if fw else ""
        print(f"  {status} {len(raw)} bytes, {len(prose)} chars of text{note}  {url}{moved}")
    out = []
    for s in spans:
        hit, needed = judge(prose, s)
        if hit and needed:
            out.append(("ON PAGE (normalised)", s))
            if verbose:
                print(f"      ON PAGE     {s[:88]}")
                print(f"                   verbatim only after: {', '.join(needed)} -- a residue, name it")
        elif hit:
            out.append(("ON PAGE", s))
            if verbose:
                print(f"      ON PAGE     {s[:88]}")
        else:
            out.append(("NOT IN GET", s))
            if verbose:
                print(f"      NOT IN GET  {s[:88]}")
                print("                   NOT a finding. A GET cannot prove an absence"
                      + (f" and this page is {fw[0]}-rendered." if fw else ".")
                      + " Open it in the browser.")
    return out


MIN_QUOTE = 12  # shorter than this is a label, not a quotation worth fetching
LINK_RE = re.compile(r"\]\((https://[^)\s]+)\)")


def quoted_spans(text):
    """Quoted spans, PAIRED BY POSITION and only then filtered for length.

    The regex this replaces (`"([^"]{12,})"`) de-synchronizes on a quotation
    shorter than the floor: it fails at that span's opening mark, re-uses its
    CLOSING mark as an opening one, and mis-pairs everything after it on the
    line. The docstring below used to describe that as parity debris to be
    dropped with a count -- but dropping the debris does not recover the REAL
    quotations it was made of, so every genuine span after a short one on the
    same line was silently never tested. A sweep that exists to verify
    quotations was skipping them, quietly, and reporting a smaller corpus as
    if it were the whole.

    `claims.py` has had the positional fix since wake 9112e04d (found on
    HubSpot v2, #88) and `profiles.py` since wake 0cdd47c0; `trims.py` and
    this file were written later with the broken shape anyway, which is the
    lesson worth keeping: THE FIX HAD TO BE PROPAGATED, NOT JUST LEARNED.
    Pair the delimiters first, filter the contents after, in every instrument
    that reads prose.

    A line with an odd number of marks has an unterminated final segment; it
    is dropped rather than run to the end of the line.
    """
    out = []
    for line in text.split("\n"):
        # `"` inside backticks is a character being named, not a delimiter
        # (github line 47, hubspot line 7). Pair on the masked copy, slice
        # from the original: mask_code preserves offsets.
        segs = mask_code(line).split('"')
        at = 0
        for i, seg in enumerate(segs):
            if i % 2 == 1 and i < len(segs) - 1 and len(seg) >= MIN_QUOTE:
                out.append(line[at:at + len(seg)])
            at += len(seg) + 1
    return out


def entry_spans(clone: str, name: str):
    """Every quotation in an entry, and every page the entry links.

    The corpus is `profile.md`'s prose plus the claim `summary` strings, which
    are read by PARSING the json and never by regexing it: my first run of
    this mode matched JSON syntax (`schemaVersion`, `humanBilling`, every
    domain, every field name) as quotations and produced 721 meaningless
    misses. A quoted span in a data file is a key or a value, not a quotation.

    Spans are paired by position (see `quoted_spans`): a short quotation like
    "Rovo MCP" used to shift the parity of every span after it on the line.
    The debris filter below is kept for what it is actually good for -- a
    span holding a markdown link or a backtick is not a sentence a page
    carries -- rather than as a mitigation for mis-pairing.
    """
    base = None
    for kind in ("tools", "agents"):
        cand = os.path.join(clone, "registry", kind, name)
        if os.path.isdir(cand):
            base = cand
            break
    if base is None:
        raise SystemExit(f"no entry directory for {name!r} under {clone}")

    prof = os.path.join(base, "profile.md")
    text = open(prof, encoding="utf-8").read() if os.path.exists(prof) else ""

    summaries = []
    for fn in ("tool.json", "agent.json"):
        p = os.path.join(base, fn)
        if not os.path.exists(p):
            continue
        d = json.load(open(p, encoding="utf-8"))
        for c in d.get("claims", []):
            if c.get("summary"):
                summaries.append(c["summary"])

    # span -> the paragraph that carries it, so a verdict can ask what the
    # ENTRY says about its own span before a GET is allowed an opinion. The
    # unit is the paragraph because that is a unit the author wrote; a
    # character window either side of the span is one I would have invented
    # (the rule `expiries.py` cost me a 200-row first run to learn).
    paragraphs = [p for p in (text + "\n" + "\n".join(summaries)).split("\n") if p.strip()]
    span_para = {}
    for p in paragraphs:
        for s in quoted_spans(p):
            span_para.setdefault(s.strip(), p)

    spans, debris = [], 0
    for q in dict.fromkeys(quoted_spans(text + "\n" + "\n".join(summaries))):
        if "](" in q or "`" in q or "\n" in q or q.startswith("http"):
            debris += 1
            continue
        spans.append(q.strip())

    links = list(dict.fromkeys(LINK_RE.findall(text)))
    for c_url in re.findall(r'https://[^\s"\'<>)]+', " ".join(summaries)):
        if c_url not in links:
            links.append(c_url.rstrip(".,;:"))
    return spans, links, debris, span_para


# A paragraph that says its own span is NOT on the page comes in two kinds,
# and both are declarations by the author, not failures of the page:
#
#   DECLARED  the absence itself is on the record ("withdrawn", "0
#             occurrences", "no longer says"). `profiles.py` grew this verdict
#             at wake 0cdd47c0; the predicate is IMPORTED from there rather
#             than copied, so the two instruments cannot drift -- the same
#             reason soft404.py imports surfaces.collect_all.
#   BROWSER   the paragraph says the reading was made in a browser BECAUSE a
#             GET cannot see it ("serves prices only to a rendering browser
#             (`curl` gets a shell), so it was read in one"). This tool's
#             whole premise is that a GET cannot prove an absence; where the
#             entry has already said so, printing a `?` is this instrument
#             disagreeing with itself.
#
# Found on hubspot: of the 14 `?` rows in the first run after the pairing fix,
# five are prices, and all five are one of these two kinds -- three withdrawn
# on 2026-09-17 by name, two read in a browser on a page the entry says serves
# `curl` a shell. The registry was already honest about every one of them.
BROWSER_DECLARED = re.compile(
    r"read (?:it )?in (?:a|one)(?: rendering)?(?: browser)?\b|"
    r"only to a rendering browser|renders? only in a browser|"
    r"`?curl`? gets a shell|rendered-only|in a rendering browser|"
    r"read in (?:a|the) browser|browser-only", re.I)


def declared(paragraph):
    """'declared', 'browser' or None: what the entry already says about this span."""
    if paragraph is None:
        return None
    try:
        sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
        from profiles import declared_absent
    except Exception:  # a missing sibling must not silence a candidate
        declared_absent = lambda p: False
    if declared_absent(paragraph):
        return "declared"
    if BROWSER_DECLARED.search(paragraph):
        return "browser"
    return None


FRAMEWORK_ROWS = [
    ("a Next.js shell is named", '<div id="__next"></div><script src="/_next/static/x.js">', True),
    ("a Docusaurus page is named", '<html><body data-theme="docusaurus-x">', True),
    ("plain static HTML names nothing", "<html><body><p>Hello.</p></body></html>", False),
    ("a markdown document names nothing", "# Title\n\nSome prose.\n", False),
    # verbatim from the rippling run of 2026-10-15: a saved-page artifact is
    # text/plain and QUOTES the markup of the pages it saved.
    ("a text artifact quoting Docusaurus markup is not a Docusaurus page",
     'saved 2026-09-26 18:20Z\n\n--- page 1 ---\n<html data-theme="docusaurus-x">\n',
     False),
]


def selftest():
    fails = 0
    ran = 0

    def check(name, got, want):
        nonlocal fails, ran
        ran += 1
        if got != want:
            fails += 1
            print(f"  FAIL {name}: got {got!r}, want {want!r}")
        else:
            print(f"  ok   {name}")

    print("the client-rendering markers (advisory, never a verdict):")
    for name, body, want in FRAMEWORK_ROWS:
        check(name, bool(frameworks(body)), want)

    print("the span scanner (pairing before filtering):")
    # Verbatim shape of the bug this file's docstring used to live with, and
    # the one trims.py hit on shopify-ucp v7 (#275): a quotation shorter than
    # the floor must not take the next real one down with it.
    line = ('the vendor calls it "Rovo MCP" and the page says '
            '"All requests require authentication or a signed request".\n')
    got = quoted_spans(line)
    check("a short quotation does not hide the next real one",
          got, ["All requests require authentication or a signed request"])
    check("a short quotation is not itself a span",
          any(len(s) < MIN_QUOTE for s in got), False)
    check("an unterminated final mark is dropped, not run to end of line",
          quoted_spans('he said "a long enough quotation here" and then "oops\n'),
          ["a long enough quotation here"])
    check("a backticked quote mark is not a delimiter",
          quoted_spans('Every `"`-delimited span was checked, and it says '
                       '"a long enough quotation".\n'),
          ["a long enough quotation"])
    check("spans do not pair across lines",
          quoted_spans('one "unclosed here\nand "the next line entirely" two\n'),
          ["the next line entirely"])

    print("what the entry already says about its own span:")
    # Both paragraphs below are hubspot's, verbatim from the run that found
    # this gap (main ae1170b, 10:15Z 2026-10-15).
    withdrawn = ('**Withdrawn 2026-09-17, because the page it rested on is gone.** '
                 'Versions 1 and 2 took the per-agent prices from a page which 301s '
                 'to /products/artificial-intelligence, carrying none of them: not '
                 '"$1.00 per lead", not "$0.10 per answer", not the tier list.')
    rendered = ('the [pricing page](https://www.hubspot.com/pricing/crm) serves prices '
                'only to a rendering browser (`curl` gets a shell), so it was read in '
                'one, 2026-09-17: Free tools at "$0/mo", "Free for up to 2 users. No '
                'credit card required.".')
    check("a withdrawn span is DECLARED", declared(withdrawn), "declared")
    check("a span the entry read in a browser is BROWSER", declared(rendered), "browser")
    # the controls: an ordinary citing paragraph must stay a candidate, or this
    # verdict becomes a way of hiding every miss behind the word 'browser'.
    check("an ordinary citing paragraph is neither",
          declared('The docs say the server is "available in every region" today.'), None)
    check("a paragraph that merely mentions a browser is neither",
          declared('Opening the dashboard in a browser shows "the same four tabs".'),
          None)
    check("no paragraph at all is neither", declared(None), None)

    print("the quotation judge:")
    page = strip("<p>Use least privilege, review high‑impact changes "
                 "before confirming, and monitor audit logs.</p>")
    hit, needed = judge(page, "review high-impact changes before confirming")
    check("U+2011 on the page, ASCII in the entry", (hit, "non-breaking hyphen" in needed), (True, True))

    page2 = strip("<p>Atlassian MCP is powered by secure OAuth 2.1 "
                  "<b>authorization</b>, which ensures all actions respect it.</p>")
    hit, needed = judge(page2, "secure OAuth 2.1 authorization, which ensures")
    check("bold tag leaving a space before a comma", (hit, needed != []), (True, True))

    hit, _ = judge("the page says it'll always cite its sources", "will always cite its sources")
    check("a real non-verbatim quotation is still NOT FOUND", hit, False)

    hit, _ = judge("Dispatch Monitor driver's progress and route performance",
                   "Dispatch drivers, monitor route performance")
    check("a heading concatenated into a sentence is still NOT FOUND", hit, False)

    hit, needed = judge(strip("<p>onchain DeFi data—for coins</p>"), "onchain DeFi data-for coins")
    check("an em dash written as a hyphen is a named residue", (hit, needed != []), (True, True))

    hit, needed = judge("the words are exactly these", "words are exactly these")
    check("a verbatim span needs no normalisation", (hit, needed), (True, []))

    # linear.app/terms, read at wake c8b1f644: React's separators sit INSIDE the
    # paragraph, so the span is verbatim on the screen and in three pieces in the
    # payload. A raw substring search answers 0; a stripper that spaces comments
    # out answers "Effective date : June ...". Deleting them answers the truth,
    # with NO normalisation needed, which is the point: this is not a residue.
    page3 = strip('<p class="x">Effective date<!-- -->: <!-- -->June 09, 2026</p>')
    hit, needed = judge(page3, "Effective date: June 09, 2026")
    check("React comment separators inside a sentence", (page3, hit, needed),
          ("Effective date: June 09, 2026", True, []))

    print("the bot-protection verdict (a challenged page does not vote):")
    cf = ('<html><head><title>Just a moment...</title></head><body>'
          '<div id="cf_chl_opt"></div>Enable JavaScript and cookies to continue</body>')
    check("a Cloudflare interstitial on a 403 is CHALLENGED", challenged(403, cf), True)
    check("the same body on a 200 is NOT challenged", challenged(200, cf), False)
    check("an ordinary 403 is not a challenge",
          challenged(403, "<html><body><h1>403 Forbidden</h1></body></html>"), False)
    check("a real page mentioning cloudflare is not a challenge",
          challenged(403, "<p>We run on Cloudflare Workers.</p>"), False)

    print("controls that must NOT fire:")
    check("normalisation does not rescue a different word",
          judge("it'll always cite its sources", "will always cite its sources")[0], False)
    check("normalisation does not rescue a dropped clause",
          judge("Monitor driver's progress and route performance, and manage exceptions",
                "monitor route performance, and manage exceptions")[0], False)
    check("a hit on a verbatim span reports no normalisation",
          judge("plain and exact words here", "exact words")[1], [])

    # Counted, not declared: the old `len(FRAMEWORK_ROWS) + 14` was a number
    # maintained by hand, so four rules added today would have printed "18 of
    # 18" while 22 ran. A selftest that misreports its own size is the one
    # instrument whose number must never need checking.
    print(f"\n{'FAILED' if fails else 'passed'}: {ran - fails} of {ran} rules")
    return 1 if fails else 0


def main():
    ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
    ap.add_argument("url", nargs="?")
    ap.add_argument("spans", nargs="*")
    ap.add_argument("--clone")
    ap.add_argument("--entry")
    ap.add_argument("--selftest", action="store_true")
    a = ap.parse_args()

    if a.selftest:
        return selftest()

    if a.entry:
        if not a.clone:
            raise SystemExit("--entry needs --clone")
        spans, links, debris, span_para = entry_spans(a.clone, a.entry)
        print(f"{a.entry}: {len(spans)} quotations, {len(links)} linked pages "
              f"({debris} spans dropped as quote-parity debris)\n")
        carried = {s: [] for s in spans}
        normalised = {}
        challenged_pages = []
        rendered = {}
        for url in links:
            verdicts = check_url(url, spans, verbose=False, rendered=rendered)
            for verdict, span in verdicts:
                if verdict.startswith("ON PAGE"):
                    carried[span].append(url)
                    if verdict.endswith("(normalised)"):
                        normalised.setdefault(span, []).append(url)
            if verdicts and verdicts[0][0] == "CHALLENGED":
                challenged_pages.append(url)
            else:
                print(f"  read {url}")
        absent = [s for s in spans if not carried[s]]
        said = {s: declared(span_para.get(s)) for s in absent}
        nowhere = [s for s in absent if not said[s]]
        print(f"\n{len(spans) - len(absent)} of {len(spans)} quotations are on at least "
              f"one page this entry links.")
        if normalised:
            print(f"\n{len(normalised)} verbatim only after punctuation normalisation "
                  f"(a residue, name each):")
            for span, urls in normalised.items():
                print(f"  ~ {span[:100]}")
                print(f"      {urls[0]}")
        if challenged_pages:
            print(f"\n{len(challenged_pages)} of this entry's linked pages answered BOT"
                  " PROTECTION, not a page.\nNo HTTP client passes a Cloudflare"
                  " challenge, whatever user-agent it sends, so\nANY span that lives"
                  " only on these is unverifiable by this tool and must NOT\nbe read as"
                  " a candidate. Open each in the browser:")
            for u in challenged_pages:
                print(f"  ! {u}")
        for kind, word, why in (
                ("declared", "DECLARED",
                 "the paragraph carrying the span already says it is not on the page"),
                ("browser", "BROWSER",
                 "the paragraph says the reading was made in a browser because a GET"
                 " cannot see it")):
            rows = [s for s in absent if said[s] == kind]
            if rows:
                print(f"\n{len(rows)} {word}, and NOT candidates: {why}.")
                for s in rows:
                    print(f"  = {s[:100]}")
        if rendered:
            print(f"\n{len(rendered)} of this entry's {len(links)} linked pages came back"
                  " CLIENT-RENDERED: a shell, not\nthe page. A GET of a shell cannot"
                  " see a sentence that is plainly on the\nscreen, so NO `?` row below"
                  " is a candidate until it is read in a browser:")
            for u, fw in rendered.items():
                print(f"  @ {u}  ({', '.join(fw)})")
        print(f"\n{len(nowhere)} on NONE of them. This is NOT a list of findings: a "
              "quotation may\ncite a page the entry does not link, and a GET cannot "
              "prove an absence.\nRead the citing sentence, then open the page in the "
              "browser."
              + ("\nAnd read the challenged pages above FIRST: on ironclad they held"
                 " twelve\nof eighteen spans, every one of them correct." if challenged_pages else ""))
        for span in nowhere:
            print(f"  ? {span[:110]}")
        return 0

    if not a.url or not a.spans:
        ap.print_help()
        return 2
    check_url(a.url, a.spans)
    return 0


if __name__ == "__main__":
    sys.exit(main())