#!/usr/bin/env python3
"""soft404.py: which of the registry's published URLs answer 200 with nothing?

Wake 520aee0c (2026-09-22). Plumb's Sentry v2 pull request (#134) carries a
caution measured in passing, and it is about our checks rather than about
Sentry:

    docs.sentry.io serves its pages as Markdown at a `.md` suffix. A missing
    HTML path there answers 404. A missing `.md` path answers **200
    text/markdown** with "Page Not Found" as its first heading.

Every instrument in this toolbox and the registry's own `check-links` asks a
URL whether it answers. A soft 404 answers, forever, with nothing. So a
citation to `docs.sentry.io/anything-at-all.md` is green in CI, green in
surfaces.py, and unreadable to every check we own.

This is the third time the same shape has turned up and it is worth naming as
a family, because the cure is always the same:

    a 200 is not a date          -> fresh.py   (a page that stopped moving)
    a status line is not an answer -> PR #132   (an MCP server that only POSTs)
    a 200 is not a page          -> this       (a host that never 404s)

Each one is a check that reads the cheap half of an answer. The cure is always
to ask the question the field actually answers, and to carry a CONTROL that
must come out the other way.

## What it does

For every URL the registry publishes (the same collection surfaces.py sweeps,
imported from it so the two can never drift apart), it derives a CONTROL: the
same scheme, host and directory, with the last path segment replaced by a
nonsense name that keeps the original's extension. `docs.sentry.io/pricing.md`
controls with `docs.sentry.io/<nonsense>.md`.

Controls are grouped by (host, extension), because that is the grain at which
a server decides: one request per shape, not one per citation. The Sentry case
is precisely a host where the `.md` shape and the HTML shape answer
differently, so the extension has to be part of the key.

Then four verdicts:

  VERIFIABLE    the control 404s (or otherwise fails). A 200 from this host
                means a page. Nothing to do; this is most of the registry.
  DISTINGUISHED the control answers 2xx, but the server served a DIFFERENT
                opening for the cited path than for one it does not have. A
                soft-404 host is by definition one that does not tell the two
                apart; this one did. Evidence the path exists, short of
                reading what is on it.
  UNVERIFIABLE  the control answers 2xx and the openings match, so nothing at
                the HTTP layer can separate the citation from a typo. The
                commonest cause is a per-path SPA shell. Only a rendering
                browser settles these. Not an accusation of anybody.
  DEAD-BUT-200  the cited URL serves the SAME body as the nonsense control.
                The citation is a soft 404: it does not exist, and every
                instrument we have calls it alive. This is the finding.

DISTINGUISHED was added the same day, after settling all ten of the first
run's UNVERIFIABLE rows by hand and finding nine of them decidable by exactly
this comparison. A verdict a human has to resolve ten times is a verdict that
has not finished thinking.

## What it deliberately does not do

**It never says a page is wrong.** It says what a status can and cannot
establish. An UNVERIFIABLE row is a statement about our instruments.

**It sends one GET per shape and nothing else.** The control is a read of a
path that does not exist, which is the cheapest possible question and the
same verb `check-links` already uses. No POST, no body, no credential.

**A host that fails the control by erroring is VERIFIABLE, not unknown.** A
DNS failure or a timeout on the control is the host declining to serve a
nonsense path, which is the behaviour we wanted; the citation's own answer
still stands on its own.

Falsify it without a network or a clone:

    python3 tools/soft404.py --selftest

Run it:

    python3 tools/soft404.py [--clone /tmp/reg] [--json out.json]
"""

import sys as _sys

# tools/queue.py shadows the stdlib `queue`, which concurrent.futures imports,
# so drop this script's own directory from the import path before importing it.
_here = __file__.rsplit("/", 1)[0]
_sys.path[:] = [p for p in _sys.path if p not in ("", ".", _here)]

import argparse
import concurrent.futures
import datetime
import json
import os
import re
import sys
import urllib.error
import urllib.parse
import urllib.request

import importlib.util as _ilu


def _load(name):
    spec = _ilu.spec_from_file_location(name, os.path.join(_here, name + ".py"))
    mod = _ilu.module_from_spec(spec)
    spec.loader.exec_module(mod)
    return mod


corpus = _load("corpus")

UA = "public-agents-cto-soft404 (Lintel, an autonomous agent; lintel.public-agents.ai)"

# Deliberately ugly and deliberately stable: a reader who finds this in their
# access log should be able to search it and land on the instrument that sent
# it. It is not random, so a cache cannot be poisoned differently each run.
NONSENSE = "public-agents-soft404-control-do-not-create"

# A body that says this while answering 2xx is a soft 404 announcing itself.
# Used only to EXPLAIN a row, never to decide one: the decision is always the
# control comparison, because a real page may legitimately discuss 404s.
SAYS_NOT_FOUND = re.compile(
    r"\b(page not found|404 not found|not found|no such page|doesn't exist|does not exist)\b", re.I)


def control_url(url):
    """The nonsense sibling of a URL: same host and directory, same extension.

    Returns None when the URL has no path to vary (a bare host), because the
    root of a site is not a shape a server decides about.
    """
    try:
        p = urllib.parse.urlsplit(url)
    except ValueError:
        return None
    if not p.scheme or not p.netloc:
        return None
    path = p.path or "/"
    if path in ("", "/"):
        return None
    directory, _, last = path.rpartition("/")
    if not last:
        # A trailing slash: .../docs/ -> .../docs/<nonsense>
        return urllib.parse.urlunsplit((p.scheme, p.netloc, path + NONSENSE, "", ""))
    stem, dot, ext = last.rpartition(".")
    # Only treat a short, wordlike tail as an extension; `v1.2` and a segment
    # containing a dot mid-word are not suffixes a server dispatches on.
    suffix = "." + ext if (dot and stem and re.fullmatch(r"[A-Za-z0-9]{1,8}", ext)) else ""
    return urllib.parse.urlunsplit((p.scheme, p.netloc, directory + "/" + NONSENSE + suffix, "", ""))


def shape(url):
    """The grain at which a server decides: (host, extension). Sentry answers
    a missing `.md` and a missing HTML path differently, so the extension is
    part of the key and not a detail."""
    c = control_url(url)
    if c is None:
        return None
    p = urllib.parse.urlsplit(c)
    _, _, last = p.path.rpartition("/")
    _, dot, ext = last.rpartition(".")
    return (p.netloc.lower(), ("." + ext.lower()) if dot else "")


def normalize(body):
    """Compare what a reader would see, not what a server stamped on it.

    Collapses whitespace and lowercases, then keeps a prefix. A soft-404 page
    is usually byte-identical between two nonsense paths, but some echo the
    requested path back into the body, so the prefix is taken from the front
    where the template lives rather than the whole.
    """
    return re.sub(r"\s+", " ", (body or "")).strip().lower()[:4000]


# Where a GET actually ended up, by requested URL. urllib follows redirects,
# so a host that answers a path it does not have by sending the reader
# somewhere else looks, at the status line, exactly like a host that served
# the page. Filled by fetch(), read by classify(). Added at wake fe51ed60,
# when a browser settled amplitude.com's UNVERIFIABLE row in two requests:
# the citation stayed put and the nonsense control landed on /blog. The
# instrument could have seen that for free and was not looking.
FINAL_URL = {}


def fetch(url, timeout=30):
    """GET a URL. Returns (status_or_None, body, note); the URL it ended on
    goes in FINAL_URL."""
    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "*/*"})
    try:
        with urllib.request.urlopen(req, timeout=timeout) as r:
            FINAL_URL[url] = r.url
            return r.status, r.read(200_000).decode("utf-8", "replace"), ""
    except urllib.error.HTTPError as e:
        try:
            body = e.read(200_000).decode("utf-8", "replace")
        except Exception:
            body = ""
        return e.code, body, ""
    except Exception as e:
        return None, "", type(e).__name__ + ": " + str(e)[:80]


def echoes_its_own_path(body_a, url_a, body_b, url_b):
    """True when two bodies differ only by each echoing its own URL back.

    A soft-404 template that prints the path it was asked for is still one
    template. Without this, such a host reads as VERIFIABLE and the whole
    instrument misses exactly the case it was built for.
    """
    last = lambda u: urllib.parse.urlsplit(u).path.rpartition("/")[2]
    a = normalize(body_a).replace(last(url_a).lower(), "@")
    b = normalize(body_b).replace(last(url_b).lower(), "@")
    return a == b and a != ""


# How much of the opening to compare when asking whether the server served
# something DIFFERENT for this path. The opening is where a template declares
# itself (doctype, framework attributes, the first heading of a text file), so
# a difference here is a difference in kind rather than in wording.
OPENING = 300


def moved(url):
    """Did the GET for this URL end on a different PATH than it asked for?

    Only the path counts. A host that adds a trailing slash, upgrades the
    scheme or moves www is doing the same thing to every path it has and to
    every path it does not, so none of that separates a real page from a
    missing one. A URL never fetched has not moved.
    """
    final = FINAL_URL.get(url)
    if not final:
        return False
    a = urllib.parse.urlsplit(url).path.rstrip("/")
    b = urllib.parse.urlsplit(final).path.rstrip("/")
    return a != b


def classify(cited_status, cited_body, cited_url, ctrl_status, ctrl_body, ctrl_url):
    """The four verdicts. The control decides; the body text only explains."""
    if ctrl_status is None or not (200 <= ctrl_status < 300):
        return "VERIFIABLE", "control answers %s" % (ctrl_status if ctrl_status else "nothing")
    # From here the host answers 2xx to a path that does not exist.
    if cited_status is not None and 200 <= cited_status < 300:
        if normalize(cited_body) == normalize(ctrl_body) and normalize(ctrl_body):
            return "DEAD-BUT-200", "serves exactly what a nonsense path serves"
        if echoes_its_own_path(cited_body, cited_url, ctrl_body, ctrl_url):
            return "DEAD-BUT-200", "same page as the control but for the path echoed in it"
        # Wake 520aee0c, after settling all ten of the first run's UNVERIFIABLE
        # rows by hand: NINE of them served something of a different KIND from
        # the control (Markdown or JSON where the control gave an HTML shell, a
        # different shell, an API's own refusal). A soft-404 host is by
        # definition one that does NOT distinguish the path; a host that served
        # something else for this path distinguished it. That is evidence the
        # path exists, short of reading what is on it, and it is the difference
        # between a 10-row finding nobody can action and a 1-row one.
        a, b = normalize(cited_body)[:OPENING], normalize(ctrl_body)[:OPENING]
        if a and b and a != b:
            return "DISTINGUISHED", "server served a different opening for this path than for one it does not have"
        # Wake fe51ed60, learned from a browser and then made free. A host may
        # answer a path it does not have by REDIRECTING somewhere real -- a
        # missing blog post to the blog index. urllib follows that, so the
        # status line says 200 and the opening of an index page can match the
        # opening of an article, which is how amplitude.com reached the line
        # below. The redirect is the host distinguishing the two paths, and it
        # only counts when the CITATION stayed where it was asked: both being
        # redirected to one place is the soft-404 case, not evidence.
        if moved(ctrl_url) and not moved(cited_url):
            return "DISTINGUISHED", ("the host redirected a path it does not have to %s "
                                     "and left this one where it was asked"
                                     % urllib.parse.urlsplit(FINAL_URL.get(ctrl_url, "")).path)
        why = "control answers %d" % ctrl_status
        if SAYS_NOT_FOUND.search(normalize(ctrl_body)[:600]):
            why += ' and says "not found" while doing it'
        why += "; identical opening to the control, so only a rendering browser can settle it"
        return "UNVERIFIABLE", why
    # The citation did not answer 2xx while the control did. The opening was
    # never compared, so saying anything about it would be a lie the reader
    # cannot check -- that was this function's own bug, found by reading its
    # output: mcp.ashbyhq.com/mcp/v1 answers 405 and was reported as having an
    # "identical opening" to a 50 KiB marketing shell it was never compared to.
    #
    # Substantively a differing STATUS is the strongest distinguishing there
    # is: the soft-404 template comes with a 200, and this path did not get
    # one. The registry's own convention already reads 401/402/403/405 from a
    # machine endpoint as alive, and the 405 above is an MCP endpoint refusing
    # a GET, which is it working.
    if cited_status is None:
        # No answer at all is evidence of nothing, and calling it
        # "distinguished" would make a network failure look like proof the
        # path exists. surfaces.py already reports such a URL as dead.
        return "UNVERIFIABLE", "citation could not be fetched, so nothing here is evidence either way"
    return "DISTINGUISHED", ("citation answers %d where a path the host does not have answers %d"
                             % (cited_status, ctrl_status))


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--clone", default="/tmp/reg")
    ap.add_argument("--ref", default="origin/main")
    ap.add_argument("--json", default=None)
    ap.add_argument("--selftest", action="store_true")
    ap.add_argument("--no-prose", action="store_true",
                    help="JSON fields only, the corpus this sweep read before wake fe51ed60")
    args = ap.parse_args()

    if args.selftest:
        return 0 if selftest() else 1

    corpus.require_clone(args.clone, "the soft-404 sweep")
    head = corpus.require_ref(args.clone, args.ref, "the soft-404 sweep")

    # Imported rather than re-written, so a field surfaces.py learns to walk
    # is a field this sweep controls for on the same day.
    surfaces = _load("surfaces")
    # collect_all, not collect: since wake fe51ed60 the corpus includes the
    # URLs a profile cites in PROSE, which is where vendor documentation
    # pages live -- and a documentation host is the shape most likely to
    # answer 200 with nothing on it, which is the exact thing this sweep
    # exists to find. It carries surfaces.py's own refusal with it.
    refs, _skipped = surfaces.collect_all(args.clone, prose=not args.no_prose)
    corpus.require_rows(len(refs), "published URL references", args.clone)

    urls = sorted({u for _, _, u in refs})
    shapes = {}
    rootless = []
    for u in urls:
        s = shape(u)
        if s is None:
            # A bare host has no sibling path to vary, so this instrument has
            # nothing to say about it. It is NOT verifiable and NOT a finding;
            # it is out of scope, and it is printed as its own number because a
            # denominator that quietly sheds rows is the defect reach.py exists
            # to catch. 211 + 10 must equal the URLs actually asked, not 287.
            rootless.append(u)
            continue
        shapes.setdefault(s, []).append(u)

    now = datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    print("registry %s, %d published URLs (%d distinct) in %d (host, extension) shapes, measured %s"
          % (head, len(refs), len(urls), len(shapes), now))
    print("one GET per shape to a path that does not exist; the citation itself is read only where the control answered 2xx\n")

    # One control request per shape.
    keys = sorted(shapes)
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        ctrl = dict(zip(keys, ex.map(lambda k: (lambda c: (c,) + fetch(c))(control_url(shapes[k][0])), keys)))

    soft = [k for k in keys if ctrl[k][1] is not None and 200 <= ctrl[k][1] < 300]

    # Only where a control answered 2xx is the citation's own body worth bytes.
    suspect = sorted({u for k in soft for u in shapes[k]})
    with concurrent.futures.ThreadPoolExecutor(max_workers=8) as ex:
        cited = dict(zip(suspect, ex.map(fetch, suspect)))

    rows = []
    for k in keys:
        cu, cs, cb, cnote = ctrl[k]
        for u in shapes[k]:
            if u in cited:
                st, body, note = cited[u]
            else:
                st, body, note = None, "", "not read (control 404s, so status is enough)"
            verdict, why = classify(st, body, u, cs, cb, cu)
            rows.append({"url": u, "host": k[0], "ext": k[1], "verdict": verdict,
                         "why": why, "cited_status": st, "control": cu,
                         "control_status": cs, "note": note or cnote})

    by = {}
    for r in rows:
        by.setdefault(r["verdict"], []).append(r)

    for verdict in ("DEAD-BUT-200", "UNVERIFIABLE", "DISTINGUISHED"):
        group = by.get(verdict, [])
        if not group:
            continue
        print("%s (%d reference(s), %d host shape(s))"
              % (verdict, len(group), len({(r["host"], r["ext"]) for r in group})))
        for r in sorted(group, key=lambda r: (r["host"], r["ext"], r["url"])):
            print("   %-9s %s" % (r["ext"] or "(none)", r["url"]))
            print("             %s; control %s" % (r["why"], r["control"]))
        print()

    n_ok = len(by.get("VERIFIABLE", []))
    n_un = len(by.get("UNVERIFIABLE", []))
    n_dead = len(by.get("DEAD-BUT-200", []))
    n_dist = len(by.get("DISTINGUISHED", []))
    asked = n_ok + n_un + n_dead + n_dist
    print("%d of %d distinct URLs could be asked this question at all; %d are a bare host"
          % (asked, len(urls), len(rootless)))
    print("   with no sibling path to vary, which is out of scope rather than clean.")
    print("%d of the %d sit on a host shape that refuses a path it does not have."
          % (n_ok, asked))
    print("%d more served a DISTINGUISHED opening, so the server told the two paths apart."
          % n_dist)
    print("%d are UNVERIFIABLE even then (only a rendering browser can settle those); %d are DEAD-BUT-200."
          % (n_un, n_dead))
    assert asked + len(rootless) == len(urls), "rows lost between collection and report"
    print("\ncontrol: a 200 is not a page. surfaces.py and check-links both report every")
    print("URL above as alive, and for the rows named here that report is empty.")

    if args.json:
        json.dump({"head": head, "measured": now, "rows": rows}, open(args.json, "w"), indent=1)
        print("\nwrote %s" % args.json)
    return 0


def selftest():
    """No network, no clone. Every rule this instrument has, falsified."""
    ok = True

    def check(name, got, want):
        nonlocal ok
        if got != want:
            print("FAIL: %s\n      got  %r\n      want %r" % (name, got, want))
            ok = False
        else:
            print("ok: %s" % name)

    # The control keeps the extension, because that is what Sentry dispatches on.
    check("a .md citation controls with a .md path",
          control_url("https://docs.sentry.io/pricing.md"),
          "https://docs.sentry.io/" + NONSENSE + ".md")
    check("an extensionless page controls with an extensionless path",
          control_url("https://docs.sentry.io/organization/billing"),
          "https://docs.sentry.io/organization/" + NONSENSE)
    check("a trailing slash controls inside the directory",
          control_url("https://a.example/docs/"),
          "https://a.example/docs/" + NONSENSE)
    check("a bare host has no shape to vary", control_url("https://a.example"), None)
    check("a bare host with a slash has none either", control_url("https://a.example/"), None)
    # A version number is not a file extension; treating it as one would send
    # the control to a path the server never dispatches the same way.
    check("a dotted version is not an extension",
          control_url("https://api.example/v1.2/users"),
          "https://api.example/v1.2/" + NONSENSE)
    check("a long dotted tail is not an extension",
          control_url("https://a.example/some.page.with.a.verylongtail"),
          "https://a.example/" + NONSENSE)

    # The shape key is (host, extension), so one host can be both ways at once.
    check("the same host with two extensions is two shapes",
          (shape("https://docs.sentry.io/pricing.md"), shape("https://docs.sentry.io/pricing")),
          (("docs.sentry.io", ".md"), ("docs.sentry.io", "")))

    # The redirect signal (wake fe51ed60). Six rules, because the thing that
    # makes it evidence is that the citation did NOT move.
    FINAL_URL.clear()
    check("a URL never fetched has not moved", moved("https://a.example/p"), False)
    FINAL_URL["https://a.example/p"] = "https://a.example/p"
    check("a URL that ended where it asked has not moved", moved("https://a.example/p"), False)
    FINAL_URL["https://a.example/q"] = "https://www.a.example/q/"
    check("a trailing slash, a www and a scheme are not a move",
          moved("https://a.example/q"), False)
    FINAL_URL["https://a.example/x"] = "https://a.example/blog"
    check("a different path is a move", moved("https://a.example/x"), True)
    # The two fixtures share an opening longer than OPENING and differ after
    # it, which is the only state that REACHES the redirect rule: a shorter
    # difference is already DISTINGUISHED and an identical body is already
    # DEAD-BUT-200. This is the amplitude.com case in miniature -- one blog
    # shell, two pages.
    shell = "amplitude blog " * 40
    check("a control redirected away while the citation stayed is DISTINGUISHED",
          classify(200, shell + "the article", "https://a.example/p",
                   200, shell + "the index", "https://a.example/x")[0],
          "DISTINGUISHED")
    FINAL_URL["https://a.example/p"] = "https://a.example/blog"
    check("both redirected to one place is NOT evidence, it is the soft-404 case",
          classify(200, shell + "the article", "https://a.example/p",
                   200, shell + "the index", "https://a.example/x")[0],
          "UNVERIFIABLE")
    FINAL_URL.clear()

    # A host that 404s properly is verifiable whatever its body says.
    check("a proper 404 control makes the shape verifiable",
          classify(200, "real page", "https://a.example/p", 404, "Not Found", "https://a.example/x")[0],
          "VERIFIABLE")
    check("a control that cannot be fetched is not held against the citation",
          classify(200, "real page", "https://a.example/p", None, "", "https://a.example/x")[0],
          "VERIFIABLE")

    # The Sentry case: the control answers 200, and the cited page is a real
    # page that says something else. Before the DISTINGUISHED verdict this row
    # read UNVERIFIABLE, which was true and useless: I had to settle it by hand
    # anyway, and the hand method was exactly this comparison.
    check("a real citation on a soft-404 host is DISTINGUISHED, not merely unverifiable",
          classify(200, "# Pricing\nSeer is billed at $40", "https://docs.sentry.io/pricing.md",
                   200, "# Page Not Found", "https://docs.sentry.io/x.md")[0],
          "DISTINGUISHED")
    # UNVERIFIABLE now means genuinely undecidable, and the commonest way in is
    # a per-path SPA shell: same template, same opening, differing only in a
    # preload hint far down the file. This is the rippling row, which only a
    # rendering browser could settle (its control renders h1 "Page Not Found").
    shell = "<!doctype html><html data-has-hydrated=false><title></title>" + "x" * 400
    check("a per-path SPA shell with an identical opening stays UNVERIFIABLE",
          classify(200, shell + "real", "https://d.example/real",
                   200, shell + "ctrl", "https://d.example/x")[0],
          "UNVERIFIABLE")
    check("and it tells the reader what would settle it",
          "rendering browser" in classify(200, shell + "real", "https://d.example/real",
                                          200, shell + "ctrl", "https://d.example/x")[1],
          True)
    # Found by reading this instrument's own output rather than by thinking.
    check("a differing status is the strongest distinguishing there is",
          classify(405, '{"error":"method not allowed"}', "https://mcp.a.example/mcp/v1",
                   200, "<!doctype html>" + "x" * 500, "https://mcp.a.example/mcp/x"),
          ("DISTINGUISHED", "citation answers 405 where a path the host does not have answers 200"))
    check("but a citation that could not be fetched is evidence of NOTHING, not of existence",
          classify(None, "", "https://a.example/p", 200, "shell", "https://a.example/x")[0],
          "UNVERIFIABLE")

    check("a citation that answers nothing is not distinguished by an empty body",
          classify(200, "", "https://a.example/p.md",
                   200, "# Page Not Found", "https://a.example/x.md")[0],
          "UNVERIFIABLE")

    # The finding: a citation serving the same thing a nonsense path serves.
    # The refinement that turned 10 unactionable rows into 1 (wake 520aee0c).
    check("a different kind of body for this path is the server distinguishing it",
          classify(200, "# Vercel\n> Vercel is a cloud platform", "https://vercel.com/llms.txt",
                   200, "<!doctype html><html data-dpl-id=...", "https://vercel.com/x.txt")[0],
          "DISTINGUISHED")
    check("an API's own refusal is a distinguished answer, not a page",
          classify(200, '{"error":"method not allowed"}', "https://mcp.a.example/mcp/v1",
                   200, "<!doctype html><html lang=en>...", "https://mcp.a.example/mcp/x")[0],
          "DISTINGUISHED")
    # The rippling case: per-path SPA shells are byte-identical at the HTTP
    # layer, so the instrument must STOP rather than guess. A browser settled
    # that one; this rule is what hands it over.

    check("a citation identical to the control is DEAD-BUT-200",
          classify(200, "# Page Not Found", "https://a.example/gone.md",
                   200, "# Page Not Found", "https://a.example/x.md")[0],
          "DEAD-BUT-200")
    check("an empty body on both sides is not evidence of anything",
          classify(200, "", "https://a.example/gone.md", 200, "", "https://a.example/x.md")[0],
          "UNVERIFIABLE")

    # The template that echoes its own path is still one template. Without
    # this rule the instrument misses the case it was built for.
    check("a soft-404 template echoing the path is still the same template",
          classify(200, "No page at /gone.md here", "https://a.example/gone.md",
                   200, "No page at /%s.md here" % NONSENSE, "https://a.example/%s.md" % NONSENSE)[0],
          "DEAD-BUT-200")
    check("two genuinely different pages are not the same template",
          echoes_its_own_path("Pricing: Seer costs $40", "https://a.example/pricing.md",
                              "Page not found", "https://a.example/x.md"),
          False)

    # Whitespace and case are not evidence of difference.
    check("normalize collapses what a reader would not see",
          normalize("  # Page   Not\n\n Found "), "# page not found")

    print("\n%s" % ("all rules hold." if ok else "SELFTEST FAILED."))
    return ok


if __name__ == "__main__":
    sys.exit(main())