#!/usr/bin/env python3
"""trims.py: what did this version DROP?

The merge seat's question when a profile is rewritten to fit the 16,000-byte
cap: a diff that adds one paragraph and shortens fourteen others is a diff
whose additions everyone reads and whose removals nobody does. Three wakes
running (#181, #185, #187) the interesting finding was in the removals, and
the one that mattered was a VENDOR's sentence, not the researcher's narration:
#181 dropped "We recommend that you start with the supervised mode before
using the autonomous mode." while its body said it had dropped four asides.

So this asks one question per changed file: which PROTECTED ATOMS were in the
base and are not in the head? An atom is something no trim is allowed to lose
silently -- a quoted span (the vendor's words), a link, a date, a money figure,
a backticked identifier. Everything else is narration, and narration is the
author's to cut.

It is a reading aid, not a gate: a lost atom can be perfectly correct (a
retired quotation, a page that died). It names them so the reading happens.

WHY NOT THE OBVIOUS ONE-LINER. The quick version of this (set of `"..."`
matches in base minus head) RE-PAIRS EVERY QUOTE AFTER THE FIRST CHANGE: drop
one quoted span and the opening quote of the next pairs with the closing quote
of the one after it, so the tool reports losses and gains that are artifacts of
its own scanner. I ran exactly that one-liner at wake 22dc036f and it printed
six "gains" that were not in the diff at all. Quotes here are paired WITHIN A
LINE only, which is what a paragraph-per-line markdown corpus actually holds.

  python3 tools/trims.py --clone /tmp/reg --base origin/main --head pr187
  python3 tools/trims.py --clone /tmp/reg --ref <merge sha>      # one commit
  python3 tools/trims.py --selftest                              # no network, no clone
"""

import argparse
import os
import re
import subprocess
import sys

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from corpus import mask_code  # noqa: E402  (one shared masker, never a copy)

CAP = 16000  # the registry's profile byte cap

# --- atoms ------------------------------------------------------------------
# Every pattern is line-local on purpose (see the docstring).

MIN_QUOTE = 4  # shorter than this is punctuation, not a quotation worth tracking
CURLY_QUOTE = re.compile(r'“([^”\n]{%d,})”' % MIN_QUOTE)


def straight_quotes(line):
    """Quoted spans of one line, PAIRED BY POSITION and only then filtered.

    The obvious regex -- `"([^"\\n]{4,})"` -- de-synchronizes on a SHORT
    quotation. Found on shopify-ucp v7 (#275): the tier row the head quotes as
    `"Yes"` is three characters, so the pattern failed at its opening mark,
    re-used its CLOSING mark as an opening one, and mis-paired every span after
    it in that paragraph. The vendor's own exclusion sentence (`complete_checkout
    and order tools aren't available at this tier`) was byte-identical in base
    and head and came out LOST, with a fake atom ('under Checkout tools, while
    ...') reported as a gain beside it.

    This is the THIRD generation of the same parity bug: the one-liner that
    re-paired across the whole document (this file's own docstring), the same
    across-paragraphs bug in profiles.py, and now a filter that silently
    consumed a delimiter. The lesson each time: pair the delimiters FIRST,
    filter the contents AFTER -- never let a content test decide what a
    delimiter is.

    An unterminated final segment (an odd number of marks on the line) is
    dropped rather than run to the end of the paragraph.
    """
    # A `"` inside backticks is a character being named, not a delimiter:
    # github and hubspot both write ``Every `"`-delimited span`` and each
    # leaves its file with an odd number of marks. Mask first, then pair on
    # the masked copy, then take the slice from the ORIGINAL (same offsets,
    # because mask_code preserves length).
    masked = mask_code(line)
    segs = masked.split('"')
    at = 0
    for i, seg in enumerate(segs):
        if i % 2 == 1 and i < len(segs) - 1 and len(seg) >= MIN_QUOTE:
            yield line[at:at + len(seg)]
        at += len(seg) + 1


LINK = re.compile(r'\[([^\]\n]+)\]\((https?://[^)\s]+)\)')
BARE_URL = re.compile(r'(?<![(\]])\bhttps?://[^\s)\]}>"\'`,;]+')
DATE = re.compile(r'\b(?:19|20)\d\d-\d\d-\d\d\b')
MONEY = re.compile(r'\$\s?\d[\d,]*(?:\.\d+)?')
# `%` is not a word character, so a trailing \b after it never matches before a
# space: "12% of list" found nothing until the boundary was moved inside the
# spelled-out alternative. Caught by rule "a dropped percent is reported".
PERCENT = re.compile(r'\b\d+(?:\.\d+)?\s?(?:%|percent\b)')
CODE = re.compile(r'`([^`\n]{2,})`')

KINDS = ("quote", "link", "url", "date", "money", "percent", "code")


def atoms(text):
    """kind -> {atom: count}. Counts matter: losing one of three copies is a
    change too, and a set would hide it."""
    out = {k: {} for k in KINDS}

    def add(kind, value):
        value = re.sub(r"\s+", " ", value).strip()
        if value:
            out[kind][value] = out[kind].get(value, 0) + 1

    for line in text.split("\n"):
        for span in straight_quotes(line):
            add("quote", span)
        for m in CURLY_QUOTE.finditer(line):
            add("quote", m.group(1))
        for m in LINK.finditer(line):
            add("link", m.group(2))
        # A markdown link's URL is not also a bare URL: blank the links first.
        bare_line = LINK.sub(lambda m: "[" + m.group(1) + "]()", line)
        for m in BARE_URL.finditer(bare_line):
            add("url", m.group(0).rstrip(".)"))
        for m in DATE.finditer(line):
            add("date", m.group(0))
        for m in MONEY.finditer(line):
            add("money", m.group(0))
        for m in PERCENT.finditer(line):
            add("percent", m.group(0))
        for m in CODE.finditer(line):
            add("code", m.group(1))
    return out


def widened(atom, head_atoms):
    """Did this lost QUOTE survive inside a LONGER quotation in the head?

    A profile at the byte cap that re-attributes a span routinely quotes MORE
    of the vendor's sentence than before: shopify-ucp v7 (#275) replaced the
    unattributed `always remains the merchant of record` with the whole
    sentence `The merchant always remains the merchant of record`, and
    `aren't available at this tier` with the row it came from. Both printed
    LOST, both were standing verbatim in the head, and a true loss delivered
    beside two artifacts is a true loss I will average away -- the same
    failure the `moved` and `thinned` classifications already fixed.

    Two deliberate narrownesses, each a selftest rule:

    1. QUOTES ONLY. For code, urls and links, containment means a DIFFERENT
       thing: `get_order` inside `get_orders` is another identifier and
       `.../api` inside `.../api/mcp` is another page. Widening is normal
       only for a quoted span, which is also the kind this report tells the
       reader to read first, so a false positive there costs the most.
    2. INSIDE A HEAD *ATOM*, never merely inside the head TEXT. A span the
       author kept but stopped quoting has lost its attribution: it is now
       the author's sentence, which is exactly the change worth reading.

    Returns the containing head atom, or None.
    """
    for cand in head_atoms:
        if len(cand) <= len(atom):
            continue
        i = cand.find(atom)
        while i != -1:
            before_ok = i == 0 or not (cand[i - 1].isalnum() or cand[i - 1] == "_")
            j = i + len(atom)
            after_ok = j == len(cand) or not (cand[j].isalnum() or cand[j] == "_")
            if before_ok and after_ok:
                return cand
            i = cand.find(atom, i + 1)
    return None


def delta(before, after):
    """kind -> (lost, gained, thinned), each a list of (atom, count).

    LOST means the head holds the atom ZERO times. THINNED means the head
    holds fewer copies but still holds one, which is not a loss: the
    vendor's sentence still stands in the file. Until wake 27836fa9 this
    function counted a 2->1 drop as a loss, and on the statsig v6 rewrite
    (f0a9c34) that made it print three LOST quotations of which two were
    plainly still in the head, one of them the vendor's own "This installs
    the Statsig MCP Server with OAuth." A duplicate falls whenever a
    profile at the byte cap folds its own correction history into the
    paragraph that already carried the quotation, which is a normal and
    good edit. An instrument that cries loss there is the same failure as
    the one the 'moved' classification fixed: a true positive delivered
    inside false ones is one I will average away.
    """
    result = {}
    for kind in KINDS:
        b, a = before[kind], after[kind]
        lost = [(k, b[k]) for k in b if a.get(k, 0) == 0]
        thinned = [(k, b[k] - a[k]) for k in b if 0 < a.get(k, 0) < b[k]]
        gained = [(k, a[k] - b.get(k, 0)) for k in a if a[k] > b.get(k, 0)]
        result[kind] = (sorted(lost), sorted(gained), sorted(thinned))
    return result


# --- git --------------------------------------------------------------------

def git(clone, *args):
    r = subprocess.run(["git", "-C", clone, *args], capture_output=True, text=True)
    if r.returncode != 0:
        raise SystemExit("git " + " ".join(args) + ": " + r.stderr.strip())
    return r.stdout


def show(clone, ref, path):
    r = subprocess.run(["git", "-C", clone, "show", f"{ref}:{path}"],
                       capture_output=True, text=True)
    return r.stdout if r.returncode == 0 else None


def changed_files(clone, base, head):
    out = git(clone, "diff", "--name-only", f"{base}...{head}")
    return [p for p in out.split("\n") if p.strip()]


# --- report -----------------------------------------------------------------

def sibling_text(clone, head, files, exclude):
    """Everything the head says in the OTHER files this commit touched, flattened.

    An atom that leaves profile.md and lands in a probe record was MOVED, not
    lost, and the first calibration of this tool reported six such moves as
    losses on the notion entry alone (the Bearer challenge's `invalid_token`,
    `error_description` and the protected-resource URL moved out of the profile
    into the evidence JSON filed in the same commit). A tool that cries wolf on
    a restructuring is one whose real findings get skimmed."""
    out = []
    for p in files:
        if p == exclude:
            continue
        t = show(clone, head, p)
        if t:
            out.append(t)
    return re.sub(r"\s+", " ", "\n".join(out))


def report(clone, base, head, only_md=True, quiet_gains=False):
    all_changed = changed_files(clone, base, head)
    files = [f for f in all_changed
             if f.endswith(".md") or (not only_md and f.endswith(".json"))]
    if not files:
        print("no markdown files changed between these two trees")
        return 0
    total_lost = {k: 0 for k in KINDS}
    total_moved = 0
    total_thinned = 0
    total_widened = 0
    for path in files:
        old, new = show(clone, base, path), show(clone, head, path)
        if old is None:
            print(f"\n=== {path}: new file ({len(new or '')} bytes), nothing can be lost")
            continue
        if new is None:
            print(f"\n=== {path}: DELETED. every atom in it is lost; read the reason on the thread")
            continue
        d = delta(atoms(old), atoms(new))
        head_quotes = list(atoms(new)["quote"])
        elsewhere = sibling_text(clone, head, all_changed, path)
        nlost = sum(1 for k in KINDS for a, _ in d[k][0]
                    if a not in elsewhere
                    and not (k == "quote" and widened(a, head_quotes)))
        headroom = CAP - len(new)
        print(f"\n=== {path}: {len(old)} -> {len(new)} bytes "
              f"({len(new) - len(old):+d}, {headroom} under the {CAP} cap)")
        if nlost == 0:
            print("    no protected atom lost: every quotation, link, date, figure "
                  "and identifier in the base survives here or in a file this "
                  "commit moved it to")
        for kind in KINDS:
            lost, gained, thinned = d[kind]
            for atom, n in thinned:
                total_thinned += 1
                print(f"  thin  {kind:<7} (-{n}, still in the file) {atom[:120]}")
            for atom, n in lost:
                times = "" if n == 1 else f" (x{n})"
                if atom in elsewhere:
                    total_moved += 1
                    print(f"  moved {kind:<7}{times} {atom[:120]}"
                          f"   -> another file of this commit")
                    continue
                wider = widened(atom, head_quotes) if kind == "quote" else None
                if wider:
                    total_widened += 1
                    print(f"  wider {kind:<7}{times} {atom[:110]}")
                    print(f"        still quoted, inside: \"{wider[:110]}\"")
                    continue
                total_lost[kind] += n
                print(f"  LOST  {kind:<7}{times} {atom[:150]}")
            if not quiet_gains:
                for atom, n in gained:
                    times = "" if n == 1 else f" (x{n})"
                    print(f"  new   {kind:<7}{times} {atom[:150]}")
    print()
    total = sum(total_lost.values())
    if total_moved:
        print(f"{total_moved} atom(s) left the file and are in another file this "
              f"commit changed: moved, not lost.")
    if total_thinned:
        print(f"{total_thinned} atom(s) lost a duplicate and still stand in the "
              f"file: thinned, not lost.")
    if total_widened:
        print(f"{total_widened} quotation(s) are quoted inside a LONGER span in "
              f"the head: widened, not lost (usually a re-attribution).")
    if total == 0:
        print("0 protected atoms lost across every changed markdown file.")
    else:
        parts = ", ".join(f"{total_lost[k]} {k}" for k in KINDS if total_lost[k])
        print(f"{total} protected atoms lost: {parts}.")
        print("A LOST QUOTE IS THE ONE TO READ FIRST: it is the vendor's sentence, "
              "not the author's. Ask the thread whether the removal is deliberate.")
    return total


# --- selftest ---------------------------------------------------------------

def selftest():
    """Falsify the rules with no clone and no network."""
    checks = []

    def ok(name, cond):
        checks.append((name, bool(cond)))

    base = (
        'The vendor says "We recommend that you start with the supervised mode '
        'before using the autonomous mode." on the page.\n'
        'See [the page](https://example.com/docs) and https://example.com/bare, '
        'read 2026-09-27, priced $619 at 12% of list, field `humanBilling`.\n'
    )

    # 1. nothing changed -> nothing lost
    d = delta(atoms(base), atoms(base))
    ok("identical trees lose nothing", all(not d[k][0] and not d[k][1] for k in KINDS))

    # 2. a dropped vendor sentence is named
    head = base.replace('The vendor says "We recommend that you start with the '
                        'supervised mode before using the autonomous mode." on the page.\n', '')
    d = delta(atoms(base), atoms(head))
    ok("a dropped quotation is LOST/quote",
       any("supervised mode" in a for a, _ in d["quote"][0]))

    # 3. THE BUG THIS TOOL EXISTS TO NOT HAVE: dropping one quoted span must not
    #    re-pair the quotes after it into phantom losses and gains.
    two = 'He said "alpha bravo" here.\nShe said "charlie delta" there.\nThey said "echo foxtrot" last.\n'
    one = 'She said "charlie delta" there.\nThey said "echo foxtrot" last.\n'
    d = delta(atoms(two), atoms(one))
    ok("dropping one quote reports exactly one loss", len(d["quote"][0]) == 1)
    ok("dropping one quote reports no phantom gain", len(d["quote"][1]) == 0)

    # 4. narration edits around a quotation are invisible
    reworded = base.replace("The vendor says", "Per the vendor,").replace("on the page.", "there.")
    d = delta(atoms(base), atoms(reworded))
    ok("rewording narration loses no atom", not any(d[k][0] for k in KINDS))

    # 5. each atom kind is detected and its loss reported
    for kind, gone, needle in [
        ("link", "[the page](https://example.com/docs)", "example.com/docs"),
        ("url", "https://example.com/bare", "example.com/bare"),
        ("date", "2026-09-27", "2026-09-27"),
        ("money", "$619", "619"),
        ("percent", "12%", "12"),
        ("code", "`humanBilling`", "humanBilling"),
    ]:
        d = delta(atoms(base), atoms(base.replace(gone, "")))
        ok(f"a dropped {kind} is reported",
           any(needle in a for a, _ in d[kind][0]))

    # 6. a markdown link's URL is not double-counted as a bare URL
    a = atoms("[x](https://example.com/docs)\n")
    ok("a link url is not also a bare url", a["url"] == {} and len(a["link"]) == 1)

    # 7. counts, not sets, but a surviving copy is not a loss. Until wake
    # 27836fa9 this rule asserted the opposite and pinned the bug: a 3->2 drop
    # was reported LOST, and on statsig v6 (f0a9c34) that printed the vendor's
    # "This installs the Statsig MCP Server with OAuth." as lost while it was
    # still in the head. Thinning is reported, quietly, under its own word.
    three = 'a "same span" b\nc "same span" d\ne "same span" f\n'
    twice = 'a "same span" b\nc "same span" d\n'
    d = delta(atoms(three), atoms(twice))
    ok("losing one copy of a repeated quote is not a loss", not d["quote"][0])
    ok("losing one copy of a repeated quote is reported as thinned",
       d["quote"][2] == [("same span", 1)])

    # 7b. the last copy going is still a loss, however many there were
    d = delta(atoms(three), atoms('a b\nc d\n'))
    ok("losing the last copy of a repeated quote is a loss",
       d["quote"][0] == [("same span", 3)] and not d["quote"][2])

    # 7bis. A SHORT QUOTATION MUST NOT SHIFT THE PAIRING OF THE ONES AFTER IT.
    # Verbatim from shopify-ucp v7 (#275): the tier row quoted as "Yes" is three
    # characters, under the old regex it consumed a delimiter, and the vendor's
    # exclusion sentence after it -- byte-identical in base and head -- was
    # reported LOST with an invented atom beside it as a gain.
    shifted = ('the row reads "Yes" there, and the sentence says '
               '"complete_checkout and order tools aren\'t available at this tier".\n')
    qs = list(atoms(shifted)["quote"])
    ok("a short quotation does not shift the pairing after it",
       "complete_checkout and order tools aren't available at this tier" in qs)
    ok("a quotation under the minimum length is not itself an atom",
       "Yes" not in qs and len(qs) == 1)
    d = delta(atoms(shifted), atoms(shifted))
    ok("the mis-pairing case loses and gains nothing against itself",
       not any(d[k][0] or d[k][1] for k in KINDS))

    # 7c. WIDENING, every case below lifted from the shopify-ucp v7 run (#275,
    # base ae1170b, head a97f156), which printed six LOST of which two were
    # standing verbatim in the head inside a longer quotation.
    narrow = 'The docs say the merchant "always remains the merchant of record".\n'
    wide = ('The carts page carries "The merchant always remains the merchant '
            'of record".\n')
    d = delta(atoms(narrow), atoms(wide))
    ok("a re-attributed span is still LOST to delta (widening is report-level)",
       d["quote"][0] == [("always remains the merchant of record", 1)])
    ok("a span quoted inside a longer head quotation is widened",
       widened("always remains the merchant of record",
               list(atoms(wide)["quote"])) ==
       "The merchant always remains the merchant of record")
    # the trap: kept but no longer QUOTED is a lost attribution, not a widening
    unquoted = "The merchant always remains the merchant of record.\n"
    ok("a span the head keeps but stops quoting is not widened",
       widened("always remains the merchant of record",
               list(atoms(unquoted)["quote"])) is None)
    # the other trap: containment across a word boundary is a different claim
    ok("widening does not match across a word boundary",
       widened("the merchant", list(atoms('x "the merchants of record" y\n')
                                    ["quote"])) is None)
    ok("an identical span is not its own widening",
       widened("always remains the merchant of record",
               ["always remains the merchant of record"]) is None)

    # 8. whitespace normalisation: a quote re-wrapped across the same line is the same atom
    wrapped = base.replace("start with the supervised", "start with  the   supervised")
    d = delta(atoms(base), atoms(wrapped))
    ok("re-spaced quotation is not a loss", not d["quote"][0])

    # 9. a quotation that moves to another paragraph is not a loss
    moved = "\n".join(reversed(base.strip().split("\n"))) + "\n"
    d = delta(atoms(base), atoms(moved))
    ok("a moved quotation is not a loss", not any(d[k][0] for k in KINDS))

    # 10. curly quotes are read like straight ones
    curly = 'The vendor says “How do I?” questions here.\n'
    ok("curly quotes are atoms", "How do I?" in atoms(curly)["quote"])

    # 11. an unbalanced quote does not swallow the rest of the file
    odd = 'He said "alpha bravo" and a lone " here.\nShe said "charlie delta".\n'
    ok("an odd quote does not swallow the next line",
       "charlie delta" in atoms(odd)["quote"])

    # 12. a trailing period outside the quotation is not part of the atom
    ok("quote atoms exclude surrounding punctuation",
       all(not a.endswith('"') for a in atoms(base)["quote"]))

    # 13/14. MOVED, not lost: the classification `atom in elsewhere` has to
    # survive the sibling file reflowing the sentence across lines, and must
    # not fire on a sibling that merely mentions a neighbouring phrase.
    atom = list(atoms('a "We recommend the supervised mode first." b')["quote"])[0]
    reflowed = re.sub(r"\s+", " ", '{"finding": "We recommend the\n'
                                   '   supervised mode first."}')
    ok("an atom reflowed in a sibling file counts as moved", atom in reflowed)
    ok("a sibling that only mentions the topic does not count as moved",
       atom not in re.sub(r"\s+", " ", '{"finding": "supervised mode"}'))

    width = max(len(n) for n, _ in checks)
    for name, good in checks:
        print(f"  {'ok  ' if good else 'FAIL'} {name:<{width}}")
    bad = [n for n, g in checks if not g]
    print(f"\n{len(checks) - len(bad)} of {len(checks)} rules hold")
    return 1 if bad else 0


def main():
    p = argparse.ArgumentParser()
    p.add_argument("--clone")
    p.add_argument("--base", default="origin/main")
    p.add_argument("--head")
    p.add_argument("--ref", help="one commit: compares it with its first parent")
    p.add_argument("--json-too", action="store_true",
                   help="also read changed .json files (their strings are atoms too)")
    p.add_argument("--no-gains", action="store_true")
    p.add_argument("--selftest", action="store_true")
    a = p.parse_args()

    if a.selftest:
        sys.exit(selftest())
    if not a.clone or not (a.head or a.ref):
        p.error("need --clone with --head (or --ref), or --selftest")
    base, head = (f"{a.ref}^", a.ref) if a.ref else (a.base, a.head)
    report(a.clone, base, head, only_md=not a.json_too, quiet_gains=a.no_gains)


if __name__ == "__main__":
    main()