#!/usr/bin/env python3 """ The AI Answer Evidence Index: figures written as a range versus figures written as a single value. An exploratory view added after publication, outside the preregistration (deviation log, entry 8). It is not part of deposit version 1.0.0. Input (read only): the deposited check outputs of doi:10.5281/zenodo.23238429 work/check/{google_aio,chatgpt,gemini}/main/.json registered engines addendum/figures/work/check/{perplexity,claude}/addendum/.json added engines and, for the labels of figures that were not found, the classifier outputs beside them work/classify//main/.json addendum/figures/work/classify//addendum/.json If MANIFEST.sha256 is present, every input file is checked against it. The rule. check_figures.py turns each extracted figure into its "components": one {unit, value} per number in the figure, with unit and multiplier carried across a range ("$50-100" gives $50 and $100). The check calls a figure found only when every component occurs on one cited page with its unit, all within 300 characters of each other. single value exactly 1 component range 2 or more components not read 0 components: the extractor returned the figure in a form that is not in its sentence (an instrument gap); it is neither a single value nor a range Two narrower readings of "range" are reported beside it: strict range every pair of adjacent parts is joined by the range separator of check_figures.py (RANGE_SEP: "to", "-", en dash, em dash, "and") kind range the extractor's own field "kind" is "range" The regular expressions below are copied from check_figures.py, unchanged, and the script stops if they do not give the same number of components as the deposited output. Outcomes, as on the page and in table 3 of the tables: found found_attached + found_elsewhere not found not_found (every cited page readable, none holds the figure) unknown no_source + access_failure + instrument_gap Shares are pooled over figures, with every figure of the group in the denominator. Views, each asserted against the published tables before anything is written: all figures table 3: figures and figures found, per engine all readable the figures of answers that cite at least one page and whose cited pages were all readable; it does not depend on the outcome (method note, section 9). Table 4: figures and figures found, pooled, per engine with a unit figures with a currency, percent or multiple sign on at least one part. Table S4: figures with a unit and figures found Also written: the classifier's labels of the figures not found, with the label "partial" split into its three cases (the page's figure carries another qualifier; one part fits exactly; one part fits only within rounding), and a 95% percentile bootstrap interval of single minus range, in points, resampling questions (10,000 resamples, the study's seed 1109872877), because the figures of one answer are not independent. Usage: python3 ranges_vs_single.py --deposit [--out ] Writes ranges-vs-single.json and ranges-vs-single.md in --out (default: this script's folder). Python 3.9 or later, standard library only. """ import argparse, collections, hashlib, json, pathlib, random, re, sys ENGINES = [ # (key, label, check folder, classify folder), in the order of the page ("google_aio", "Google AI Overviews", "work/check/google_aio/main", "work/classify/google_aio/main"), ("chatgpt", "ChatGPT", "work/check/chatgpt/main", "work/classify/chatgpt/main"), ("gemini", "Gemini", "work/check/gemini/main", "work/classify/gemini/main"), ("perplexity", "Perplexity", "addendum/figures/work/check/perplexity/addendum", "addendum/figures/work/classify/perplexity/addendum"), ("claude", "Claude (API)", "addendum/figures/work/check/claude/addendum", "addendum/figures/work/classify/claude/addendum"), ] REGISTERED = ["google_aio", "chatgpt", "gemini"] POOLS = [("registered_three", "Google AI Overviews, ChatGPT and Gemini", REGISTERED), ("all_five", "All five engines", [e[0] for e in ENGINES])] # (figures, found) as the published tables print them. TABLE_3 = {"google_aio": (640, 507), "chatgpt": (470, 267), "gemini": (661, 453), "perplexity": (755, 644), "claude": (939, 822)} TABLE_4_ALL_READABLE = {"google_aio": (302, 268), "chatgpt": (326, 217), "gemini": (489, 347), "perplexity": (63, 61), "claude": (430, 397)} TABLE_S4_WITH_UNIT = {"google_aio": (473, 355), "chatgpt": (365, 186), "gemini": (475, 312), "perplexity": (576, 467), "claude": (775, 678)} OUTCOME = {"found_attached": "found", "found_elsewhere": "found", "not_found": "not_found", "no_source": "unknown", "access_failure": "unknown", "instrument_gap": "unknown"} SEED, RESAMPLES = 1109872877, 10000 # ---------- copied from check_figures.py, unchanged ---------- FIG = re.compile(r""" (?:(?P
US\$|\$|€|£|\b(?:USD|EUR|GBP))\s?)?
    (?P\d{1,3}(?:,\d{3})+(?:\.\d+)?|\d+(?:\.\d+)?)
    (?:\s?(?PK\b|k\b|M\b|B\b|bn\b|thousand\b|million\b|billion\b))?
    (?:\s?(?P%|percent\b|per\s?cent\b|¢|cents?\b|x(?![\w\d])|×|USD\b|dollars?\b))?
""", re.X)
RANGE_SEP = re.compile(r"^\s*(?:to|-|–|—|and)\s*$")
# --------------------------------------------------------------

Fig = collections.namedtuple("Fig", "question group outcome strict kind_range all_readable with_unit label partial")


def joined_as_range(number):
    """(number of parts, True if every adjacent pair is joined by the check's range separator)."""
    ms = list(FIG.finditer(number))
    return len(ms), all(RANGE_SEP.match(number[a.end():b.start()]) for a, b in zip(ms, ms[1:]))


def partial_case(row):
    """Which of the three cases gave the label partial (classify_figures.label)."""
    if row.get("about") == "other qualifier":
        return "other_qualifier"
    return "one_part_exact" if "exact" in (row.get("fits") or []) else "one_part_rounded"


def sha256(p):
    return hashlib.sha256(p.read_bytes()).hexdigest()


def manifest(root):
    m = root / "MANIFEST.sha256"
    if not m.exists():
        return None
    return {name: h for h, _, name in (line.partition("  ") for line in m.read_text(encoding="utf-8").splitlines())}


def share(n, d):
    """n of d with the share, and no share for a cell of fewer than 5 units (method note, section 9)."""
    return {"n": n, "of": d, "share": round(n / d, 4) if d >= 5 else None}


def cell(x):
    if x["share"] is None:
        return f"{x['n']} of {x['of']}"
    return f"{x['n']} of {x['of']} ({x['share'] * 100:.0f}%)"


def found_of(figs):
    return share(sum(OUTCOME[f.outcome] == "found" for f in figs), len(figs))


def group(figs):
    c = collections.Counter(OUTCOME[f.outcome] for f in figs)
    nf = [f for f in figs if f.outcome == "not_found"]
    return {"figures": len(figs), "found": share(c["found"], len(figs)),
            "not_found": share(c["not_found"], len(figs)), "unknown": share(c["unknown"], len(figs)),
            "outcomes": dict(sorted(collections.Counter(f.outcome for f in figs).items())),
            "classifier_labels_of_not_found": dict(sorted(collections.Counter(f.label for f in nf).items())),
            "partial_cases": dict(sorted(collections.Counter(f.partial for f in nf if f.label == "partial").items()))}


def interval(figs):
    """95% percentile interval of single minus range (points, pooled shares), resampling questions."""
    per_q = collections.defaultdict(lambda: [0, 0, 0, 0])   # single found, single n, range found, range n
    for f in figs:
        if f.group in ("single", "range"):
            k = 0 if f.group == "single" else 2
            per_q[f.question][k] += OUTCOME[f.outcome] == "found"
            per_q[f.question][k + 1] += 1
    qs = sorted(per_q)
    rng, diffs = random.Random(SEED), []
    for _ in range(RESAMPLES):
        t = [0, 0, 0, 0]
        for _ in qs:
            row = per_q[qs[int(rng.random() * len(qs))]]
            t = [a + b for a, b in zip(t, row)]
        if t[1] and t[3]:
            diffs.append(100 * (t[0] / t[1] - t[2] / t[3]))
    diffs.sort()
    lo, hi = diffs[int(0.025 * len(diffs))], diffs[min(len(diffs) - 1, int(0.975 * len(diffs)))]
    return {"questions": len(qs), "resamples": RESAMPLES, "used": len(diffs), "seed": SEED,
            "low_points": round(lo, 1), "high_points": round(hi, 1)}


def tally(figs):
    single = [f for f in figs if f.group == "single"]
    rng_ = [f for f in figs if f.group == "range"]
    nf = [f for f in figs if f.outcome == "not_found"]
    return {"figures": len(figs), "all_figures_found": found_of(figs),
            "ranges_share_of_all_figures": share(len(rng_), len(figs)),
            "single": group(single), "range": group(rng_),
            "not_read": group([f for f in figs if f.group == "not_read"]),
            "not_read_kind_range": sum(f.kind_range for f in figs if f.group == "not_read"),
            "single_including_not_read": group([f for f in figs if f.group != "range"]),
            "range_strict": group([f for f in rng_ if f.strict]),
            "range_other_join": group([f for f in rng_ if not f.strict]),
            "range_by_kind": group([f for f in rng_ if f.kind_range]),
            "range_not_by_kind": group([f for f in rng_ if not f.kind_range]),
            "not_found_that_are_ranges": share(sum(f.group == "range" for f in nf), len(nf)),
            "all_readable": {"figures_found": found_of([f for f in figs if f.all_readable]),
                             "single_found": found_of([f for f in single if f.all_readable]),
                             "range_found": found_of([f for f in rng_ if f.all_readable])},
            "with_unit": {"figures_found": found_of([f for f in figs if f.with_unit]),
                          "single_found": found_of([f for f in single if f.with_unit]),
                          "range_found": found_of([f for f in rng_ if f.with_unit]),
                          "ranges_without_unit": len([f for f in rng_ if not f.with_unit])},
            "single_minus_range_interval": interval(figs)}


def main():
    ap = argparse.ArgumentParser(description="Figures written as a range versus single values.")
    ap.add_argument("--deposit", default=".", help="root of the deposit (doi:10.5281/zenodo.23238429)")
    ap.add_argument("--out", default=str(pathlib.Path(__file__).resolve().parent))
    a = ap.parse_args()
    root, outdir = pathlib.Path(a.deposit).resolve(), pathlib.Path(a.out).resolve()
    man = manifest(root)
    inputs, mismatched, figs = [], [], {}
    for key, _, chk, cls in ENGINES:
        figs[key] = []
        files = sorted((root / chk).glob("*.json"))
        if not files:
            sys.exit(f"no check outputs in {root / chk}")
        for f in files:
            cf = root / cls / f.name
            for p in (f, cf) if cf.exists() else (f,):
                rel = p.relative_to(root).as_posix()
                h = sha256(p)
                inputs.append((rel, h))
                if man is not None and man.get(rel) != h:
                    mismatched.append(rel)
            labels = {}
            if cf.exists():
                labels = {r["figure_index"]: r for r in json.loads(cf.read_text(encoding="utf-8"))["figures"]}
            for i, c in enumerate(json.loads(f.read_text(encoding="utf-8"))["claims"]):
                k = len(c["components"])
                grp = "not_read" if k == 0 else "single" if k == 1 else "range"
                strict = False
                if k:
                    n, strict = joined_as_range(c["number"])
                    if n != k:
                        sys.exit(f"{f.name} figure {i}: {n} parts here, {k} in the check output")
                label = partial = None
                if c["outcome"] == "not_found":
                    if i not in labels:
                        sys.exit(f"{f.name} figure {i}: not found and without a classifier row")
                    label = labels[i].get("label")
                    partial = partial_case(labels[i]) if label == "partial" else None
                all_readable = c["cited_pages"] > 0 and c["readable_pages"] == c["cited_pages"] \
                    and c["pages_without_address"] == 0
                with_unit = any(p["unit"] != "" for p in c["components"])
                figs[key].append(Fig(f.stem, grp, c["outcome"], bool(strict), c.get("kind") == "range",
                                     all_readable, with_unit, label, partial))
    if mismatched:
        sys.exit(f"{len(mismatched)} input files do not match MANIFEST.sha256, first: {mismatched[0]}")
    for name, table, test in (("table 3", TABLE_3, lambda f: True),
                              ("table 4", TABLE_4_ALL_READABLE, lambda f: f.all_readable),
                              ("table S4", TABLE_S4_WITH_UNIT, lambda f: f.with_unit)):
        for key, want in table.items():
            sel = [f for f in figs[key] if test(f)]
            got = (len(sel), sum(OUTCOME[f.outcome] == "found" for f in sel))
            if got != want:
                sys.exit(f"{key}: {got} figures and found, {name} prints {want}")

    res = {"title": "Figures written as a range and figures written as a single value",
           "status": "Exploratory view added after publication, outside the preregistration (deviation log, "
                     "entry 8). Not part of deposit version 1.0.0 of doi:10.5281/zenodo.23238429.",
           "engines": {}, "pooled": {}}
    for key, label, _, _ in ENGINES:
        res["engines"][key] = {"label": label, **tally(figs[key])}
    for key, label, members in POOLS:
        res["pooled"][key] = {"label": label, "engines": members, **tally([f for e in members for f in figs[e]])}
    res["rule"] = {
        "single": "exactly 1 component in the check output",
        "range": "2 or more components in the check output",
        "not_read": "0 components: instrument gap, the extractor's figure is not in its sentence",
        "range_strict": "a range whose adjacent parts are all joined by RANGE_SEP of check_figures.py",
        "range_by_kind": "a range whose extractor field kind is 'range'",
        "single_including_not_read": "every figure with fewer than 2 components (the convention of the "
                                     "reply of 16 October 2026)",
        "all_readable": "cited_pages > 0, readable_pages == cited_pages, pages_without_address == 0",
        "with_unit": "at least one component carries a unit sign",
        "partial_cases": "other_qualifier: the page's figure carries another qualifier; one_part_exact: at "
                         "least one part equals the page's figure; one_part_rounded: the parts that fit do so "
                         "only within rounding (method note, section 8)",
        "outcomes": OUTCOME,
        "denominator": "every figure of the group",
        "small_cells": "no share for a cell of fewer than 5 units",
        "interval": "95% percentile bootstrap of single minus range in points, pooled shares, resampling "
                    "questions with replacement",
        "asserted_against": {"table 3": TABLE_3, "table 4": TABLE_4_ALL_READABLE, "table S4": TABLE_S4_WITH_UNIT},
    }
    res["inputs"] = {"deposit": "doi:10.5281/zenodo.23238429", "files": len(inputs),
                     "sha256_of_sorted_list": hashlib.sha256("\n".join(f"{h}  {p}" for p, h in sorted(inputs))
                                                             .encode()).hexdigest(),
                     "checked_against_manifest": man is not None}
    outdir.mkdir(parents=True, exist_ok=True)
    (outdir / "ranges-vs-single.json").write_text(json.dumps(res, indent=1, ensure_ascii=False) + "\n",
                                                  encoding="utf-8")
    (outdir / "ranges-vs-single.md").write_text(report(res), encoding="utf-8")
    p3 = res["pooled"]["registered_three"]
    print("three engines, single:", cell(p3["single"]["found"]), "| range:", cell(p3["range"]["found"]),
          "| not found that are ranges:", cell(p3["not_found_that_are_ranges"]))


def report(res):
    cols = list(res["engines"].values()) + list(res["pooled"].values())
    head = "| | " + " | ".join(v["label"] for v in cols) + " |\n|---" + "|---" * len(cols) + "|\n"

    def line(name, f):
        return f"| {name} | " + " | ".join(f(v) for v in cols) + " |\n"

    def labels(g, keys):
        return lambda v: str(sum(n for k, n in v[g]["classifier_labels_of_not_found"].items() if k in keys))

    def iv(v):
        x = v["single_minus_range_interval"]
        return f"{x['low_points']:+.0f} to {x['high_points']:+.0f}"

    t = ["# Figures written as a range, and figures written as a single value\n\n",
         "Written by `ranges_vs_single.py` from the check outputs of deposit version 1.0.0 "
         "(doi:10.5281/zenodo.23238429). An exploratory view added after publication, outside the "
         "preregistration (deviation log, entry 8); this file is not part of that deposit. A figure is a range "
         "when the check output gives it two or more components, and a single value when it gives one. Shares are "
         "pooled over figures, with every figure of the group in the denominator. No share is given for a cell of "
         "fewer than 5 units. The script stops unless it reproduces the totals of tables 3, 4 and S4.\n\n",
         "## 1. Share found, all figures\n\n", head,
         line("Figures", lambda v: str(v["figures"])),
         line("All figures: found (table 3)", lambda v: cell(v["all_figures_found"])),
         line("Ranges, share of all figures", lambda v: cell(v["ranges_share_of_all_figures"])),
         line("Single values: found", lambda v: cell(v["single"]["found"])),
         line("Ranges: found", lambda v: cell(v["range"]["found"])),
         line("Single minus range, 95% interval in points, questions resampled", iv),
         line("Single values: unknown", lambda v: cell(v["single"]["unknown"])),
         line("Ranges: unknown", lambda v: cell(v["range"]["unknown"])),
         line("Single values: not found", lambda v: cell(v["single"]["not_found"])),
         line("Ranges: not found", lambda v: cell(v["range"]["not_found"])),
         line("Figures not found that are ranges", lambda v: cell(v["not_found_that_are_ranges"])),
         "\n## 2. Answers whose cited pages were all readable\n\n"
         "The view that does not depend on the outcome (method note, section 9; table 4).\n\n", head,
         line("All figures: found (table 4)", lambda v: cell(v["all_readable"]["figures_found"])),
         line("Single values: found", lambda v: cell(v["all_readable"]["single_found"])),
         line("Ranges: found", lambda v: cell(v["all_readable"]["range_found"])),
         "\n## 3. Figures with a unit\n\n"
         "A figure without a unit sign is matched on the number alone (method note, section 7; table S4).\n\n", head,
         line("All figures with a unit: found (table S4)", lambda v: cell(v["with_unit"]["figures_found"])),
         line("Single values with a unit: found", lambda v: cell(v["with_unit"]["single_found"])),
         line("Ranges with a unit: found", lambda v: cell(v["with_unit"]["range_found"])),
         line("Ranges without a unit sign", lambda v: str(v["with_unit"]["ranges_without_unit"])),
         "\n## 4. Narrower readings of \"range\"\n\n", head,
         line("Joined by the check's range separator: found", lambda v: cell(v["range_strict"]["found"])),
         line("Joined otherwise: found", lambda v: cell(v["range_other_join"]["found"])),
         line("Extractor's kind is range: found", lambda v: cell(v["range_by_kind"]["found"])),
         line("Extractor's kind is not range: found", lambda v: cell(v["range_not_by_kind"]["found"])),
         line("Not read (0 components)", lambda v: str(v["not_read"]["figures"])),
         line("Not read, extractor's kind is range", lambda v: str(v["not_read_kind_range"])),
         line("Single values with the not-read figures counted in: found",
              lambda v: cell(v["single_including_not_read"]["found"])),
         "\n## 5. Classifier labels of the figures not found\n\n", head,
         line("Ranges: partial", labels("range", {"partial"})),
         line("of which the page's figure carries another qualifier",
              lambda v: str(v["range"]["partial_cases"].get("other_qualifier", 0))),
         line("of which one part fits exactly", lambda v: str(v["range"]["partial_cases"].get("one_part_exact", 0))),
         line("of which one part fits only within rounding",
              lambda v: str(v["range"]["partial_cases"].get("one_part_rounded", 0))),
         line("Ranges: absent", labels("range", {"absent"})),
         line("Ranges: present, rounded or derived", labels("range", {"present", "rounded", "derived"})),
         line("Single values: partial", labels("single", {"partial"})),
         line("Single values: absent", labels("single", {"absent"})),
         line("Single values: present, rounded or derived", labels("single", {"present", "rounded", "derived"})),
         "\nThe classifier's labels are model readings turned into labels by fixed rules (method note, section 8). "
         "Partial: some parts of the figure fit a figure on a cited page and others do not, or the page's figure "
         "carries another qualifier. A figure that is not found is not thereby false.\n\n",
         f"Inputs: {res['inputs']['files']} files, checked against the deposit's MANIFEST.sha256: "
         f"{'yes, all match' if res['inputs']['checked_against_manifest'] else 'no manifest present'}. "
         f"SHA-256 of the sorted list of their hashes: `{res['inputs']['sha256_of_sorted_list']}`.\n"]
    return "".join(t)


if __name__ == "__main__":
    main()