#!/usr/bin/env python3 """Reproduce the retained GGUF answer analysis. Python 3.12+, standard library. No model invocation. Default execution is offline. --fetch-sources explicitly downloads immutable public files and checks the projection against them. """ from __future__ import annotations import argparse from collections import Counter import csv import hashlib import io import itertools import json import math from pathlib import Path import sys from urllib.request import urlopen sys.dont_write_bytecode = True from paired_exact_match import paired_exact_match_statistics import review HERE = Path(__file__).resolve().parent MODELS = ["qwen3.5-9b", "qwen3.8-27b", "ministral3-8b"] LABELS = ["Qwen3.5 9B", "Qwen3.8 27B", "Ministral 3 8B"] FIELDS = ["model", "order", "record_id", "input_sha256", "expected", "baseline", "subject", "baseline_score", "subject_score", "category"] def require(condition, message): if not condition: raise ValueError(message) def digest(data): return hashlib.sha256(data).hexdigest() def encode(value): return (json.dumps(value, indent=2, ensure_ascii=False) + "\n").encode() def category(expected, baseline, subject): a, b = baseline == expected, subject == expected return ("preserved" if a and b else "loss" if a else "gain" if b else "changed_wrong" if baseline != subject else "same_wrong") def project(model, sources): """Require complete, ordered joins and original scores; never intersect/drop.""" schedule = sources["schedule"]["records"] baseline = sources["baseline"]["records"] subject = sources["subject"]["records"] paired = sources["paired"]["records"] require(len(schedule) == len(baseline) == len(subject) == len(paired) == 400, "Missing or extra source records") rows = [] seen = set() for index, (s, a, b, p) in enumerate(zip(schedule, baseline, subject, paired, strict=True)): rid = s["record_id"] require(rid not in seen, "Duplicate source record") seen.add(rid) require(all(x["record_id"] == rid and x["input_sha256"] == s["input_sha256"] for x in (a, b, p)), "Source order/identity mismatch") for observation in (a, b): require(observation["status"] == "ok" and observation["error_code"] is None, "Unsuccessful source observation") require(digest(observation["output_text"].encode()) == observation["output_sha256"], "Output digest mismatch") av, bv = a["output_text"], b["output_text"] expected = s["expected_output"] require(p["baseline"]["score"] == int(av == expected) and p["subject"]["score"] == int(bv == expected), "Original score differs from literal equality") rows.append(dict(zip(FIELDS, [model, str(index + 1), rid, s["input_sha256"], expected, av, bv, str(int(av == expected)), str(int(bv == expected)), category(expected, av, bv)], strict=True))) return rows def csv_bytes(rows): stream = io.StringIO(newline="") writer = csv.DictWriter(stream, fieldnames=FIELDS, lineterminator="\n") writer.writeheader() writer.writerows(rows) return stream.getvalue().encode() def verify_files(directory): manifest = json.loads((directory / "package-manifest.json").read_text()) for name, expected in manifest.items(): require(Path(name).name == name, "Unexpected package path") require(digest((directory / name).read_bytes()) == expected, "Package file differs: " + name) def calculate(data, reports): reader = csv.DictReader(io.StringIO(data.decode())) require(reader.fieldnames == FIELDS, "CSV schema differs") rows = list(reader) require(len(rows) == 1200 and {r["model"] for r in rows} == set(MODELS), "Expected three complete 400-record comparisons") result, loss_sets, schedules, outputs = {}, {}, [], {} for model in MODELS: records = [r for r in rows if r["model"] == model] require(len(records) == 400 and len({r["record_id"] for r in records}) == 400, "Missing/duplicate records") require([r["order"] for r in records] == list(map(str, range(1, 401))), "Record order differs") for r in records: require(r["expected"] in list("ABCDEFGHIJ"), "Unexpected answer key") require(len(r["input_sha256"]) == 64 and all(c in "0123456789abcdef" for c in r["input_sha256"]), "Invalid input digest") require(all(r[s + "_score"] == str(int(r[s] == r["expected"])) for s in ("baseline", "subject")), "Score differs from literal equality") require(r["category"] == category(r["expected"], r["baseline"], r["subject"]), "Category differs") schedules.append([(r["record_id"], r["expected"]) for r in records]) outputs[model] = records counts = Counter(r["category"] for r in records) stats = paired_exact_match_statistics([int(r["baseline_score"]) for r in records], [int(r["subject_score"]) for r in records]) report = reports[model] binary = report["paired_binary"] for key, value in [("both_pass", counts["preserved"]), ("both_fail", counts["same_wrong"] + counts["changed_wrong"]), ("baseline_pass_subject_fail", counts["loss"]), ("baseline_fail_subject_pass", counts["gain"]), ("discordant_pairs", counts["gain"] + counts["loss"])]: require(binary[key] == value, "Original report count differs: " + key) interval = stats.effect_size_confidence_interval observed = { "baseline_accuracy": stats.baseline_pass_count / 400, "subject_accuracy": stats.subject_pass_count / 400, "effect_pp": stats.effect_size_pp, "lower_pp": interval.lower_pp, "upper_pp": interval.upper_pp, "mcnemar_p": stats.mcnemar_exact_two_sided_p_value, } original = [report["baseline"]["mean_score"], report["subject"]["mean_score"], report["comparison"]["value"], report["uncertainty"]["lower"], report["uncertainty"]["upper"], binary["mcnemar_exact_two_sided_p_value"]] require(all(math.isclose(a, b, rel_tol=0, abs_tol=1e-12) for a, b in zip(observed.values(), original, strict=True)), "Original statistics differ") require(report["uncertainty"]["method"] == interval.method and report["record_count"] == 400, "Original method/count differs") passed = (400 >= report["sample_qualification"]["record_count"]["minimum"] and interval.upper_pp - interval.lower_pp <= report["sample_qualification"]["interval_width"]["maximum"] and interval.lower_pp >= report["comparison"]["minimum"] and min(observed["baseline_accuracy"], observed["subject_accuracy"]) >= report["side_accuracy"]["minimum"]) require(passed and report["verdict"] == "pass", "Original policy outcome differs") loss_sets[model] = {r["record_id"] for r in records if r["category"] == "loss"} result[model] = { "records": 400, "counts": dict(sorted(counts.items())), "changed_answers": sum(r["baseline"] != r["subject"] for r in records), "net_additional_correct": counts["gain"] - counts["loss"], "statistics": observed, "empty_outputs": {side: [r["record_id"] for r in records if r[side] == ""] for side in ("baseline", "subject")}, "non_letter_outputs": [{"record_id": r["record_id"], "baseline": r["baseline"], "subject": r["subject"], "category": r["category"]} for r in records if any(r[side] not in list("ABCDEFGHIJ") for side in ("baseline", "subject"))], "loss_ids": sorted(loss_sets[model]), } require(schedules[0] == schedules[1] == schedules[2], "Semantic IDs or answer keys differ across comparisons") result["cross_comparison"] = { "distinct_question_ids": 400, "loss_union": sorted(set.union(*loss_sets.values())), "pairwise_loss_overlap": {a + " / " + b: sorted(loss_sets[a] & loss_sets[b]) for a, b in itertools.combinations(MODELS, 2)}, "identical_qwen_input_digests": all(a["input_sha256"] == b["input_sha256"] for a, b in zip(outputs[MODELS[0]], outputs[MODELS[1]], strict=True)), } return result def chart(summary): parts = ['', 'Small net gains conceal different answer changes', 'Changed answers in three retained 400-question comparisons, split into gains, losses and changed answers that remain wrong. These are descriptive counts, not a model ranking.', '', '', 'Small net gains conceal different answer changes', 'Each row uses the same 400 question IDs. Bar length counts changed answer strings.'] for i, (text, color) in enumerate([("Wrong → correct", "#3048a5"), ("Correct → wrong", "#a02c24"), ("Changed, still wrong", "#566176")]): x = 48 + i * 340 parts += [f'', f'{text}'] for i, (model, label) in enumerate(zip(MODELS, LABELS, strict=True)): y = 202 + 100 * i entry = summary[model] parts += [f'{label}'] x = 260 for key, color in [("gain", "#3048a5"), ("loss", "#a02c24"), ("changed_wrong", "#566176")]: count = entry["counts"][key]; width = count * 11 parts += [f'', f'{count}'] x += width parts += [f'{entry["changed_answers"]} changed', f'Net +{entry["net_additional_correct"]} correct'] for tick in range(0, 61, 10): x = 260 + tick * 11 parts += [f'{tick}'] parts += ['Number of changed answers', 'All original accuracy intervals include zero. Fewer changed answers do not establish better quality.', '\n'] return '\n'.join(parts).encode() def verify_sources(directory, source_root=None, fetch=False): provenance = json.loads((directory / "sources.json").read_text()) payloads = {} for item in provenance["files"]: if fetch: with urlopen(item["url"], timeout=45) as response: raw = response.read() else: raw = (source_root / item["path"]).read_bytes() require(digest(raw) == item["sha256"], "Immutable source bytes differ: " + item["path"]) payloads[(item["model"], item["role"])] = json.loads(raw) rows = [] all_sources = {} for model in MODELS: sources = {role: value for (m, role), value in payloads.items() if m == model} all_sources[model] = sources for side in ("baseline", "subject"): binding = sources[side + "_manifest"]["runtime_provider"] for role, key in [(side + "_receipt", "receipt"), (side, "scoring_observation")]: original_hash = next(i["sha256"] for i in provenance["files"] if i["model"] == model and i["role"] == role) require(binding[key]["sha256"] == original_hash, "Runtime source binding differs") rows.extend(project(model, sources)) require(sources["report"] == json.loads((directory / (model + '-report.json')).read_text()), "Report snapshot differs") require(csv_bytes(rows) == (directory / "answers.csv").read_bytes(), "Projection differs from original public sources") options, contexts = review.project(all_sources) require(options == (directory / "options.csv").read_bytes(), "Offered option projection differs") require(encode(contexts) == (directory / "runtime-context.json").read_bytes(), "Runtime context projection differs") def main(): parser = argparse.ArgumentParser(description=__doc__) source = parser.add_mutually_exclusive_group() source.add_argument("--source-root", type=Path, help="Local checkout of the immutable public source revision") source.add_argument("--fetch-sources", action="store_true", help="Explicit network read: verify 27 immutable public JSON files") parser.add_argument("--output", type=Path, help="New directory for reproduced summary and chart") args = parser.parse_args() verify_files(HERE) reports = {m: json.loads((HERE / (m + '-report.json')).read_text()) for m in MODELS} summary = calculate((HERE / "answers.csv").read_bytes(), reports) require(encode(summary) == (HERE / "summary.json").read_bytes(), "Published summary differs") require(chart(summary) == (HERE / "answer-changes.svg").read_bytes(), "Published chart differs") extra = review.calculate((HERE / "answers.csv").read_bytes(), (HERE / "options.csv").read_bytes(), json.loads((HERE / "runtime-context.json").read_text())) require(encode(extra) == (HERE / "review-summary.json").read_bytes(), "Review summary differs") figures = review.charts(extra) for name, data in figures.items(): require(data == (HERE / name).read_bytes(), "Review chart differs: " + name) if args.source_root or args.fetch_sources: verify_sources(HERE, args.source_root, args.fetch_sources) print("Source projection matches all 27 immutable public files.") if args.output: args.output.mkdir(parents=True, exist_ok=False) (args.output / "summary.json").write_bytes(encode(summary)) (args.output / "answer-changes.svg").write_bytes(chart(summary)) (args.output / "review-summary.json").write_bytes(encode(extra)) for name, data in figures.items(): (args.output / name).write_bytes(data) print("Verified 1,200 rows / 400 shared question IDs; original counts, accuracies, intervals, McNemar statistics and policy passes; new answer categories, option validity, shared errors, loss overlap and four charts. No model runs. This is not signature verification.") if __name__ == "__main__": main()