{
  "qwen3.5-9b": {
    "records": 400,
    "counts": {
      "changed_wrong": 27,
      "gain": 17,
      "loss": 10,
      "preserved": 202,
      "same_wrong": 144
    },
    "changed_answers": 54,
    "net_additional_correct": 7,
    "statistics": {
      "baseline_accuracy": 0.53,
      "subject_accuracy": 0.5475,
      "effect_pp": 1.7500000000000002,
      "lower_pp": -0.8298925545805376,
      "upper_pp": 4.3217785714311985,
      "mcnemar_p": 0.24778856337070465
    },
    "empty_outputs": {
      "baseline": [],
      "subject": [
        "mmlu_pro_01020",
        "mmlu_pro_09998"
      ]
    },
    "non_letter_outputs": [
      {
        "record_id": "mmlu_pro_01020",
        "baseline": "E",
        "subject": "",
        "category": "changed_wrong"
      },
      {
        "record_id": "mmlu_pro_09998",
        "baseline": "B",
        "subject": "",
        "category": "changed_wrong"
      }
    ],
    "loss_ids": [
      "mmlu_pro_00111",
      "mmlu_pro_00346",
      "mmlu_pro_03140",
      "mmlu_pro_03663",
      "mmlu_pro_03721",
      "mmlu_pro_04628",
      "mmlu_pro_07552",
      "mmlu_pro_10700",
      "mmlu_pro_12056",
      "mmlu_pro_12199"
    ]
  },
  "qwen3.8-27b": {
    "records": 400,
    "counts": {
      "changed_wrong": 20,
      "gain": 13,
      "loss": 7,
      "preserved": 245,
      "same_wrong": 115
    },
    "changed_answers": 40,
    "net_additional_correct": 6,
    "statistics": {
      "baseline_accuracy": 0.63,
      "subject_accuracy": 0.645,
      "effect_pp": 1.5,
      "lower_pp": -0.7440266824329054,
      "upper_pp": 3.7463947748609603,
      "mcnemar_p": 0.26317596435546875
    },
    "empty_outputs": {
      "baseline": [],
      "subject": [
        "mmlu_pro_00087",
        "mmlu_pro_00564",
        "mmlu_pro_00807",
        "mmlu_pro_08378"
      ]
    },
    "non_letter_outputs": [
      {
        "record_id": "mmlu_pro_00087",
        "baseline": "A",
        "subject": "",
        "category": "loss"
      },
      {
        "record_id": "mmlu_pro_00564",
        "baseline": "B",
        "subject": "",
        "category": "changed_wrong"
      },
      {
        "record_id": "mmlu_pro_00807",
        "baseline": "D",
        "subject": "",
        "category": "changed_wrong"
      },
      {
        "record_id": "mmlu_pro_08378",
        "baseline": "B",
        "subject": "",
        "category": "changed_wrong"
      }
    ],
    "loss_ids": [
      "mmlu_pro_00087",
      "mmlu_pro_01020",
      "mmlu_pro_01449",
      "mmlu_pro_03861",
      "mmlu_pro_03992",
      "mmlu_pro_05383",
      "mmlu_pro_11770"
    ]
  },
  "ministral3-8b": {
    "records": 400,
    "counts": {
      "changed_wrong": 15,
      "gain": 6,
      "loss": 4,
      "preserved": 175,
      "same_wrong": 200
    },
    "changed_answers": 25,
    "net_additional_correct": 2,
    "statistics": {
      "baseline_accuracy": 0.4475,
      "subject_accuracy": 0.4525,
      "effect_pp": 0.5,
      "lower_pp": -1.119943249438997,
      "upper_pp": 2.118915687037841,
      "mcnemar_p": 0.75390625
    },
    "empty_outputs": {
      "baseline": [],
      "subject": []
    },
    "non_letter_outputs": [
      {
        "record_id": "mmlu_pro_11894",
        "baseline": "**",
        "subject": "B",
        "category": "changed_wrong"
      }
    ],
    "loss_ids": [
      "mmlu_pro_04748",
      "mmlu_pro_05543",
      "mmlu_pro_05992",
      "mmlu_pro_06633"
    ]
  },
  "cross_comparison": {
    "distinct_question_ids": 400,
    "loss_union": [
      "mmlu_pro_00087",
      "mmlu_pro_00111",
      "mmlu_pro_00346",
      "mmlu_pro_01020",
      "mmlu_pro_01449",
      "mmlu_pro_03140",
      "mmlu_pro_03663",
      "mmlu_pro_03721",
      "mmlu_pro_03861",
      "mmlu_pro_03992",
      "mmlu_pro_04628",
      "mmlu_pro_04748",
      "mmlu_pro_05383",
      "mmlu_pro_05543",
      "mmlu_pro_05992",
      "mmlu_pro_06633",
      "mmlu_pro_07552",
      "mmlu_pro_10700",
      "mmlu_pro_11770",
      "mmlu_pro_12056",
      "mmlu_pro_12199"
    ],
    "pairwise_loss_overlap": {
      "qwen3.5-9b / qwen3.8-27b": [],
      "qwen3.5-9b / ministral3-8b": [],
      "qwen3.8-27b / ministral3-8b": []
    },
    "identical_qwen_input_digests": true
  }
}