{ "scope": "Descriptive follow-up on 400 shared questions; six outputs per question are not independent trials.", "validity": { "qwen3.5-9b": { "invalid_ids": { "baseline": [], "subject": [ "mmlu_pro_01020", "mmlu_pro_09998" ] }, "normalization_changed_score_ids": { "baseline": [], "subject": [] }, "changed_groups": { "gain": { "count": 17, "offered_option": 17, "empty": 0, "other_invalid": 0 }, "loss": { "count": 10, "offered_option": 10, "empty": 0, "other_invalid": 0 }, "changed_wrong": { "count": 27, "offered_option": 25, "empty": 2, "other_invalid": 0 } } }, "qwen3.8-27b": { "invalid_ids": { "baseline": [], "subject": [ "mmlu_pro_00087", "mmlu_pro_00564", "mmlu_pro_00807", "mmlu_pro_08378" ] }, "normalization_changed_score_ids": { "baseline": [], "subject": [] }, "changed_groups": { "gain": { "count": 13, "offered_option": 13, "empty": 0, "other_invalid": 0 }, "loss": { "count": 7, "offered_option": 6, "empty": 1, "other_invalid": 0 }, "changed_wrong": { "count": 20, "offered_option": 17, "empty": 3, "other_invalid": 0 } } }, "ministral3-8b": { "invalid_ids": { "baseline": [ "mmlu_pro_11894" ], "subject": [] }, "normalization_changed_score_ids": { "baseline": [], "subject": [] }, "changed_groups": { "gain": { "count": 6, "offered_option": 6, "empty": 0, "other_invalid": 0 }, "loss": { "count": 4, "offered_option": 4, "empty": 0, "other_invalid": 0 }, "changed_wrong": { "count": 15, "offered_option": 15, "empty": 0, "other_invalid": 0 } } } }, "shared_baseline_correct": { "count": 137, "ids": [ "mmlu_pro_00154", "mmlu_pro_00202", "mmlu_pro_00436", "mmlu_pro_00685", "mmlu_pro_00831", "mmlu_pro_00947", "mmlu_pro_01092", "mmlu_pro_01761", "mmlu_pro_02110", "mmlu_pro_02188", "mmlu_pro_02203", "mmlu_pro_02225", "mmlu_pro_02228", "mmlu_pro_02238", "mmlu_pro_02356", "mmlu_pro_02483", "mmlu_pro_02601", "mmlu_pro_02667", "mmlu_pro_02670", "mmlu_pro_02706", "mmlu_pro_02851", "mmlu_pro_02854", "mmlu_pro_02872", "mmlu_pro_02905", "mmlu_pro_02961", "mmlu_pro_02965", "mmlu_pro_02993", "mmlu_pro_03009", "mmlu_pro_03030", "mmlu_pro_03057", "mmlu_pro_03073", "mmlu_pro_03124", "mmlu_pro_03214", "mmlu_pro_03221", "mmlu_pro_03279", "mmlu_pro_03310", "mmlu_pro_03373", "mmlu_pro_03415", "mmlu_pro_03434", "mmlu_pro_03472", "mmlu_pro_03493", "mmlu_pro_03509", "mmlu_pro_03513", "mmlu_pro_03690", "mmlu_pro_03780", "mmlu_pro_03797", "mmlu_pro_03861", "mmlu_pro_04267", "mmlu_pro_04628", "mmlu_pro_04653", "mmlu_pro_04715", "mmlu_pro_04739", "mmlu_pro_04760", "mmlu_pro_04778", "mmlu_pro_04828", "mmlu_pro_04836", "mmlu_pro_04837", "mmlu_pro_04861", "mmlu_pro_04864", "mmlu_pro_04881", "mmlu_pro_04898", "mmlu_pro_04914", "mmlu_pro_04942", "mmlu_pro_04951", "mmlu_pro_04962", "mmlu_pro_04993", "mmlu_pro_05449", "mmlu_pro_05628", "mmlu_pro_05675", "mmlu_pro_05752", "mmlu_pro_05782", "mmlu_pro_05785", "mmlu_pro_05927", "mmlu_pro_05929", "mmlu_pro_05974", "mmlu_pro_05992", "mmlu_pro_06001", "mmlu_pro_06179", "mmlu_pro_06229", "mmlu_pro_06266", "mmlu_pro_06325", "mmlu_pro_06340", "mmlu_pro_06455", "mmlu_pro_06481", "mmlu_pro_06615", "mmlu_pro_06625", "mmlu_pro_06633", "mmlu_pro_06678", "mmlu_pro_06702", "mmlu_pro_06738", "mmlu_pro_06767", "mmlu_pro_06890", "mmlu_pro_06891", "mmlu_pro_06990", "mmlu_pro_07081", "mmlu_pro_07145", "mmlu_pro_07147", "mmlu_pro_07276", "mmlu_pro_07336", "mmlu_pro_07369", "mmlu_pro_07465", "mmlu_pro_07575", "mmlu_pro_07595", "mmlu_pro_07597", "mmlu_pro_08114", "mmlu_pro_08332", "mmlu_pro_08446", "mmlu_pro_08981", "mmlu_pro_09433", "mmlu_pro_09727", "mmlu_pro_10264", "mmlu_pro_10380", "mmlu_pro_10413", "mmlu_pro_10414", "mmlu_pro_10442", "mmlu_pro_10501", "mmlu_pro_10508", "mmlu_pro_10517", "mmlu_pro_10530", "mmlu_pro_10531", "mmlu_pro_10586", "mmlu_pro_10595", "mmlu_pro_10602", "mmlu_pro_10620", "mmlu_pro_10627", "mmlu_pro_10652", "mmlu_pro_10700", "mmlu_pro_10737", "mmlu_pro_10745", "mmlu_pro_10876", "mmlu_pro_10881", "mmlu_pro_10904", "mmlu_pro_10927", "mmlu_pro_11041", "mmlu_pro_11514", "mmlu_pro_12084", "mmlu_pro_12236" ], "loss_ids": { "qwen3.5-9b": [ "mmlu_pro_04628", "mmlu_pro_10700" ], "qwen3.8-27b": [ "mmlu_pro_03861" ], "ministral3-8b": [ "mmlu_pro_05992", "mmlu_pro_06633" ] }, "affected_ids": [ "mmlu_pro_03861", "mmlu_pro_04628", "mmlu_pro_05992", "mmlu_pro_06633", "mmlu_pro_10700" ], "all_preserved": 132 }, "shared_errors": { "all_six_wrong_ids": [ "mmlu_pro_00166", "mmlu_pro_00250", "mmlu_pro_00292", "mmlu_pro_00306", "mmlu_pro_00564", "mmlu_pro_00663", "mmlu_pro_00700", "mmlu_pro_00807", "mmlu_pro_00934", "mmlu_pro_00941", "mmlu_pro_01368", "mmlu_pro_01423", "mmlu_pro_01660", "mmlu_pro_01717", "mmlu_pro_01739", "mmlu_pro_01851", "mmlu_pro_01940", "mmlu_pro_02018", "mmlu_pro_02256", "mmlu_pro_02626", "mmlu_pro_02820", "mmlu_pro_03597", "mmlu_pro_03752", "mmlu_pro_03875", "mmlu_pro_03998", "mmlu_pro_04684", "mmlu_pro_04941", "mmlu_pro_04994", "mmlu_pro_05017", "mmlu_pro_05238", "mmlu_pro_05266", "mmlu_pro_05268", "mmlu_pro_05507", "mmlu_pro_05694", "mmlu_pro_05706", "mmlu_pro_05879", "mmlu_pro_05910", "mmlu_pro_06022", "mmlu_pro_06485", "mmlu_pro_06598", "mmlu_pro_06619", "mmlu_pro_06781", "mmlu_pro_07296", "mmlu_pro_07427", "mmlu_pro_07568", "mmlu_pro_07632", "mmlu_pro_07742", "mmlu_pro_07756", "mmlu_pro_07939", "mmlu_pro_08037", "mmlu_pro_08073", "mmlu_pro_08378", "mmlu_pro_08489", "mmlu_pro_08836", "mmlu_pro_08847", "mmlu_pro_08868", "mmlu_pro_08938", "mmlu_pro_09251", "mmlu_pro_09332", "mmlu_pro_09338", "mmlu_pro_09461", "mmlu_pro_09624", "mmlu_pro_09664", "mmlu_pro_09709", "mmlu_pro_09738", "mmlu_pro_09862", "mmlu_pro_09908", "mmlu_pro_10037", "mmlu_pro_10217", "mmlu_pro_10223", "mmlu_pro_10231", "mmlu_pro_10637", "mmlu_pro_10761", "mmlu_pro_10791", "mmlu_pro_10809", "mmlu_pro_10856", "mmlu_pro_10901", "mmlu_pro_10912", "mmlu_pro_10923", "mmlu_pro_10949", "mmlu_pro_11195", "mmlu_pro_11229", "mmlu_pro_11268", "mmlu_pro_11410", "mmlu_pro_11539", "mmlu_pro_11547", "mmlu_pro_11671", "mmlu_pro_11694", "mmlu_pro_11841", "mmlu_pro_11886", "mmlu_pro_11916", "mmlu_pro_12197" ], "same_wrong_option_ids": [ "mmlu_pro_00292", "mmlu_pro_01423", "mmlu_pro_01851", "mmlu_pro_02018", "mmlu_pro_02256", "mmlu_pro_03875", "mmlu_pro_04684", "mmlu_pro_04994", "mmlu_pro_05017", "mmlu_pro_05507", "mmlu_pro_05694", "mmlu_pro_05706", "mmlu_pro_06022", "mmlu_pro_06781", "mmlu_pro_07296", "mmlu_pro_07427", "mmlu_pro_07568", "mmlu_pro_07742", "mmlu_pro_07756", "mmlu_pro_08037", "mmlu_pro_09664", "mmlu_pro_10637", "mmlu_pro_10761", "mmlu_pro_10791", "mmlu_pro_10856", "mmlu_pro_10901", "mmlu_pro_10912", "mmlu_pro_11229", "mmlu_pro_11268", "mmlu_pro_11886" ], "all_six_correct_ids": [ "mmlu_pro_00154", "mmlu_pro_00202", "mmlu_pro_00436", "mmlu_pro_00685", "mmlu_pro_00831", "mmlu_pro_00947", "mmlu_pro_01092", "mmlu_pro_01761", "mmlu_pro_02110", "mmlu_pro_02188", "mmlu_pro_02203", "mmlu_pro_02225", "mmlu_pro_02228", "mmlu_pro_02238", "mmlu_pro_02356", "mmlu_pro_02483", "mmlu_pro_02601", "mmlu_pro_02667", "mmlu_pro_02670", "mmlu_pro_02706", "mmlu_pro_02851", "mmlu_pro_02854", "mmlu_pro_02872", "mmlu_pro_02905", "mmlu_pro_02961", "mmlu_pro_02965", "mmlu_pro_02993", "mmlu_pro_03009", "mmlu_pro_03030", "mmlu_pro_03057", "mmlu_pro_03073", "mmlu_pro_03124", "mmlu_pro_03214", "mmlu_pro_03221", "mmlu_pro_03279", "mmlu_pro_03310", "mmlu_pro_03373", "mmlu_pro_03415", "mmlu_pro_03434", "mmlu_pro_03472", "mmlu_pro_03493", "mmlu_pro_03509", "mmlu_pro_03513", "mmlu_pro_03690", "mmlu_pro_03780", "mmlu_pro_03797", "mmlu_pro_04267", "mmlu_pro_04653", "mmlu_pro_04715", "mmlu_pro_04739", "mmlu_pro_04760", "mmlu_pro_04778", "mmlu_pro_04828", "mmlu_pro_04836", "mmlu_pro_04837", "mmlu_pro_04861", "mmlu_pro_04864", "mmlu_pro_04881", "mmlu_pro_04898", "mmlu_pro_04914", "mmlu_pro_04942", "mmlu_pro_04951", "mmlu_pro_04962", "mmlu_pro_04993", "mmlu_pro_05449", "mmlu_pro_05628", "mmlu_pro_05675", "mmlu_pro_05752", "mmlu_pro_05782", "mmlu_pro_05785", "mmlu_pro_05927", "mmlu_pro_05929", "mmlu_pro_05974", "mmlu_pro_06001", "mmlu_pro_06179", "mmlu_pro_06229", "mmlu_pro_06266", "mmlu_pro_06325", "mmlu_pro_06340", "mmlu_pro_06455", "mmlu_pro_06481", "mmlu_pro_06615", "mmlu_pro_06625", "mmlu_pro_06678", "mmlu_pro_06702", "mmlu_pro_06738", "mmlu_pro_06767", "mmlu_pro_06890", "mmlu_pro_06891", "mmlu_pro_06990", "mmlu_pro_07081", "mmlu_pro_07145", "mmlu_pro_07147", "mmlu_pro_07276", "mmlu_pro_07336", "mmlu_pro_07369", "mmlu_pro_07465", "mmlu_pro_07575", "mmlu_pro_07595", "mmlu_pro_07597", "mmlu_pro_08114", "mmlu_pro_08332", "mmlu_pro_08446", "mmlu_pro_08981", "mmlu_pro_09433", "mmlu_pro_09727", "mmlu_pro_10264", "mmlu_pro_10380", "mmlu_pro_10413", "mmlu_pro_10414", "mmlu_pro_10442", "mmlu_pro_10501", "mmlu_pro_10508", "mmlu_pro_10517", "mmlu_pro_10530", "mmlu_pro_10531", "mmlu_pro_10586", "mmlu_pro_10595", "mmlu_pro_10602", "mmlu_pro_10620", "mmlu_pro_10627", "mmlu_pro_10652", "mmlu_pro_10737", "mmlu_pro_10745", "mmlu_pro_10876", "mmlu_pro_10881", "mmlu_pro_10904", "mmlu_pro_10927", "mmlu_pro_11041", "mmlu_pro_11514", "mmlu_pro_12084", "mmlu_pro_12236" ], "mixed_correctness_ids": [ "mmlu_pro_00075", "mmlu_pro_00087", "mmlu_pro_00111", "mmlu_pro_00257", "mmlu_pro_00284", "mmlu_pro_00294", "mmlu_pro_00323", "mmlu_pro_00346", "mmlu_pro_00372", "mmlu_pro_00375", "mmlu_pro_00536", "mmlu_pro_00569", "mmlu_pro_00575", "mmlu_pro_00614", "mmlu_pro_00650", "mmlu_pro_00695", "mmlu_pro_01004", "mmlu_pro_01020", "mmlu_pro_01134", "mmlu_pro_01179", "mmlu_pro_01244", "mmlu_pro_01284", "mmlu_pro_01337", "mmlu_pro_01387", "mmlu_pro_01413", "mmlu_pro_01419", "mmlu_pro_01449", "mmlu_pro_01529", "mmlu_pro_01578", "mmlu_pro_01866", "mmlu_pro_01894", "mmlu_pro_01974", "mmlu_pro_02033", "mmlu_pro_02086", "mmlu_pro_02090", "mmlu_pro_02091", "mmlu_pro_02147", "mmlu_pro_02236", "mmlu_pro_02321", "mmlu_pro_02459", "mmlu_pro_02519", "mmlu_pro_02528", "mmlu_pro_02635", "mmlu_pro_02645", "mmlu_pro_02728", "mmlu_pro_02838", "mmlu_pro_02986", "mmlu_pro_03074", "mmlu_pro_03106", "mmlu_pro_03140", "mmlu_pro_03582", "mmlu_pro_03602", "mmlu_pro_03663", "mmlu_pro_03721", "mmlu_pro_03766", "mmlu_pro_03819", "mmlu_pro_03861", "mmlu_pro_03871", "mmlu_pro_03900", "mmlu_pro_03908", "mmlu_pro_03931", "mmlu_pro_03992", "mmlu_pro_04011", "mmlu_pro_04090", "mmlu_pro_04129", "mmlu_pro_04168", "mmlu_pro_04206", "mmlu_pro_04234", "mmlu_pro_04379", "mmlu_pro_04628", "mmlu_pro_04708", "mmlu_pro_04748", "mmlu_pro_04831", "mmlu_pro_04891", "mmlu_pro_04929", "mmlu_pro_04943", "mmlu_pro_04965", "mmlu_pro_04988", "mmlu_pro_05044", "mmlu_pro_05141", "mmlu_pro_05276", "mmlu_pro_05307", "mmlu_pro_05383", "mmlu_pro_05471", "mmlu_pro_05474", "mmlu_pro_05527", "mmlu_pro_05537", "mmlu_pro_05543", "mmlu_pro_05776", "mmlu_pro_05992", "mmlu_pro_06064", "mmlu_pro_06198", "mmlu_pro_06271", "mmlu_pro_06297", "mmlu_pro_06457", "mmlu_pro_06533", "mmlu_pro_06536", "mmlu_pro_06633", "mmlu_pro_06652", "mmlu_pro_06710", "mmlu_pro_06893", "mmlu_pro_06948", "mmlu_pro_06962", "mmlu_pro_06993", "mmlu_pro_07168", "mmlu_pro_07272", "mmlu_pro_07527", "mmlu_pro_07552", "mmlu_pro_07565", "mmlu_pro_07608", "mmlu_pro_07639", "mmlu_pro_07657", "mmlu_pro_07779", "mmlu_pro_07903", "mmlu_pro_08078", "mmlu_pro_08118", "mmlu_pro_08163", "mmlu_pro_08299", "mmlu_pro_08449", "mmlu_pro_08459", "mmlu_pro_08554", "mmlu_pro_08601", "mmlu_pro_08679", "mmlu_pro_08798", "mmlu_pro_08988", "mmlu_pro_09138", "mmlu_pro_09149", "mmlu_pro_09319", "mmlu_pro_09440", "mmlu_pro_09563", "mmlu_pro_09592", "mmlu_pro_09810", "mmlu_pro_09998", "mmlu_pro_10035", "mmlu_pro_10125", "mmlu_pro_10243", "mmlu_pro_10360", "mmlu_pro_10411", "mmlu_pro_10460", "mmlu_pro_10475", "mmlu_pro_10500", "mmlu_pro_10521", "mmlu_pro_10550", "mmlu_pro_10609", "mmlu_pro_10700", "mmlu_pro_10771", "mmlu_pro_10776", "mmlu_pro_10812", "mmlu_pro_10822", "mmlu_pro_10861", "mmlu_pro_10875", "mmlu_pro_10897", "mmlu_pro_10922", "mmlu_pro_11095", "mmlu_pro_11128", "mmlu_pro_11135", "mmlu_pro_11178", "mmlu_pro_11230", "mmlu_pro_11234", "mmlu_pro_11344", "mmlu_pro_11363", "mmlu_pro_11472", "mmlu_pro_11515", "mmlu_pro_11533", "mmlu_pro_11549", "mmlu_pro_11642", "mmlu_pro_11716", "mmlu_pro_11731", "mmlu_pro_11770", "mmlu_pro_11894", "mmlu_pro_11922", "mmlu_pro_12055", "mmlu_pro_12056", "mmlu_pro_12149", "mmlu_pro_12199", "mmlu_pro_12225" ] }, "opposite_direction_questions": [ { "record_id": "mmlu_pro_04748", "expected": "F", "models": { "qwen3.5-9b": { "baseline": "I", "subject": "F", "category": "gain" }, "qwen3.8-27b": { "baseline": "F", "subject": "F", "category": "preserved" }, "ministral3-8b": { "baseline": "F", "subject": "G", "category": "loss" } } }, { "record_id": "mmlu_pro_11770", "expected": "D", "models": { "qwen3.5-9b": { "baseline": "A", "subject": "D", "category": "gain" }, "qwen3.8-27b": { "baseline": "D", "subject": "A", "category": "loss" }, "ministral3-8b": { "baseline": "D", "subject": "D", "category": "preserved" } } }, { "record_id": "mmlu_pro_12199", "expected": "D", "models": { "qwen3.5-9b": { "baseline": "D", "subject": "B", "category": "loss" }, "qwen3.8-27b": { "baseline": "B", "subject": "D", "category": "gain" }, "ministral3-8b": { "baseline": "B", "subject": "B", "category": "same_wrong" } } } ] }