{ "version": 1, "status": "published", "generated_at": "2026-09-25T21:22:01Z", "eval_set": { "version": "pilot-2026-09-25", "run": "2026-09-25", "questions": { "math": 190, "query": 190 }, "models_tested": 2 }, "methodology": { "version": "1.0", "url": "/docs/cloud/benchmarks/" }, "license": "CC BY 4.0", "citation": "Zabio, Model benchmarks, zabio.ai/benchmarks, 2026-09-25", "min_questions": 25, "task_types": [ { "id": "math", "name": { "en": "Math", "es": "Matemáticas" }, "graded_by": "machine", "bands": [ "easy", "medium", "hard", "frontier" ] }, { "id": "query", "name": { "en": "SQL and data queries", "es": "SQL y consultas de datos" }, "graded_by": "machine", "bands": [ "easy", "medium", "hard", "frontier" ] } ], "bands": [ { "id": "easy", "difficulty": [ 1, 2 ] }, { "id": "medium", "difficulty": [ 3 ] }, { "id": "hard", "difficulty": [ 4 ] }, { "id": "frontier", "difficulty": [ 5 ] } ], "settings": [ { "id": "off", "name": { "en": "Thinking off", "es": "Razonamiento desactivado" } }, { "id": "low", "name": { "en": "Thinking low", "es": "Razonamiento bajo" } }, { "id": "high", "name": { "en": "Thinking high", "es": "Razonamiento alto" } } ], "lines": [ { "task": "math", "band": "easy", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 0.27 }, "balanced": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 0.27 }, "cheap": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 0.27 } }, { "task": "math", "band": "medium", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 0.62 }, "balanced": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 0.62 }, "cheap": { "model": "deepseek-flash", "setting": "off", "cost_per_1k_usd": 0.59 } }, { "task": "math", "band": "hard", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 2.26 }, "balanced": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 2.26 }, "cheap": { "model": "deepseek-flash", "setting": "off", "cost_per_1k_usd": 1.66 } }, { "task": "math", "band": "frontier", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "high", "cost_per_1k_usd": 9.18 }, "balanced": { "model": "deepseek-flash", "setting": "high", "cost_per_1k_usd": 9.18 }, "cheap": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 6.38 } }, { "task": "query", "band": "easy", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "off", "cost_per_1k_usd": 0.24 }, "balanced": { "model": "deepseek-flash", "setting": "off", "cost_per_1k_usd": 0.24 }, "cheap": { "model": "deepseek-flash", "setting": "off", "cost_per_1k_usd": 0.24 } }, { "task": "query", "band": "medium", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "high", "cost_per_1k_usd": 1.87 }, "balanced": { "model": "deepseek-flash", "setting": "high", "cost_per_1k_usd": 1.87 }, "cheap": { "model": "deepseek-flash", "setting": "off", "cost_per_1k_usd": 0.45 } }, { "task": "query", "band": "hard", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 2.1 }, "balanced": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 2.1 }, "cheap": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 2.1 } }, { "task": "query", "band": "frontier", "among": [ "deepseek-flash", "deepseek-v4-pro" ], "best": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 4.29 }, "balanced": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 4.29 }, "cheap": { "model": "deepseek-flash", "setting": "low", "cost_per_1k_usd": 4.29 } } ], "models": [ { "id": "deepseek-flash", "provider": "deepseek", "path": "/models/deepseek/deepseek-flash/", "name": "DeepSeek (China)", "tested": "2026-09-25", "settings_tested": [ "off", "low", "high" ], "zabio_default_setting": "off", "results": [ { "task": "math", "band": "easy", "setting": "high", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 0.32, "right_per_usd": 3153.8, "answer_tokens": 40, "reasoning_tokens_est": 202, "latency_s_p50": 1.7, "cut_off_pct": 0.0 }, { "task": "math", "band": "easy", "setting": "low", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [ "balanced", "best", "cheap" ], "cost_per_1k_usd": 0.27, "right_per_usd": 3641.7, "answer_tokens": 38, "reasoning_tokens_est": 169, "latency_s_p50": 1.6, "cut_off_pct": 0.0 }, { "task": "math", "band": "easy", "setting": "off", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 0.36, "right_per_usd": 2793.7, "answer_tokens": 282, "reasoning_tokens_est": 0, "latency_s_p50": 2.1, "cut_off_pct": 0.0 }, { "task": "math", "band": "medium", "setting": "high", "right_pct": 96.4, "range_pct": [ 89.6, 98.8 ], "questions": 55, "group": "near", "lines": [], "cost_per_1k_usd": 1.04, "right_per_usd": 924.3, "answer_tokens": 69, "reasoning_tokens_est": 776, "latency_s_p50": 2.0, "cut_off_pct": 0.0 }, { "task": "math", "band": "medium", "setting": "low", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [ "balanced", "best" ], "cost_per_1k_usd": 0.62, "right_per_usd": 1590.4, "answer_tokens": 63, "reasoning_tokens_est": 428, "latency_s_p50": 1.9, "cut_off_pct": 0.0 }, { "task": "math", "band": "medium", "setting": "off", "right_pct": 94.5, "range_pct": [ 87.1, 97.8 ], "questions": 55, "group": "near", "lines": [ "cheap" ], "cost_per_1k_usd": 0.59, "right_per_usd": 1605.8, "answer_tokens": 473, "reasoning_tokens_est": 0, "latency_s_p50": 2.3, "cut_off_pct": 0.0 }, { "task": "math", "band": "hard", "setting": "off", "right_pct": 96.3, "range_pct": [ 89.4, 98.8 ], "questions": 54, "group": "near", "lines": [ "cheap" ], "cost_per_1k_usd": 1.66, "right_per_usd": 578.5, "answer_tokens": 1362, "reasoning_tokens_est": 0, "latency_s_p50": 4.8, "cut_off_pct": 0.0 }, { "task": "math", "band": "hard", "setting": "low", "right_pct": 100.0, "range_pct": [ 95.2, 100.0 ], "questions": 54, "group": "top", "lines": [ "balanced", "best" ], "cost_per_1k_usd": 2.26, "right_per_usd": 441.9, "answer_tokens": 139, "reasoning_tokens_est": 1715, "latency_s_p50": 5.7, "cut_off_pct": 0.0 }, { "task": "math", "band": "hard", "setting": "high", "right_pct": 100.0, "range_pct": [ 95.2, 100.0 ], "questions": 54, "group": "top", "lines": [], "cost_per_1k_usd": 3.0, "right_per_usd": 333.2, "answer_tokens": 145, "reasoning_tokens_est": 2324, "latency_s_p50": 5.9, "cut_off_pct": 0.0 }, { "task": "math", "band": "frontier", "setting": "low", "right_pct": 96.4, "range_pct": [ 89.6, 98.8 ], "questions": 55, "group": "near", "lines": [ "cheap" ], "cost_per_1k_usd": 6.38, "right_per_usd": 151.0, "answer_tokens": 223, "reasoning_tokens_est": 5059, "latency_s_p50": 17.2, "cut_off_pct": 1.4 }, { "task": "math", "band": "frontier", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [ "balanced", "best" ], "cost_per_1k_usd": 9.18, "right_per_usd": 107.0, "answer_tokens": 243, "reasoning_tokens_est": 7372, "latency_s_p50": 22.2, "cut_off_pct": 0.0 }, { "task": "math", "band": "frontier", "setting": "off", "right_pct": 73.3, "range_pct": [ 62.6, 81.9 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 3.16, "right_per_usd": 232.4, "answer_tokens": 2602, "reasoning_tokens_est": 0, "latency_s_p50": 10.8, "cut_off_pct": 0.0 }, { "task": "query", "band": "easy", "setting": "off", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [ "balanced", "best", "cheap" ], "cost_per_1k_usd": 0.24, "right_per_usd": 4124.2, "answer_tokens": 32, "reasoning_tokens_est": 0, "latency_s_p50": 1.1, "cut_off_pct": 0.0 }, { "task": "query", "band": "easy", "setting": "low", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 0.35, "right_per_usd": 2859.0, "answer_tokens": 28, "reasoning_tokens_est": 87, "latency_s_p50": 1.5, "cut_off_pct": 0.0 }, { "task": "query", "band": "easy", "setting": "high", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 0.42, "right_per_usd": 2358.8, "answer_tokens": 28, "reasoning_tokens_est": 149, "latency_s_p50": 1.6, "cut_off_pct": 0.0 }, { "task": "query", "band": "medium", "setting": "off", "right_pct": 96.4, "range_pct": [ 89.6, 98.8 ], "questions": 55, "group": "near", "lines": [ "cheap" ], "cost_per_1k_usd": 0.45, "right_per_usd": 2123.6, "answer_tokens": 184, "reasoning_tokens_est": 0, "latency_s_p50": 1.5, "cut_off_pct": 0.0 }, { "task": "query", "band": "medium", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [ "balanced", "best" ], "cost_per_1k_usd": 1.87, "right_per_usd": 524.3, "answer_tokens": 145, "reasoning_tokens_est": 1215, "latency_s_p50": 4.7, "cut_off_pct": 0.0 }, { "task": "query", "band": "medium", "setting": "low", "right_pct": 96.4, "range_pct": [ 89.6, 98.8 ], "questions": 55, "group": "near", "lines": [], "cost_per_1k_usd": 1.26, "right_per_usd": 762.5, "answer_tokens": 146, "reasoning_tokens_est": 707, "latency_s_p50": 4.0, "cut_off_pct": 0.0 }, { "task": "query", "band": "hard", "setting": "off", "right_pct": 93.9, "range_pct": [ 86.3, 97.4 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 0.58, "right_per_usd": 1608.3, "answer_tokens": 283, "reasoning_tokens_est": 0, "latency_s_p50": 1.7, "cut_off_pct": 0.0 }, { "task": "query", "band": "hard", "setting": "high", "right_pct": 100.0, "range_pct": [ 95.3, 100.0 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 3.48, "right_per_usd": 287.1, "answer_tokens": 228, "reasoning_tokens_est": 2464, "latency_s_p50": 8.5, "cut_off_pct": 0.0 }, { "task": "query", "band": "hard", "setting": "low", "right_pct": 100.0, "range_pct": [ 95.3, 100.0 ], "questions": 55, "group": "top", "lines": [ "balanced", "best", "cheap" ], "cost_per_1k_usd": 2.1, "right_per_usd": 475.9, "answer_tokens": 226, "reasoning_tokens_est": 1315, "latency_s_p50": 5.8, "cut_off_pct": 0.0 }, { "task": "query", "band": "frontier", "setting": "off", "right_pct": 88.5, "range_pct": [ 79.5, 93.8 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 0.75, "right_per_usd": 1178.6, "answer_tokens": 417, "reasoning_tokens_est": 0, "latency_s_p50": 2.2, "cut_off_pct": 0.0 }, { "task": "query", "band": "frontier", "setting": "low", "right_pct": 100.0, "range_pct": [ 95.3, 100.0 ], "questions": 55, "group": "top", "lines": [ "balanced", "best", "cheap" ], "cost_per_1k_usd": 4.29, "right_per_usd": 232.9, "answer_tokens": 314, "reasoning_tokens_est": 3049, "latency_s_p50": 12.4, "cut_off_pct": 0.0 }, { "task": "query", "band": "frontier", "setting": "high", "right_pct": 100.0, "range_pct": [ 95.3, 100.0 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 7.42, "right_per_usd": 134.8, "answer_tokens": 320, "reasoning_tokens_est": 5649, "latency_s_p50": 22.0, "cut_off_pct": 0.0 } ], "reliability": { "stability_flip_pct": 7.1 }, "history": [ { "run": "2026-09-25", "results": 24 } ] }, { "id": "deepseek-v4-pro", "provider": "deepseek", "path": "/models/deepseek/deepseek-v4-pro/", "name": "DeepSeek (China)", "tested": "2026-09-25", "settings_tested": [ "off", "low", "high" ], "zabio_default_setting": "off", "results": [ { "task": "math", "band": "easy", "setting": "low", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 0.81, "right_per_usd": 1237.9, "answer_tokens": 36, "reasoning_tokens_est": 146, "latency_s_p50": 2.1, "cut_off_pct": 0.0 }, { "task": "math", "band": "easy", "setting": "high", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 1.28, "right_per_usd": 778.5, "answer_tokens": 40, "reasoning_tokens_est": 236, "latency_s_p50": 2.7, "cut_off_pct": 0.0 }, { "task": "math", "band": "easy", "setting": "off", "right_pct": 96.0, "range_pct": [ 83.9, 99.1 ], "questions": 25, "group": "near", "lines": [], "cost_per_1k_usd": 1.04, "right_per_usd": 921.1, "answer_tokens": 242, "reasoning_tokens_est": 0, "latency_s_p50": 2.9, "cut_off_pct": 0.0 }, { "task": "math", "band": "medium", "setting": "low", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 1.57, "right_per_usd": 625.9, "answer_tokens": 59, "reasoning_tokens_est": 314, "latency_s_p50": 2.9, "cut_off_pct": 0.0 }, { "task": "math", "band": "medium", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 3.44, "right_per_usd": 285.2, "answer_tokens": 69, "reasoning_tokens_est": 751, "latency_s_p50": 4.0, "cut_off_pct": 0.0 }, { "task": "math", "band": "medium", "setting": "off", "right_pct": 94.5, "range_pct": [ 87.1, 97.8 ], "questions": 55, "group": "near", "lines": [], "cost_per_1k_usd": 1.6, "right_per_usd": 591.4, "answer_tokens": 381, "reasoning_tokens_est": 0, "latency_s_p50": 3.3, "cut_off_pct": 0.0 }, { "task": "math", "band": "hard", "setting": "off", "right_pct": 86.4, "range_pct": [ 77.0, 92.4 ], "questions": 54, "group": "below", "lines": [], "cost_per_1k_usd": 3.94, "right_per_usd": 219.5, "answer_tokens": 960, "reasoning_tokens_est": 0, "latency_s_p50": 6.8, "cut_off_pct": 0.0 }, { "task": "math", "band": "hard", "setting": "low", "right_pct": 98.1, "range_pct": [ 92.0, 99.6 ], "questions": 53, "group": "near", "lines": [], "cost_per_1k_usd": 5.98, "right_per_usd": 164.1, "answer_tokens": 112, "reasoning_tokens_est": 1364, "latency_s_p50": 10.0, "cut_off_pct": 0.0 }, { "task": "math", "band": "hard", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 15.63, "right_per_usd": 62.8, "answer_tokens": 141, "reasoning_tokens_est": 3747, "latency_s_p50": 17.7, "cut_off_pct": 0.0 }, { "task": "math", "band": "frontier", "setting": "off", "right_pct": 44.2, "range_pct": [ 33.8, 55.3 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 6.52, "right_per_usd": 67.8, "answer_tokens": 1611, "reasoning_tokens_est": 0, "latency_s_p50": 11.6, "cut_off_pct": 0.0 }, { "task": "math", "band": "frontier", "setting": "low", "right_pct": 89.1, "range_pct": [ 80.3, 94.3 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 25.1, "right_per_usd": 35.5, "answer_tokens": 166, "reasoning_tokens_est": 6136, "latency_s_p50": 32.6, "cut_off_pct": 6.8 }, { "task": "math", "band": "frontier", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 48.68, "right_per_usd": 20.2, "answer_tokens": 251, "reasoning_tokens_est": 11978, "latency_s_p50": 57.0, "cut_off_pct": 1.4 }, { "task": "query", "band": "easy", "setting": "off", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 1.03, "right_per_usd": 974.4, "answer_tokens": 33, "reasoning_tokens_est": 0, "latency_s_p50": 1.1, "cut_off_pct": 0.0 }, { "task": "query", "band": "easy", "setting": "low", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 1.76, "right_per_usd": 568.8, "answer_tokens": 28, "reasoning_tokens_est": 190, "latency_s_p50": 2.2, "cut_off_pct": 0.0 }, { "task": "query", "band": "easy", "setting": "high", "right_pct": 100.0, "range_pct": [ 90.2, 100.0 ], "questions": 25, "group": "top", "lines": [], "cost_per_1k_usd": 2.35, "right_per_usd": 425.3, "answer_tokens": 28, "reasoning_tokens_est": 313, "latency_s_p50": 2.2, "cut_off_pct": 0.0 }, { "task": "query", "band": "medium", "setting": "off", "right_pct": 90.9, "range_pct": [ 82.5, 95.5 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 1.76, "right_per_usd": 516.2, "answer_tokens": 186, "reasoning_tokens_est": 0, "latency_s_p50": 1.9, "cut_off_pct": 0.0 }, { "task": "query", "band": "medium", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 12.51, "right_per_usd": 78.5, "answer_tokens": 146, "reasoning_tokens_est": 2729, "latency_s_p50": 15.2, "cut_off_pct": 0.0 }, { "task": "query", "band": "medium", "setting": "low", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 8.92, "right_per_usd": 110.0, "answer_tokens": 145, "reasoning_tokens_est": 1849, "latency_s_p50": 13.7, "cut_off_pct": 0.0 }, { "task": "query", "band": "hard", "setting": "off", "right_pct": 84.8, "range_pct": [ 75.3, 91.1 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 2.24, "right_per_usd": 379.2, "answer_tokens": 293, "reasoning_tokens_est": 0, "latency_s_p50": 2.3, "cut_off_pct": 0.0 }, { "task": "query", "band": "hard", "setting": "low", "right_pct": 97.6, "range_pct": [ 91.3, 99.4 ], "questions": 55, "group": "near", "lines": [], "cost_per_1k_usd": 13.45, "right_per_usd": 72.5, "answer_tokens": 225, "reasoning_tokens_est": 2901, "latency_s_p50": 17.9, "cut_off_pct": 0.0 }, { "task": "query", "band": "hard", "setting": "high", "right_pct": 100.0, "range_pct": [ 95.3, 100.0 ], "questions": 55, "group": "top", "lines": [], "cost_per_1k_usd": 22.8, "right_per_usd": 43.9, "answer_tokens": 229, "reasoning_tokens_est": 5230, "latency_s_p50": 25.6, "cut_off_pct": 0.0 }, { "task": "query", "band": "frontier", "setting": "off", "right_pct": 84.8, "range_pct": [ 75.3, 91.1 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 3.06, "right_per_usd": 277.6, "answer_tokens": 494, "reasoning_tokens_est": 0, "latency_s_p50": 3.2, "cut_off_pct": 0.0 }, { "task": "query", "band": "frontier", "setting": "low", "right_pct": 94.5, "range_pct": [ 87.1, 97.8 ], "questions": 55, "group": "below", "lines": [], "cost_per_1k_usd": 26.46, "right_per_usd": 35.7, "answer_tokens": 326, "reasoning_tokens_est": 6077, "latency_s_p50": 47.1, "cut_off_pct": 0.0 }, { "task": "query", "band": "frontier", "setting": "high", "right_pct": 98.2, "range_pct": [ 92.3, 99.6 ], "questions": 55, "group": "near", "lines": [], "cost_per_1k_usd": 41.95, "right_per_usd": 23.4, "answer_tokens": 326, "reasoning_tokens_est": 9963, "latency_s_p50": 72.9, "cut_off_pct": 0.0 } ], "reliability": { "stability_flip_pct": 12.7 }, "history": [ { "run": "2026-09-25", "results": 24 } ] } ], "comparisons": [ { "path": "/compare/deepseek-flash-vs-deepseek-v4-pro/", "models": [ "deepseek-flash", "deepseek-v4-pro" ], "cells": [ { "task": "math", "band": "easy", "verdict": "same", "a": { "setting": "low", "right_pct": 100.0, "cost_per_1k_usd": 0.27 }, "b": { "setting": "low", "right_pct": 100.0, "cost_per_1k_usd": 0.81 } }, { "task": "math", "band": "medium", "verdict": "same", "a": { "setting": "low", "right_pct": 98.2, "cost_per_1k_usd": 0.62 }, "b": { "setting": "low", "right_pct": 98.2, "cost_per_1k_usd": 1.57 } }, { "task": "math", "band": "hard", "verdict": "same", "a": { "setting": "low", "right_pct": 100.0, "cost_per_1k_usd": 2.26 }, "b": { "setting": "high", "right_pct": 98.2, "cost_per_1k_usd": 15.63 } }, { "task": "math", "band": "frontier", "verdict": "b", "a": { "setting": "low", "right_pct": 96.4, "cost_per_1k_usd": 6.38 }, "b": { "setting": "high", "right_pct": 98.2, "cost_per_1k_usd": 48.68 } }, { "task": "query", "band": "easy", "verdict": "same", "a": { "setting": "off", "right_pct": 100.0, "cost_per_1k_usd": 0.24 }, "b": { "setting": "off", "right_pct": 100.0, "cost_per_1k_usd": 1.03 } }, { "task": "query", "band": "medium", "verdict": "same", "a": { "setting": "high", "right_pct": 98.2, "cost_per_1k_usd": 1.87 }, "b": { "setting": "low", "right_pct": 98.2, "cost_per_1k_usd": 8.92 } }, { "task": "query", "band": "hard", "verdict": "same", "a": { "setting": "low", "right_pct": 100.0, "cost_per_1k_usd": 2.1 }, "b": { "setting": "high", "right_pct": 100.0, "cost_per_1k_usd": 22.8 } }, { "task": "query", "band": "frontier", "verdict": "a", "a": { "setting": "low", "right_pct": 100.0, "cost_per_1k_usd": 4.29 }, "b": { "setting": "high", "right_pct": 98.2, "cost_per_1k_usd": 41.95 } } ] } ], "changes": [ { "date": "2026-09-25", "kind": "new_results", "model": "deepseek-flash", "note": { "en": "First results: math and SQL at three thinking settings.", "es": "Primeros resultados: matemáticas y SQL con tres niveles de razonamiento." } }, { "date": "2026-09-25", "kind": "new_results", "model": "deepseek-v4-pro", "note": { "en": "First results: math and SQL at three thinking settings.", "es": "Primeros resultados: matemáticas y SQL con tres niveles de razonamiento." } } ], "caveats": [ { "en": "AIME problems are public and may appear in model training data, so math results at the frontier level may be inflated. Differences between settings are the reliable part.", "es": "Los problemas AIME son públicos y pueden estar en los datos de entrenamiento, así que los resultados de matemáticas en el nivel más difícil pueden estar inflados. Lo fiable son las diferencias entre niveles de razonamiento." }, { "en": "SQL questions were kept only when an independent model outside the test answered them correctly, which limits how hard the hardest questions are.", "es": "Las preguntas de SQL se conservaron solo si un modelo independiente, ajeno a la prueba, las respondía bien, lo que limita la dificultad de las más difíciles." }, { "en": "These results compare DeepSeek models and settings with each other. Other providers' models have not been tested in this run.", "es": "Estos resultados comparan modelos y niveles de DeepSeek entre sí. Los modelos de otros proveedores no se probaron en esta ronda." } ], "trademark_note": { "en": "DeepSeek is a trademark of its owner. Zabio uses DeepSeek models through the DeepSeek API and is not affiliated with DeepSeek.", "es": "DeepSeek es una marca de su titular. Zabio usa los modelos de DeepSeek a través de la API de DeepSeek y no tiene relación societaria con DeepSeek." }, "aggregated": { "retrieved_at": "2026-10-01", "label_en": "Aggregated from public results", "label_es": "Agregado de resultados públicos", "sources": [ { "id": "epoch", "name": "Epoch AI", "url": "https://epoch.ai/benchmarks", "license": "CC BY 4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", "retrieved_at": "2026-10-01", "citation_en": "Epoch AI, Capabilities and Benchmarking, epoch.ai/benchmarks, retrieved 2026-10-01", "citation_es": "Epoch AI, Capabilities and Benchmarking, epoch.ai/benchmarks, consultado el 2026-10-01", "notes": "Epoch's own runs carry standard errors and logs; scores Epoch gathers from other leaderboards are marked as not own runs and weighted 0.5." }, { "id": "lmarena", "name": "LMArena", "url": "https://huggingface.co/datasets/lmarena-ai/leaderboard-dataset", "license": "CC BY 4.0", "license_url": "https://creativecommons.org/licenses/by/4.0/", "retrieved_at": "2026-10-01", "citation_en": "LMArena leaderboard dataset (lmarena-ai/leaderboard-dataset on Hugging Face), retrieved 2026-10-01", "citation_es": "Conjunto de datos del ranking de LMArena (lmarena-ai/leaderboard-dataset en Hugging Face), consultado el 2026-10-01", "notes": "Human preference ratings with confidence ranges, style controlled where the table offers it. Preference, not correctness: weighted 0.6." }, { "id": "livebench", "name": "LiveBench", "url": "https://livebench.ai", "license": "Code Apache 2.0 and MIT; leaderboard results used with attribution", "license_url": "https://github.com/LiveBench/LiveBench/blob/main/LICENSE", "retrieved_at": "2026-10-01", "version": "2026-06-25", "citation_en": "LiveBench (White et al.), leaderboard 2026-06-25, livebench.ai, retrieved 2026-10-01", "citation_es": "LiveBench (White et al.), ranking 2026-06-25, livebench.ai, consultado el 2026-10-01", "notes": "Contamination limited: new questions monthly. Scores are percent per task; question counts from the published cost table." } ], "weights": { "epoch_own_run": 1, "epoch_gathered": 0.5, "lmarena": 0.6, "livebench": 1, "inhouse": 1, "settings_close": 0.85, "settings_mismatch": 0.7, "best_standard_errors": 1, "cheap_standard_errors": 2, "z_cap_per_benchmark": 3, "indirect_comparison_weight": 0.5 }, "benchmarks": [ { "id": "epoch:gpqa_diamond", "source": "epoch", "name": "GPQA diamond", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "analysis", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:frontiermath", "source": "epoch", "name": "FrontierMath-2025-02-28-Private", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "math", "band": "frontier", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:frontiermath_tier_4", "source": "epoch", "name": "FrontierMath-Tier-4-2025-07-01-Private", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:frontiermath_tiers_1_3_v2", "source": "epoch", "name": "FrontierMath-Tiers-1-3-v2-Private", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "math", "band": "frontier", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:frontiermath_tier_4_v2", "source": "epoch", "name": "FrontierMath-Tier-4-v2-Private", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:math_level_5", "source": "epoch", "name": "MATH level 5", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "math", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:otis_mock_aime_2024_2025", "source": "epoch", "name": "OTIS Mock AIME 2024-2025", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "math", "band": "frontier", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:swe_bench_verified", "source": "epoch", "name": "SWE-Bench verified", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "code_fix", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:simpleqa_verified", "source": "epoch", "name": "SimpleQA Verified", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "conversation", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:chess_puzzles", "source": "epoch", "name": "Chess Puzzles", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:ebr_bench", "source": "epoch", "name": "EBR-bench", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:mystery_game_puzzles", "source": "epoch", "name": "Mystery Game Puzzles", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:mirrorcode", "source": "epoch", "name": "MirrorCode", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "task_type": "code_write", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:aider_polyglot_external", "source": "epoch", "name": "Aider polyglot", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "code_fix", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 225, "weight": 0.5 }, { "id": "epoch:adversarial_nli_external", "source": "epoch", "name": "ANLI", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:arc_agi_external", "source": "epoch", "name": "ARC-AGI", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "analysis", "band": "frontier", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 400, "weight": 0.5 }, { "id": "epoch:arc_ai2_external", "source": "epoch", "name": "ARC AI2", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:balrog_external", "source": "epoch", "name": "Balrog", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:bbh_external", "source": "epoch", "name": "BBH", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:cad_eval_external", "source": "epoch", "name": "CadEval", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:cybench_external", "source": "epoch", "name": "Cybench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:deepresearchbench_external", "source": "epoch", "name": "DeepResearch Bench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:fictionlivebench_external", "source": "epoch", "name": "Fiction.LiveBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "document_qa", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:geobench_external", "source": "epoch", "name": "GeoBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:gsm8k_external", "source": "epoch", "name": "GSM8K", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:gso_external", "source": "epoch", "name": "GSO-Bench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:hella_swag_external", "source": "epoch", "name": "HellaSwag", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:lambada_external", "source": "epoch", "name": "LAMBADA", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:lech_mazur_writing_external", "source": "epoch", "name": "Lech Mazur Writing", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:mmlu_external", "source": "epoch", "name": "MMLU", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 14042, "weight": 0.5 }, { "id": "epoch:open_book_qa_external", "source": "epoch", "name": "OpenBookQA", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:os_world_external", "source": "epoch", "name": "OSWorld", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "tool_use", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 369, "weight": 0.5 }, { "id": "epoch:piqa_external", "source": "epoch", "name": "PIQA", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:science_qa_external", "source": "epoch", "name": "ScienceQA", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:simplebench_external", "source": "epoch", "name": "SimpleBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "analysis", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 200, "weight": 0.5 }, { "id": "epoch:superglue_external", "source": "epoch", "name": "SuperGLUE", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:terminalbench_external", "source": "epoch", "name": "Terminal Bench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "tool_use", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 89, "weight": 0.5 }, { "id": "epoch:the_agent_company_external", "source": "epoch", "name": "The Agent Company", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "tool_use", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 175, "weight": 0.5 }, { "id": "epoch:trivia_qa_external", "source": "epoch", "name": "TriviaQA", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:vpct_external", "source": "epoch", "name": "VPCT", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:weirdml_external", "source": "epoch", "name": "WeirdML", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:wino_grande_external", "source": "epoch", "name": "Winogrande", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:apex_agents_external", "source": "epoch", "name": "APEX-Agents", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:arc_agi_2_external", "source": "epoch", "name": "ARC-AGI-2", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "analysis", "band": "frontier", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 120, "weight": 0.5 }, { "id": "epoch:hle_external", "source": "epoch", "name": "HLE", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "task_type": "analysis", "band": "frontier", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:posttrainbench_external", "source": "epoch", "name": "PostTrainBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:exploitbench_external", "source": "epoch", "name": "ExploitBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:rli_external", "source": "epoch", "name": "Remote Labor Index", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:gdpval_external", "source": "epoch", "name": "GDPval", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:cl_bench_external", "source": "epoch", "name": "CL-bench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:cl_bench_life_external", "source": "epoch", "name": "CL-bench Life", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:osworld_2_external", "source": "epoch", "name": "OSWorld 2.0", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:surface_evolver_bench_external", "source": "epoch", "name": "Surface Evolver Bench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:metr_time_horizons_external", "source": "epoch", "name": "METR Time Horizons", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:deepswe_external", "source": "epoch", "name": "DeepSWE", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:proofbench_external", "source": "epoch", "name": "ProofBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:frontiercode_external", "source": "epoch", "name": "FrontierCode", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:lmca_external", "source": "epoch", "name": "LMCA", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:dtbench_external", "source": "epoch", "name": "DTBench", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "epoch:furniture_assembly", "source": "epoch", "name": "Furniture Assembly", "url": "https://epoch.ai/benchmarks", "measures_en": "Share of right answers, run by Epoch AI with a standard error.", "measures_es": "Porcentaje de respuestas correctas, ejecutado por Epoch AI con error estándar.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 1 }, { "id": "epoch:frontierswe_external", "source": "epoch", "name": "FrontierSWE", "url": "https://epoch.ai/benchmarks", "measures_en": "Score gathered by Epoch AI from the benchmark's own leaderboard or the vendor.", "measures_es": "Puntuación recogida por Epoch AI del ranking del propio benchmark o del proveedor.", "kind": "correct", "unit": "percent", "higher_is_better": true, "weight": 0.5 }, { "id": "lmarena:text_style_control/overall", "source": "lmarena", "name": "LMArena text style control, overall", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/chinese", "source": "lmarena", "name": "LMArena text style control, chinese", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/coding", "source": "lmarena", "name": "LMArena text style control, coding", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "code_write", "band": "hard", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/creative_writing", "source": "lmarena", "name": "LMArena text style control, creative writing", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "creative", "band": "medium", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/english", "source": "lmarena", "name": "LMArena text style control, english", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/exclude_ties", "source": "lmarena", "name": "LMArena text style control, exclude ties", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/expert", "source": "lmarena", "name": "LMArena text style control, expert", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "analysis", "band": "frontier", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/french", "source": "lmarena", "name": "LMArena text style control, french", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/german", "source": "lmarena", "name": "LMArena text style control, german", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/hard_prompts", "source": "lmarena", "name": "LMArena text style control, hard prompts", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "analysis", "band": "hard", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/hard_prompts_english", "source": "lmarena", "name": "LMArena text style control, hard prompts english", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "source": "lmarena", "name": "LMArena text style control, industry business and management and financial operations", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "source": "lmarena", "name": "LMArena text style control, industry entertainment and sports and media", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_legal_and_government", "source": "lmarena", "name": "LMArena text style control, industry legal and government", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "source": "lmarena", "name": "LMArena text style control, industry life and physical and social science", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_mathematical", "source": "lmarena", "name": "LMArena text style control, industry mathematical", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare", "source": "lmarena", "name": "LMArena text style control, industry medicine and healthcare", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_software_and_it_services", "source": "lmarena", "name": "LMArena text style control, industry software and it services", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language", "source": "lmarena", "name": "LMArena text style control, industry writing and literature and language", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/instruction_following", "source": "lmarena", "name": "LMArena text style control, instruction following", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "writing", "band": "medium", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/japanese", "source": "lmarena", "name": "LMArena text style control, japanese", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/korean", "source": "lmarena", "name": "LMArena text style control, korean", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/longer_query", "source": "lmarena", "name": "LMArena text style control, longer query", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "document_qa", "band": "medium", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/math", "source": "lmarena", "name": "LMArena text style control, math", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "math", "band": "medium", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/multi_turn", "source": "lmarena", "name": "LMArena text style control, multi turn", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "conversation", "band": "medium", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/non_english", "source": "lmarena", "name": "LMArena text style control, non english", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/polish", "source": "lmarena", "name": "LMArena text style control, polish", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/russian", "source": "lmarena", "name": "LMArena text style control, russian", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:text_style_control/spanish", "source": "lmarena", "name": "LMArena text style control, spanish", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:webdev/overall", "source": "lmarena", "name": "LMArena webdev, overall", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "code_write", "band": "hard", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:webdev/image_to_webdev", "source": "lmarena", "name": "LMArena webdev, image to webdev", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:webdev/webdev", "source": "lmarena", "name": "LMArena webdev, webdev", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:webdev/webdev-html", "source": "lmarena", "name": "LMArena webdev, webdev-html", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:webdev/webdev-react", "source": "lmarena", "name": "LMArena webdev, webdev-react", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/overall", "source": "lmarena", "name": "LMArena vision, overall", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "vision", "band": "medium", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/captioning", "source": "lmarena", "name": "LMArena vision, captioning", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/chinese", "source": "lmarena", "name": "LMArena vision, chinese", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/creative_writing", "source": "lmarena", "name": "LMArena vision, creative writing", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/creative_writing_vision", "source": "lmarena", "name": "LMArena vision, creative writing vision", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/diagram", "source": "lmarena", "name": "LMArena vision, diagram", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/english", "source": "lmarena", "name": "LMArena vision, english", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/entity_recognition", "source": "lmarena", "name": "LMArena vision, entity recognition", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/homework", "source": "lmarena", "name": "LMArena vision, homework", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/humor", "source": "lmarena", "name": "LMArena vision, humor", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:vision/ocr", "source": "lmarena", "name": "LMArena vision, ocr", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:document/overall", "source": "lmarena", "name": "LMArena document, overall", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "document_qa", "band": "hard", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:agent/overall", "source": "lmarena", "name": "LMArena agent, overall", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "task_type": "tool_use", "band": "hard", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "lmarena:search/overall", "source": "lmarena", "name": "LMArena search, overall", "url": "https://lmarena.ai/leaderboard", "measures_en": "Human preference rating from blind pairwise votes, with a confidence range.", "measures_es": "Puntuación de preferencia humana a partir de votos ciegos por pares, con rango de confianza.", "kind": "preference", "unit": "rating", "higher_is_better": true, "weight": 0.6 }, { "id": "livebench:task/AMPS_Hard", "source": "livebench", "name": "LiveBench AMPS Hard", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 100, "weight": 1 }, { "id": "livebench:task/code_completion", "source": "livebench", "name": "LiveBench code completion", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 46, "weight": 1 }, { "id": "livebench:task/code_generation", "source": "livebench", "name": "LiveBench code generation", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 71, "weight": 1 }, { "id": "livebench:task/connections", "source": "livebench", "name": "LiveBench connections", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/consecutive_events", "source": "livebench", "name": "LiveBench consecutive events", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 47, "weight": 1 }, { "id": "livebench:task/integrals_with_game", "source": "livebench", "name": "LiveBench integrals with game", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 100, "weight": 1 }, { "id": "livebench:task/javascript", "source": "livebench", "name": "LiveBench javascript", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 22, "weight": 1 }, { "id": "livebench:task/logic_with_navigation", "source": "livebench", "name": "LiveBench logic with navigation", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/math_comp", "source": "livebench", "name": "LiveBench math comp", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 102, "weight": 1 }, { "id": "livebench:task/olympiad", "source": "livebench", "name": "LiveBench olympiad", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 77, "weight": 1 }, { "id": "livebench:task/paraphrase", "source": "livebench", "name": "LiveBench paraphrase", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "task_type": "writing", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/plot_unscrambling", "source": "livebench", "name": "LiveBench plot unscrambling", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/python", "source": "livebench", "name": "LiveBench python", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 20, "weight": 1 }, { "id": "livebench:task/simplify", "source": "livebench", "name": "LiveBench simplify", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/spatial", "source": "livebench", "name": "LiveBench spatial", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/story_generation", "source": "livebench", "name": "LiveBench story generation", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "task_type": "creative", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/summarize", "source": "livebench", "name": "LiveBench summarize", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "task_type": "summarization", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/tablejoin", "source": "livebench", "name": "LiveBench tablejoin", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "task_type": "query", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 52, "weight": 1 }, { "id": "livebench:task/tablereformat", "source": "livebench", "name": "LiveBench tablereformat", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "task_type": "extraction", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 51, "weight": 1 }, { "id": "livebench:task/theory_of_mind", "source": "livebench", "name": "LiveBench theory of mind", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 52, "weight": 1 }, { "id": "livebench:task/typescript", "source": "livebench", "name": "LiveBench typescript", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 30, "weight": 1 }, { "id": "livebench:task/typos", "source": "livebench", "name": "LiveBench typos", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:task/zebra_puzzle", "source": "livebench", "name": "LiveBench zebra puzzle", "url": "https://livebench.ai", "measures_en": "Share of right answers on this task, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en esta tarea, con preguntas renovadas cada mes.", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 50, "weight": 1 }, { "id": "livebench:category/Agentic Coding", "source": "livebench", "name": "LiveBench Agentic Coding", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "code_fix", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 72, "weight": 1 }, { "id": "livebench:category/Coding", "source": "livebench", "name": "LiveBench Coding", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "code_write", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 117, "weight": 1 }, { "id": "livebench:category/Data Analysis", "source": "livebench", "name": "LiveBench Data Analysis", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "extraction", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 150, "weight": 1 }, { "id": "livebench:category/IF", "source": "livebench", "name": "LiveBench IF", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "classification", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 200, "weight": 1 }, { "id": "livebench:category/Language", "source": "livebench", "name": "LiveBench Language", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "writing", "band": "medium", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 150, "weight": 1 }, { "id": "livebench:category/Mathematics", "source": "livebench", "name": "LiveBench Mathematics", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "math", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 379, "weight": 1 }, { "id": "livebench:category/Reasoning", "source": "livebench", "name": "LiveBench Reasoning", "url": "https://livebench.ai", "measures_en": "Share of right answers across the category's tasks, questions refreshed monthly.", "measures_es": "Porcentaje de respuestas correctas en las tareas de la categoría, con preguntas renovadas cada mes.", "task_type": "analysis", "band": "hard", "kind": "correct", "unit": "percent", "higher_is_better": true, "questions": 202, "weight": 1 } ], "results": [ { "id": "epoch:aider_polyglot_external:codestral-2501", "source": "epoch", "benchmark": "epoch:aider_polyglot_external", "model": "codestral-2508", "source_model": "codestral-2501", "settings": "exact", "score": 11.1, "se": 2.09, "n": 225, "unit": "percent", "own_run": false, "date": "2025-01-13" }, { "id": "epoch:aider_polyglot_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:aider_polyglot_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 52.4, "se": 3.33, "n": 225, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:aider_polyglot_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:aider_polyglot_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 32.4, "se": 3.12, "n": 225, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:aider_polyglot_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:aider_polyglot_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 3.6, "se": 1.24, "n": 225, "unit": "percent", "own_run": false, "date": "2024-12-21" }, { "id": "epoch:aider_polyglot_external:gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:aider_polyglot_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 41.8, "se": 3.29, "n": 225, "unit": "percent", "own_run": false }, { "id": "epoch:aider_polyglot_external:openai/gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:aider_polyglot_external", "model": "gpt-oss-120b", "source_model": "openai/gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 41.8, "se": 3.29, "n": 225, "unit": "percent", "own_run": false, "date": "2025-08-06" }, { "id": "epoch:apex_agents_external:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 59.7, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:apex_agents_external:claude-fable-5-1_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_unknown", "level": "unknown", "settings": "exact", "score": 68.6, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:apex_agents_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 48.9, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:apex_agents_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 73.5, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:apex_agents_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 65.8, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:apex_agents_external:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 43, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:apex_agents_external:claude-sonnet-5_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_unknown", "level": "unknown", "settings": "exact", "score": 54.5, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:apex_agents_external:deepseek-v4-pro-0813_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_unknown", "level": "unknown", "settings": "exact", "score": 47.3, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:apex_agents_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 35.3, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:apex_agents_external:gemini-3.5-flash_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_unknown", "level": "unknown", "settings": "exact", "score": 27.5, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:apex_agents_external:gemini-3.8-flash_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_unknown", "level": "unknown", "settings": "exact", "score": 64.3, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:apex_agents_external:gpt-5.4-2026-03-05_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_unknown", "level": "unknown", "settings": "exact", "score": 52.4, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:apex_agents_external:gpt-5.5_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gpt-5.5", "source_model": "gpt-5.5_unknown", "level": "unknown", "settings": "exact", "score": 55.1, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:apex_agents_external:gpt-6-astra_unknown", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_unknown", "level": "unknown", "settings": "exact", "score": 64.7, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:apex_agents_external:gpt-oss-120b", "source": "epoch", "benchmark": "epoch:apex_agents_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 4.4, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:arc_agi_2_external:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 88.75, "se": 2.88, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_2_external:claude-fable-5-1_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_low", "level": "low", "settings": "close", "score": 78.33, "se": 3.76, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_2_external:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 90, "se": 2.74, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_2_external:claude-fable-5-1_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_medium", "level": "medium", "settings": "mismatch", "score": 86.25, "se": 3.14, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_2_external:claude-fable-5-1_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_xhigh", "level": "xhigh", "settings": "mismatch", "score": 90, "se": 2.74, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_2_external:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1.25, "se": 1.01, "n": 120, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_2_external:claude-haiku-4-5-20251001_16K", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_16K", "level": "thinking", "settings": "close", "score": 2.78, "se": 1.5, "n": 120, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_2_external:claude-haiku-4-5-20251001_1K", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_1K", "level": "thinking", "settings": "close", "score": 1.25, "se": 1.01, "n": 120, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_2_external:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 4.03, "se": 1.8, "n": 120, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_2_external:claude-haiku-4-5-20251001_8K", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_8K", "level": "thinking", "settings": "close", "score": 1.67, "se": 1.17, "n": 120, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_2_external:claude-opus-4-8_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_high", "level": "high", "settings": "mismatch", "score": 72.08, "se": 4.1, "n": 120, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_2_external:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 62.22, "se": 4.43, "n": 120, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_2_external:claude-opus-4-8_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_medium", "level": "medium", "settings": "mismatch", "score": 71.67, "se": 4.11, "n": 120, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_2_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 91.67, "se": 2.52, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:arc_agi_2_external:claude-opus-5-5_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 92.5, "se": 2.4, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:arc_agi_2_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 88.33, "se": 2.93, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:arc_agi_2_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 88.33, "se": 2.93, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:arc_agi_2_external:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 60.42, "se": 4.46, "n": 120, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:arc_agi_2_external:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 58.33, "se": 4.5, "n": 120, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-flash-0731_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_high", "level": "high", "settings": "mismatch", "score": 55.97, "se": 4.53, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-flash-0731_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_low", "level": "low", "settings": "close", "score": 45.97, "se": 4.55, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 61.39, "se": 4.44, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-flash-0731_none", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_none", "level": "none", "settings": "exact", "score": 2.08, "se": 1.3, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-pro-0813_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_high", "level": "high", "settings": "mismatch", "score": 59.72, "se": 4.48, "n": 120, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-pro-0813_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_low", "level": "low", "settings": "close", "score": 56.25, "se": 4.53, "n": 120, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 61.25, "se": 4.45, "n": 120, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_2_external:deepseek-v4-pro-0813_none", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_none", "level": "none", "settings": "exact", "score": 0.83, "se": 0.83, "n": 120, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_2_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 77.08, "se": 3.84, "n": 120, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:arc_agi_2_external:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 10.28, "se": 2.77, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_2_external:gemini-3.5-flash-lite_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_low", "level": "low", "settings": "close", "score": 1.53, "se": 1.12, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_2_external:gemini-3.5-flash-lite_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_medium", "level": "medium", "settings": "mismatch", "score": 5.28, "se": 2.04, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_2_external:gemini-3.5-flash-lite_minimal", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_minimal", "level": "minimal", "settings": "close", "score": 0.83, "se": 0.83, "n": 120, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_2_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 72.08, "se": 4.1, "n": 120, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:arc_agi_2_external:gemini-3.5-flash_minimal", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_minimal", "level": "minimal", "settings": "close", "score": 8.89, "se": 2.6, "n": 120, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:arc_agi_2_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 0.42, "se": 0.59, "n": 120, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:arc_agi_2_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 0, "se": 0.42, "n": 120, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:arc_agi_2_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 0, "se": 0.42, "n": 120, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:arc_agi_2_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 4.44, "se": 1.88, "n": 120, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_2_external:gpt-5-mini-2025-08-07_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_low", "level": "low", "settings": "close", "score": 0.83, "se": 0.83, "n": 120, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_2_external:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 4.03, "se": 1.8, "n": 120, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_2_external:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 1.67, "se": 1.17, "n": 120, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 67.5, "se": 4.28, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 29.17, "se": 4.15, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-2026-03-05_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_medium", "level": "medium", "settings": "mismatch", "score": 55.42, "se": 4.54, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 73.95, "se": 4.01, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 13.19, "se": 3.09, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-mini-2026-03-17_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_low", "level": "low", "settings": "close", "score": 1.11, "se": 0.96, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-mini-2026-03-17_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_medium", "level": "medium", "settings": "mismatch", "score": 4.44, "se": 1.88, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 18.9, "se": 3.57, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 3.61, "se": 1.7, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 1.53, "se": 1.12, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-nano-2026-03-17_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_medium", "level": "medium", "settings": "mismatch", "score": 1.94, "se": 1.26, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.4-nano-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 5.69, "se": 2.11, "n": 120, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_2_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 83.33, "se": 3.4, "n": 120, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_2_external:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 33.33, "se": 4.3, "n": 120, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_2_external:gpt-5.5_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.5", "source_model": "gpt-5.5_medium", "level": "medium", "settings": "mismatch", "score": 70.42, "se": 4.17, "n": 120, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_2_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 85, "se": 3.26, "n": 120, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_2_external:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 92.08, "se": 2.46, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_2_external:gpt-6-astra_low", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_low", "level": "low", "settings": "close", "score": 85.42, "se": 3.22, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_2_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 95, "se": 1.99, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_2_external:gpt-6-astra_medium", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_medium", "level": "medium", "settings": "mismatch", "score": 92.08, "se": 2.46, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_2_external:gpt-6-astra_none", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_none", "level": "none", "settings": "exact", "score": 59.58, "se": 4.48, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_2_external:gpt-6-astra_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_2_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_xhigh", "level": "xhigh", "settings": "mismatch", "score": 93.33, "se": 2.28, "n": 120, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_external:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 96, "se": 0.98, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_external:claude-fable-5-1_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_low", "level": "low", "settings": "close", "score": 90, "se": 1.5, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_external:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_external:claude-fable-5-1_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_medium", "level": "medium", "settings": "mismatch", "score": 94.5, "se": 1.14, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_external:claude-fable-5-1_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_xhigh", "level": "xhigh", "settings": "mismatch", "score": 96.5, "se": 0.92, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:arc_agi_external:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 14.33, "se": 1.75, "n": 400, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_external:claude-haiku-4-5-20251001_16K", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_16K", "level": "thinking", "settings": "close", "score": 37.33, "se": 2.42, "n": 400, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_external:claude-haiku-4-5-20251001_1K", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_1K", "level": "thinking", "settings": "close", "score": 16.83, "se": 1.87, "n": 400, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_external:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 47.67, "se": 2.5, "n": 400, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_external:claude-haiku-4-5-20251001_8K", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_8K", "level": "thinking", "settings": "close", "score": 25.5, "se": 2.18, "n": 400, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:arc_agi_external:claude-opus-4-8_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_high", "level": "high", "settings": "mismatch", "score": 92, "se": 1.36, "n": 400, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_external:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 88, "se": 1.62, "n": 400, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 92.5, "se": 1.32, "n": 400, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_external:claude-opus-4-8_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_medium", "level": "medium", "settings": "mismatch", "score": 91.5, "se": 1.39, "n": 400, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:arc_agi_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:arc_agi_external:claude-opus-5-5_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:arc_agi_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:arc_agi_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:arc_agi_external:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 86.5, "se": 1.71, "n": 400, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:arc_agi_external:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 86, "se": 1.73, "n": 400, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:arc_agi_external:deepseek-v4-flash-0731_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_high", "level": "high", "settings": "mismatch", "score": 87, "se": 1.68, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_external:deepseek-v4-flash-0731_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_low", "level": "low", "settings": "close", "score": 84, "se": 1.83, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_external:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 89, "se": 1.56, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_external:deepseek-v4-flash-0731_none", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_none", "level": "none", "settings": "exact", "score": 11.83, "se": 1.62, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:arc_agi_external:deepseek-v4-pro-0813_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_high", "level": "high", "settings": "mismatch", "score": 87.17, "se": 1.67, "n": 400, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_external:deepseek-v4-pro-0813_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_low", "level": "low", "settings": "close", "score": 90.5, "se": 1.47, "n": 400, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_external:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 90, "se": 1.5, "n": 400, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_external:deepseek-v4-pro-0813_none", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_none", "level": "none", "settings": "exact", "score": 13, "se": 1.68, "n": 400, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:arc_agi_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 98, "se": 0.7, "n": 400, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:arc_agi_external:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 53.5, "se": 2.49, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_external:gemini-3.5-flash-lite_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_low", "level": "low", "settings": "close", "score": 17, "se": 1.88, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_external:gemini-3.5-flash-lite_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_medium", "level": "medium", "settings": "mismatch", "score": 32.33, "se": 2.34, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_external:gemini-3.5-flash-lite_minimal", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_minimal", "level": "minimal", "settings": "close", "score": 7.5, "se": 1.32, "n": 400, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:arc_agi_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 92.5, "se": 1.32, "n": 400, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:arc_agi_external:gemini-3.5-flash_minimal", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_minimal", "level": "minimal", "settings": "close", "score": 48.83, "se": 2.5, "n": 400, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:arc_agi_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 5.5, "se": 1.14, "n": 400, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:arc_agi_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 3.5, "se": 0.92, "n": 400, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:arc_agi_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 54.33, "se": 2.49, "n": 400, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_external:gpt-5-mini-2025-08-07_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_low", "level": "low", "settings": "close", "score": 26.33, "se": 2.2, "n": 400, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_external:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 37.33, "se": 2.42, "n": 400, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_external:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 5.33, "se": 1.12, "n": 400, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:arc_agi_external:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 92.67, "se": 1.3, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_external:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 68.17, "se": 2.33, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_external:gpt-5.4-2026-03-05_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_medium", "level": "medium", "settings": "mismatch", "score": 86.17, "se": 1.73, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 93.67, "se": 1.22, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:arc_agi_external:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 58, "se": 2.47, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-mini-2026-03-17_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_low", "level": "low", "settings": "close", "score": 13, "se": 1.68, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-mini-2026-03-17_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_medium", "level": "medium", "settings": "mismatch", "score": 40.83, "se": 2.46, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 63.67, "se": 2.4, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 38.17, "se": 2.43, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 18.33, "se": 1.93, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-nano-2026-03-17_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_medium", "level": "medium", "settings": "mismatch", "score": 33, "se": 2.35, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.4-nano-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 51.5, "se": 2.5, "n": 400, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:arc_agi_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 94.5, "se": 1.14, "n": 400, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_external:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 76.17, "se": 2.13, "n": 400, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_external:gpt-5.5_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.5", "source_model": "gpt-5.5_medium", "level": "medium", "settings": "mismatch", "score": 92.17, "se": 1.34, "n": 400, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 95, "se": 1.09, "n": 400, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:arc_agi_external:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 98.5, "se": 0.61, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_external:gpt-6-astra_low", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_low", "level": "low", "settings": "close", "score": 96.5, "se": 0.92, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_external:gpt-6-astra_medium", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_medium", "level": "medium", "settings": "mismatch", "score": 97.5, "se": 0.78, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_external:gpt-6-astra_none", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_none", "level": "none", "settings": "exact", "score": 86, "se": 1.73, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:arc_agi_external:gpt-6-astra_xhigh", "source": "epoch", "benchmark": "epoch:arc_agi_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_xhigh", "level": "xhigh", "settings": "mismatch", "score": 98.5, "se": 0.61, "n": 400, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:balrog_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 23, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:balrog_external:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 31.2, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:balrog_external:claude-haiku-4-5-20251001_1K", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_1K", "level": "thinking", "settings": "close", "score": 31.2, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:balrog_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 63.4, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:balrog_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 56.9, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:balrog_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 17.4, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:balrog_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:balrog_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 68.3, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:cad_eval_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:cad_eval_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 42, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:cad_eval_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:cad_eval_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 16, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:chess_puzzles:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 47, "se": 5.016135580465917, "unit": "percent", "own_run": true, "date": "2026-09-01" }, { "id": "epoch:chess_puzzles:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 8, "se": 2.7265992434429074, "unit": "percent", "own_run": true, "date": "2026-07-16" }, { "id": "epoch:chess_puzzles:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 29, "se": 4.560480215720686, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 34, "se": 4.760952285695233, "unit": "percent", "own_run": true, "date": "2026-05-29" }, { "id": "epoch:chess_puzzles:claude-opus-4-8_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_none", "level": "none", "settings": "exact", "score": 13, "se": 3.379976689896311, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:claude-opus-5", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-opus-5", "source_model": "claude-opus-5", "settings": "exact", "score": 33, "se": 4.725815626252608, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 20, "se": 4.020151261036848, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 42, "se": 4.9604496374885825, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:chess_puzzles:claude-sonnet-4-6_32K", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_32K", "level": "thinking", "settings": "mismatch", "score": 13, "se": 3.379976689896311, "unit": "percent", "own_run": true, "date": "2026-02-20" }, { "id": "epoch:chess_puzzles:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 5, "se": 2.1904291355759025, "unit": "percent", "own_run": true, "date": "2026-07-13" }, { "id": "epoch:chess_puzzles:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 3, "se": 1.7144660799776528, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:claude-sonnet-4-6_medium", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_medium", "level": "medium", "settings": "mismatch", "score": 8, "se": 2.7265992434429074, "unit": "percent", "own_run": true, "date": "2026-07-13" }, { "id": "epoch:chess_puzzles:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 16, "se": 3.6845294917747067, "unit": "percent", "own_run": true, "date": "2026-06-30" }, { "id": "epoch:chess_puzzles:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 35, "se": 4.793724854411022, "unit": "percent", "own_run": true, "date": "2026-06-30" }, { "id": "epoch:chess_puzzles:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 33, "se": 4.725815626252608, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:chess_puzzles:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 47, "se": 5.016135580465917, "unit": "percent", "own_run": true, "date": "2026-08-18" }, { "id": "epoch:chess_puzzles:deepseek-v4-pro_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_high", "level": "high", "settings": "mismatch", "score": 13, "se": 3.3799766898963113, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 20, "se": 4.020151261036848, "unit": "percent", "own_run": true, "date": "2026-06-16" }, { "id": "epoch:chess_puzzles:deepseek-v4-pro_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_none", "level": "none", "settings": "exact", "score": 7, "se": 2.564323999762428, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 55, "se": 5, "unit": "percent", "own_run": true, "date": "2026-02-19" }, { "id": "epoch:chess_puzzles:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 49, "se": 5.024183937956915, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 22, "se": 4.163331998932265, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:gemini-3.5-flash-lite_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_low", "level": "low", "settings": "close", "score": 18, "se": 3.8612291966536914, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:gemini-3.5-flash-lite_minimal", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_minimal", "level": "minimal", "settings": "close", "score": 21, "se": 4.093601807403323, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 50, "se": 5.02518907629606, "unit": "percent", "own_run": true, "date": "2026-05-28" }, { "id": "epoch:chess_puzzles:gemini-3.5-flash_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_low", "level": "low", "settings": "close", "score": 45, "se": 5, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:chess_puzzles:gemini-3.5-flash_minimal", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_minimal", "level": "minimal", "settings": "close", "score": 43, "se": 4.97569851956243, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 61, "se": 4.902071300001973, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:chess_puzzles:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 6, "se": 2.3868325657594203, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 7, "se": 2.564323999762428, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 0, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 30, "se": 4.605661864718382, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5-mini-2025-08-07_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_low", "level": "low", "settings": "close", "score": 12, "se": 3.2659863237109037, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 7, "se": 2.564323999762428, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 38, "se": 4.878317312145634, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 20, "se": 4.020151261036848, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gpt-5.4-2026-03-05_medium", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_medium", "level": "medium", "settings": "mismatch", "score": 38, "se": 4.878317312145633, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gpt-5.4-2026-03-05_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_none", "level": "none", "settings": "exact", "score": 5, "se": 2.1904291355759034, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:chess_puzzles:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 44, "se": 5, "unit": "percent", "own_run": true, "date": "2026-03-11" }, { "id": "epoch:chess_puzzles:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 18, "se": 3.8612291966536914, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:chess_puzzles:gpt-5.4-mini-2026-03-17_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_none", "level": "none", "settings": "exact", "score": 3, "se": 1.7144660799776528, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 24, "se": 4.2923469599092785, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 30, "se": 4.61, "unit": "percent", "own_run": true, "date": "2026-04-14" }, { "id": "epoch:chess_puzzles:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 17, "se": 3.7752516806863685, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5.4-nano-2026-03-17_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_none", "level": "none", "settings": "exact", "score": 3, "se": 1.7144660799776528, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 26, "se": 4.4084400227680804, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-5.5_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-5.5", "source_model": "gpt-5.5_none", "level": "none", "settings": "exact", "score": 10, "se": 3.015113445777636, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:chess_puzzles:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 72, "se": 4.512608598542129, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:chess_puzzles:openai/gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "gpt-oss-120b", "source_model": "openai/gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 20, "se": 3.0977345909486176, "unit": "percent", "own_run": true, "date": "2025-12-11" }, { "id": "epoch:chess_puzzles:qwen3.5-9B", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B", "settings": "exact", "score": 2, "se": 1.4070529413628967, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:chess_puzzles:qwen3.5-9B_none", "source": "epoch", "benchmark": "epoch:chess_puzzles", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B_none", "level": "none", "settings": "exact", "score": 12, "se": 3.2659863237109037, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:cl_bench_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:cl_bench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 20.8, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:cl_bench_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:cl_bench_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 27.9, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:cl_bench_life_external:deepseek-v4-pro_high", "source": "epoch", "benchmark": "epoch:cl_bench_life_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_high", "level": "high", "settings": "mismatch", "score": 13.5, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:cl_bench_life_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:cl_bench_life_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 16.9, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:cl_bench_life_external:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:cl_bench_life_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 19.3, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:cl_bench_life_external:gpt-5.4-2026-03-05_unknown", "source": "epoch", "benchmark": "epoch:cl_bench_life_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_unknown", "level": "unknown", "settings": "exact", "score": 13.8, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:cl_bench_life_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:cl_bench_life_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 21.7, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:cl_bench_life_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:cl_bench_life_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 22.2, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepresearchbench_external:claude-haiku-4-5-20251001_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_low", "level": "low", "settings": "close", "score": 45.5, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:deepresearchbench_external:claude-opus-4-8_high", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_high", "level": "high", "settings": "mismatch", "score": 50.2, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepresearchbench_external:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 49.3, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepresearchbench_external:claude-opus-4-8_medium", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_medium", "level": "medium", "settings": "mismatch", "score": 47.4, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepresearchbench_external:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 54.9, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:deepresearchbench_external:claude-sonnet-4-6_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_low", "level": "low", "settings": "close", "score": 50.4, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:deepresearchbench_external:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 47.8, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:deepresearchbench_external:gemini-3.1-pro-preview_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_low", "level": "low", "settings": "close", "score": 44.5, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:deepresearchbench_external:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 35.1, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:deepresearchbench_external:gpt-5.4-mini-2026-03-17_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_low", "level": "low", "settings": "close", "score": 36.3, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:deepresearchbench_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 54, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepresearchbench_external:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 48.7, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepresearchbench_external:gpt-5.5_medium", "source": "epoch", "benchmark": "epoch:deepresearchbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_medium", "level": "medium", "settings": "mismatch", "score": 49.6, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepswe_external:claude-opus-4-8_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_high", "level": "high", "settings": "mismatch", "score": 51.77, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepswe_external:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 40.8, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepswe_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 58.97, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepswe_external:claude-opus-4-8_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_medium", "level": "medium", "settings": "mismatch", "score": 48.67, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepswe_external:claude-opus-4-8_xhigh", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_xhigh", "level": "xhigh", "settings": "mismatch", "score": 54.36, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:deepswe_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 72.83, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:deepswe_external:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 58.13, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:deepswe_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 73.65, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:deepswe_external:claude-opus-5_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-5", "source_model": "claude-opus-5_medium", "level": "medium", "settings": "mismatch", "score": 68.9, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:deepswe_external:claude-opus-5_xhigh", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-opus-5", "source_model": "claude-opus-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 73.15, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:deepswe_external:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 29.93, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:deepswe_external:claude-sonnet-5_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_high", "level": "high", "settings": "mismatch", "score": 48.23, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:deepswe_external:claude-sonnet-5_low", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_low", "level": "low", "settings": "close", "score": 30.51, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:deepswe_external:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 53.85, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:deepswe_external:claude-sonnet-5_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_medium", "level": "medium", "settings": "mismatch", "score": 39.78, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:deepswe_external:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 49.67, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:deepswe_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 11.73, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:deepswe_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 36.06, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:deepswe_external:gemini-3.5-flash_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_medium", "level": "medium", "settings": "mismatch", "score": 37.39, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:deepswe_external:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 73.83, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:deepswe_external:gemini-3.8-flash_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_medium", "level": "medium", "settings": "mismatch", "score": 71.02, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:deepswe_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 51.77, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:deepswe_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 64.38, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepswe_external:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 26.99, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepswe_external:gpt-5.5_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-5.5", "source_model": "gpt-5.5_medium", "level": "medium", "settings": "mismatch", "score": 53.98, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepswe_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 67.04, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:deepswe_external:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 73.23, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:deepswe_external:gpt-6-astra_low", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_low", "level": "low", "settings": "close", "score": 67.04, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:deepswe_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 73.23, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:deepswe_external:gpt-6-astra_medium", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_medium", "level": "medium", "settings": "mismatch", "score": 72.79, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:deepswe_external:gpt-6-astra_xhigh", "source": "epoch", "benchmark": "epoch:deepswe_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_xhigh", "level": "xhigh", "settings": "mismatch", "score": 74.12, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:dtbench_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 59.47, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:dtbench_external:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 96.53, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:dtbench_external:claude-fable-5-1_low", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_low", "level": "low", "settings": "close", "score": 94.13, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:dtbench_external:claude-fable-5-1_medium", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_medium", "level": "medium", "settings": "mismatch", "score": 97.07, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:dtbench_external:claude-fable-5-1_unknown", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_unknown", "level": "unknown", "settings": "exact", "score": 97.6, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:dtbench_external:claude-fable-5-1_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_xhigh", "level": "xhigh", "settings": "mismatch", "score": 97.6, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:dtbench_external:claude-haiku-4-5-20251001_unknown", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_unknown", "level": "unknown", "settings": "exact", "score": 73.6, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:dtbench_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 94.93, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:dtbench_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 98.93, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:dtbench_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 97.87, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:dtbench_external:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 93.07, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:dtbench_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 97.6, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:dtbench_external:claude-opus-5_medium", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_medium", "level": "medium", "settings": "mismatch", "score": 94.93, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:dtbench_external:claude-opus-5_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 97.87, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:dtbench_external:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 89.87, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:dtbench_external:claude-sonnet-5_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_high", "level": "high", "settings": "mismatch", "score": 84.53, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:dtbench_external:claude-sonnet-5_low", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_low", "level": "low", "settings": "close", "score": 73.6, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:dtbench_external:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 92.48, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:dtbench_external:claude-sonnet-5_medium", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_medium", "level": "medium", "settings": "mismatch", "score": 79.2, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:dtbench_external:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 89.51, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:dtbench_external:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 90.93, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:dtbench_external:deepseek-v4-flash_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash_max", "level": "max", "settings": "mismatch", "score": 86.4, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:dtbench_external:deepseek-v4-pro-0813_unknown", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_unknown", "level": "unknown", "settings": "exact", "score": 93.87, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:dtbench_external:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 90.67, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:dtbench_external:deepseek-v4.1-flash_unknown", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash_unknown", "level": "unknown", "settings": "exact", "score": 89.87, "unit": "percent", "own_run": false, "date": "2026-09-09" }, { "id": "epoch:dtbench_external:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 95.47, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:dtbench_external:gemini-3.1-pro-preview_low", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_low", "level": "low", "settings": "close", "score": 94.93, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:dtbench_external:gemini-3.1-pro-preview_medium", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_medium", "level": "medium", "settings": "close", "score": 97.07, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:dtbench_external:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 83.47, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:dtbench_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 94.67, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:dtbench_external:gemini-3.8-flash_low", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_low", "level": "low", "settings": "close", "score": 94.93, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:dtbench_external:gemini-3.8-flash_medium", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_medium", "level": "medium", "settings": "mismatch", "score": 94.93, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:dtbench_external:gemini-3.8-flash_unknown", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_unknown", "level": "unknown", "settings": "exact", "score": 95.72, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:dtbench_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 68.27, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:dtbench_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 68.8, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:dtbench_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 54.4, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:dtbench_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 80.53, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:dtbench_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 94.4, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:dtbench_external:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 80, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:dtbench_external:gpt-5.4-nano-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 80.27, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:dtbench_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 96, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:dtbench_external:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 96.53, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:dtbench_external:gpt-6-astra_low", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_low", "level": "low", "settings": "close", "score": 94.93, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:dtbench_external:gpt-6-astra_medium", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_medium", "level": "medium", "settings": "mismatch", "score": 97.07, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:dtbench_external:gpt-6-astra_unknown", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_unknown", "level": "unknown", "settings": "exact", "score": 97.07, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:dtbench_external:gpt-6-astra_xhigh", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_xhigh", "level": "xhigh", "settings": "mismatch", "score": 97.33, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:dtbench_external:gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 76.27, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:dtbench_external:mistral-large-2402", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 55.9, "unit": "percent", "own_run": false, "date": "2024-02-26" }, { "id": "epoch:dtbench_external:mistral-large-2407", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 61.21, "unit": "percent", "own_run": false, "date": "2024-07-24" }, { "id": "epoch:dtbench_external:mistral-large-2411", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 60.82, "unit": "percent", "own_run": false, "date": "2024-11-18" }, { "id": "epoch:dtbench_external:mistral-large-2512", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-large-2512", "source_model": "mistral-large-2512", "settings": "exact", "score": 65.07, "unit": "percent", "own_run": false, "date": "2025-12-02" }, { "id": "epoch:dtbench_external:mistral-medium-2312", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2312", "settings": "exact", "score": 53.3, "unit": "percent", "own_run": false, "date": "2023-12-11" }, { "id": "epoch:dtbench_external:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 62.27, "unit": "percent", "own_run": false, "date": "2025-05-07" }, { "id": "epoch:dtbench_external:mistral-medium-2508", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 67.47, "unit": "percent", "own_run": false, "date": "2025-08-12" }, { "id": "epoch:dtbench_external:mistral-medium-2604", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2604", "settings": "exact", "score": 75.47, "unit": "percent", "own_run": false, "date": "2026-04-28" }, { "id": "epoch:dtbench_external:mistral-small-2402", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-small-2603", "source_model": "mistral-small-2402", "settings": "exact", "score": 52.96, "unit": "percent", "own_run": false, "date": "2024-02-26" }, { "id": "epoch:dtbench_external:mistral-small-2409", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-small-2603", "source_model": "mistral-small-2409", "settings": "exact", "score": 49.53, "unit": "percent", "own_run": false, "date": "2024-09-17" }, { "id": "epoch:dtbench_external:mistral-small-2503", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-small-2603", "source_model": "mistral-small-2503", "settings": "exact", "score": 58.64, "unit": "percent", "own_run": false, "date": "2025-03-17" }, { "id": "epoch:dtbench_external:mistral-small-2506", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 59.87, "unit": "percent", "own_run": false, "date": "2025-06-20" }, { "id": "epoch:dtbench_external:mistral-small-2603", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "mistral-small-2603", "source_model": "mistral-small-2603", "settings": "exact", "score": 70.93, "unit": "percent", "own_run": false, "date": "2026-03-16" }, { "id": "epoch:dtbench_external:qwen3.5-9B", "source": "epoch", "benchmark": "epoch:dtbench_external", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B", "settings": "exact", "score": 71.2, "unit": "percent", "own_run": false, "date": "2026-02-24" }, { "id": "epoch:ebr_bench:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 57.14, "se": 6.023386019368341, "unit": "percent", "own_run": true, "date": "2026-09-08" }, { "id": "epoch:ebr_bench:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 28.57, "se": 1.4197256999998662, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:ebr_bench:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 71.43, "se": 7.063046178186346, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:ebr_bench:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 45.71, "se": 3.5634832254989925, "unit": "percent", "own_run": true, "date": "2026-09-05" }, { "id": "epoch:ebr_bench:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 14.29, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "epoch:ebr_bench:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 4.76, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "epoch:ebr_bench:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 25.4, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "epoch:ebr_bench:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 34.29, "se": 1.9825390471106021, "unit": "percent", "own_run": true, "date": "2026-07-27" }, { "id": "epoch:ebr_bench:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:ebr_bench", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 76.19, "se": 8.384198505551907, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:exploitbench_external:claude-haiku-4-5-20251001_unknown", "source": "epoch", "benchmark": "epoch:exploitbench_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_unknown", "level": "unknown", "settings": "exact", "score": 13.4, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:exploitbench_external:claude-sonnet-4-6", "source": "epoch", "benchmark": "epoch:exploitbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 22.7, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:exploitbench_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:exploitbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 15.7, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:exploitbench_external:gpt-5.5_unknown", "source": "epoch", "benchmark": "epoch:exploitbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_unknown", "level": "unknown", "settings": "exact", "score": 32.6, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:fictionlivebench_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:fictionlivebench_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 33.3, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:fictionlivebench_external:chutes/gpt-oss-120b", "source": "epoch", "benchmark": "epoch:fictionlivebench_external", "model": "gpt-oss-120b", "source_model": "chutes/gpt-oss-120b", "settings": "exact", "score": 36.1, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:fictionlivebench_external:chutes/gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:fictionlivebench_external", "model": "gpt-oss-120b", "source_model": "chutes/gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 44.4, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:fictionlivebench_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:fictionlivebench_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 63.9, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:fictionlivebench_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:fictionlivebench_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 44.4, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:fictionlivebench_external:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:fictionlivebench_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 69.4, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:frontiercode_external:claude-fable-5-1_medium", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_medium", "level": "medium", "settings": "mismatch", "score": 50.91, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:frontiercode_external:claude-opus-4-8_unknown", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_unknown", "level": "unknown", "settings": "exact", "score": 46.5, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:frontiercode_external:claude-opus-5-5_medium", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_medium", "level": "medium", "settings": "mismatch", "score": 54.64, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:frontiercode_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 53.38, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:frontiercode_external:claude-sonnet-4-6_unknown", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_unknown", "level": "unknown", "settings": "exact", "score": 24.31, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:frontiercode_external:claude-sonnet-5_unknown", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_unknown", "level": "unknown", "settings": "exact", "score": 42.73, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:frontiercode_external:deepseek-v4-flash-0731_unknown", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_unknown", "level": "unknown", "settings": "exact", "score": 18.8, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:frontiercode_external:deepseek-v4-pro-0813_high", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_high", "level": "high", "settings": "mismatch", "score": 28.55, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:frontiercode_external:deepseek-v4-pro_none", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_none", "level": "none", "settings": "exact", "score": 17.64, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:frontiercode_external:gemini-3.8-flash_medium", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_medium", "level": "medium", "settings": "mismatch", "score": 41.19, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:frontiercode_external:gpt-5.4-mini-2026-03-17_unknown", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_unknown", "level": "unknown", "settings": "exact", "score": 27.04, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:frontiercode_external:gpt-5.5_unknown", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "gpt-5.5", "source_model": "gpt-5.5_unknown", "level": "unknown", "settings": "exact", "score": 42.96, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:frontiercode_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 53.26, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:frontiercode_external:mistral-medium-2604", "source": "epoch", "benchmark": "epoch:frontiercode_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2604", "settings": "exact", "score": 8, "unit": "percent", "own_run": false, "date": "2026-04-28" }, { "id": "epoch:frontiermath:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 4.14, "se": 1.1715642254112493, "unit": "percent", "own_run": true, "date": "2025-10-16" }, { "id": "epoch:frontiermath:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 5.9, "se": 1.391155477279583, "unit": "percent", "own_run": true, "date": "2025-10-22" }, { "id": "epoch:frontiermath:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 47.24, "se": 2.9366965952163047, "unit": "percent", "own_run": true, "date": "2026-06-08" }, { "id": "epoch:frontiermath:claude-sonnet-4-6_16K", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_16K", "level": "thinking", "settings": "mismatch", "score": 32.4, "se": 2.812163604063986, "unit": "percent", "own_run": true, "date": "2026-02-20" }, { "id": "epoch:frontiermath:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 36.9, "se": 2.8000000000000003, "unit": "percent", "own_run": true, "date": "2026-02-19" }, { "id": "epoch:frontiermath:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 38.97, "se": 2.868659061400048, "unit": "percent", "own_run": true, "date": "2026-05-22" }, { "id": "epoch:frontiermath:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 5.52, "se": 1.3430381628597847, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:frontiermath:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 4.48, "se": 1.217207560947464, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:frontiermath:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 27.24, "se": 2.6188332965202883, "unit": "percent", "own_run": true, "date": "2025-11-13" }, { "id": "epoch:frontiermath:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 20.34, "se": 2.368018421083109, "unit": "percent", "own_run": true, "date": "2025-11-13" }, { "id": "epoch:frontiermath:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 47.6, "se": 2.9000000000000004, "unit": "percent", "own_run": true, "date": "2026-03-06" }, { "id": "epoch:frontiermath:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 28.28, "se": 2.0650000000000004, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:frontiermath:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 25.86, "se": 2.58, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:frontiermath:mistral-large-2411", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 0.34, "se": 0.3448275862068965, "unit": "percent", "own_run": true, "date": "2025-03-06" }, { "id": "epoch:frontiermath:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:frontiermath", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 0.35, "se": 0.3460207612456747, "unit": "percent", "own_run": true, "date": "2025-05-07" }, { "id": "epoch:frontiermath_tier_4:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 2.08, "se": 2.083333333333333, "unit": "percent", "own_run": true, "date": "2025-10-22" }, { "id": "epoch:frontiermath_tier_4:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 31.25, "se": 6.761023307918171, "unit": "percent", "own_run": true, "date": "2026-06-08" }, { "id": "epoch:frontiermath_tier_4:claude-sonnet-4-6_16K", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_16K", "level": "thinking", "settings": "mismatch", "score": 8.3, "se": 4, "unit": "percent", "own_run": true, "date": "2026-02-21" }, { "id": "epoch:frontiermath_tier_4:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 16.7, "se": 5.4, "unit": "percent", "own_run": true, "date": "2026-02-19" }, { "id": "epoch:frontiermath_tier_4:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 14.58, "se": 5.148145136372291, "unit": "percent", "own_run": true, "date": "2026-05-25" }, { "id": "epoch:frontiermath_tier_4:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 0, "unit": "percent", "own_run": true, "date": "2025-07-01" }, { "id": "epoch:frontiermath_tier_4:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 6.25, "se": 3.530829267912701, "unit": "percent", "own_run": true, "date": "2025-10-30" }, { "id": "epoch:frontiermath_tier_4:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 4.17, "se": 2.914766351555603, "unit": "percent", "own_run": true, "date": "2025-08-07" }, { "id": "epoch:frontiermath_tier_4:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 27.1, "se": 6.4, "unit": "percent", "own_run": true, "date": "2026-03-06" }, { "id": "epoch:frontiermath_tier_4:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 2.08, "se": 2.08, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:frontiermath_tier_4:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 6.25, "se": 3.53, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:frontiermath_tier_4_v2:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 87.8, "se": 5.173952057462543, "unit": "percent", "own_run": true, "date": "2026-09-01" }, { "id": "epoch:frontiermath_tier_4_v2:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 56.1, "se": 7.846686801046543, "unit": "percent", "own_run": true, "date": "2026-06-10" }, { "id": "epoch:frontiermath_tier_4_v2:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 95, "se": 4.117547077105885, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:frontiermath_tier_4_v2:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 73.17, "se": 7.005564203095157, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:frontiermath_tier_4_v2:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 29.27, "se": 7.194088392074452, "unit": "percent", "own_run": true, "date": "2026-06-30" }, { "id": "epoch:frontiermath_tier_4_v2:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 24.39, "se": 6.789956540036613, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:frontiermath_tier_4_v2:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 26.83, "se": 7.005564203095157, "unit": "percent", "own_run": true, "date": "2026-08-19" }, { "id": "epoch:frontiermath_tier_4_v2:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 2.44, "se": 2.4390243902439024, "unit": "percent", "own_run": true, "date": "2026-06-17" }, { "id": "epoch:frontiermath_tier_4_v2:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 26.83, "se": 7.005564203095157, "unit": "percent", "own_run": true, "date": "2026-06-11" }, { "id": "epoch:frontiermath_tier_4_v2:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 0, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:frontiermath_tier_4_v2:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 26.83, "se": 7.005564203095157, "unit": "percent", "own_run": true, "date": "2026-06-10" }, { "id": "epoch:frontiermath_tier_4_v2:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 21.95, "se": 6.5445892024384085, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 12.2, "se": 5.173952057462543, "unit": "percent", "own_run": true, "date": "2026-06-12" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 49, "se": 8, "unit": "percent", "own_run": true, "date": "2026-06-11" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 9.76, "se": 4.691557088212523, "unit": "percent", "own_run": true, "date": "2026-06-12" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 12.2, "se": 5.173952057462543, "unit": "percent", "own_run": true, "date": "2026-06-12" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 72.5, "se": 7.149950690165275, "unit": "percent", "own_run": true, "date": "2026-06-11" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 97.6, "se": 2.4, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-6-astra_low", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_low", "level": "low", "settings": "close", "score": 87.8, "se": 5.173952057462543, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 97.6, "se": 2.4, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-6-astra_medium", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_medium", "level": "medium", "settings": "mismatch", "score": 97.56, "se": 2.4390243902439024, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-6-astra_none", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_none", "level": "none", "settings": "exact", "score": 82.93, "se": 5.949419959829497, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontiermath_tier_4_v2:gpt-6-astra_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tier_4_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_xhigh", "level": "xhigh", "settings": "mismatch", "score": 97.6, "se": 2.4, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontiermath_tiers_1_3_v2:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 90.18, "se": 1.7662067789031757, "unit": "percent", "own_run": true, "date": "2026-09-01" }, { "id": "epoch:frontiermath_tiers_1_3_v2:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 80, "se": 2.3735633163877066, "unit": "percent", "own_run": true, "date": "2026-06-10" }, { "id": "epoch:frontiermath_tiers_1_3_v2:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 91.23, "se": 1.6786209858442096, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:frontiermath_tiers_1_3_v2:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 85.61, "se": 2.082489457534394, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:frontiermath_tiers_1_3_v2:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 65.61, "se": 2.818576398907258, "unit": "percent", "own_run": true, "date": "2026-06-30" }, { "id": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 57.54, "se": 2.9329899789943816, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 64.56, "se": 2.8383475205435627, "unit": "percent", "own_run": true, "date": "2026-08-19" }, { "id": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 45.26, "se": 2.953609826992209, "unit": "percent", "own_run": true, "date": "2026-06-17" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 59.65, "se": 2.911181956525014, "unit": "percent", "own_run": true, "date": "2026-06-11" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 25.96, "se": 2.601675082238553, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 62.81, "se": 2.867975375107462, "unit": "percent", "own_run": true, "date": "2026-06-10" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 68.42, "se": 2.7582557383453006, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 5.96, "se": 1.4053607746337793, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 6.67, "se": 1.480176785956286, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 0.7, "se": 0.49534089619636035, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 46.67, "se": 2.9603535719125715, "unit": "percent", "own_run": true, "date": "2026-06-12" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5-mini-2025-08-07_low", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_low", "level": "low", "settings": "close", "score": 18.25, "se": 2.2917941277334046, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 5.96, "se": 1.4053607746337795, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 78.6, "se": 2.4338000553262393, "unit": "percent", "own_run": true, "date": "2026-06-11" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-mini-2026-03-17_low", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_low", "level": "low", "settings": "close", "score": 24.56, "se": 2.55425481026507, "unit": "percent", "own_run": true, "date": "2026-08-28" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-mini-2026-03-17_none", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_none", "level": "none", "settings": "exact", "score": 17.19, "se": 2.2389789548206736, "unit": "percent", "own_run": true, "date": "2026-08-28" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 51.23, "se": 2.966059084324674, "unit": "percent", "own_run": true, "date": "2026-06-12" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 44.91, "se": 2.9515543245521676, "unit": "percent", "own_run": true, "date": "2026-06-12" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 20.35, "se": 2.389037106974561, "unit": "percent", "own_run": true, "date": "2026-08-28" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-nano-2026-03-17_none", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_none", "level": "none", "settings": "exact", "score": 4.56, "se": 1.2380896644748405, "unit": "percent", "own_run": true, "date": "2026-08-28" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 85.26, "se": 2.1034091168852487, "unit": "percent", "own_run": true, "date": "2026-06-11" }, { "id": "epoch:frontiermath_tiers_1_3_v2:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:frontiermath_tiers_1_3_v2", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 93.68, "se": 1.4434038295343787, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:frontierswe_external:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:frontierswe_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 56.29, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:frontierswe_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:frontierswe_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 62.33, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:frontierswe_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:frontierswe_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 52.01, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:frontierswe_external:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:frontierswe_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 19.63, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:frontierswe_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:frontierswe_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 65.51, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:frontierswe_external:grok-4.7_xhigh", "source": "epoch", "benchmark": "epoch:frontierswe_external", "model": "grok-4.7", "source_model": "grok-4.7_xhigh", "level": "xhigh", "settings": "close", "score": 29.5, "unit": "percent", "own_run": false, "date": "2026-09-21" }, { "id": "epoch:furniture_assembly:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 70, "se": 5.724365605193373, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 42.5, "se": 6.380695638890652, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 83.33, "se": 4.704047221664923, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:furniture_assembly:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 60.83, "se": 6.185876325418331, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 26.67, "se": 5.6331678510110805, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 31.67, "se": 5.93832076297605, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 37.5, "se": 6.246467928511352, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 44.17, "se": 6.410142328406845, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 80, "se": 4.928872620664329, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:furniture_assembly:grok-4.7_xhigh", "source": "epoch", "benchmark": "epoch:furniture_assembly", "model": "grok-4.7", "source_model": "grok-4.7_xhigh", "level": "xhigh", "settings": "close", "score": 20.83, "se": 5.219972869740667, "unit": "percent", "own_run": true, "date": "2026-09-24" }, { "id": "epoch:geobench_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:geobench_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 72, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:geobench_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:geobench_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 64, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:gpqa_diamond:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 47.44, "se": 2.8523027562061025, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:gpqa_diamond:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 60.48, "se": 2.815453502302083, "unit": "percent", "own_run": true, "date": "2025-10-16" }, { "id": "epoch:gpqa_diamond:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 71.21, "se": 3.225883512300997, "unit": "percent", "own_run": true, "date": "2025-10-22" }, { "id": "epoch:gpqa_diamond:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 88.38, "se": 2.2828881775249354, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 91.04, "se": 1.9157125794612027, "unit": "percent", "own_run": true, "date": "2026-06-07" }, { "id": "epoch:gpqa_diamond:claude-opus-4-8_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_none", "level": "none", "settings": "exact", "score": 85.35, "se": 2.5190921114603886, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:claude-opus-5", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-5", "source_model": "claude-opus-5", "settings": "exact", "score": 92.93, "se": 1.8263105420199484, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 90.59, "se": 1.9434397190192136, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:gpqa_diamond:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 87.88, "se": 2.325315795194205, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 93.88, "se": 1.4782015056089346, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:gpqa_diamond:claude-sonnet-4-6_32K", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_32K", "level": "thinking", "settings": "mismatch", "score": 87.37, "se": 2.0034282226331888, "unit": "percent", "own_run": true, "date": "2026-02-20" }, { "id": "epoch:gpqa_diamond:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 83.33, "se": 2.6552207828215257, "unit": "percent", "own_run": true, "date": "2026-07-13" }, { "id": "epoch:gpqa_diamond:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 78.79, "se": 2.9126522834586774, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:claude-sonnet-4-6_medium", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_medium", "level": "medium", "settings": "mismatch", "score": 83.33, "se": 2.6552207828215257, "unit": "percent", "own_run": true, "date": "2026-07-13" }, { "id": "epoch:gpqa_diamond:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 80.3, "se": 2.8335609732463336, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 90.53, "se": 1.7838321313410213, "unit": "percent", "own_run": true, "date": "2026-07-01" }, { "id": "epoch:gpqa_diamond:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 91.04, "se": 1.7655466909172535, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:gpqa_diamond:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 91.67, "se": 1.5242573629934337, "unit": "percent", "own_run": true, "date": "2026-08-18" }, { "id": "epoch:gpqa_diamond:deepseek-v4-pro_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_high", "level": "high", "settings": "mismatch", "score": 90.91, "se": 2.0482086775424224, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 89.65, "se": 1.7473014662966007, "unit": "percent", "own_run": true, "date": "2026-06-16" }, { "id": "epoch:gpqa_diamond:deepseek-v4-pro_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_none", "level": "none", "settings": "exact", "score": 73.23, "se": 3.1544498882702823, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 94.1, "se": 1.7000000000000002, "unit": "percent", "own_run": true, "date": "2026-02-20" }, { "id": "epoch:gpqa_diamond:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 94.44, "se": 1.6319950700767385, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 83.33, "se": 2.6552207828215257, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:gemini-3.5-flash-lite_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_low", "level": "low", "settings": "close", "score": 75.76, "se": 3.053289223393202, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:gemini-3.5-flash-lite_minimal", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_minimal", "level": "minimal", "settings": "close", "score": 74.24, "se": 3.1156269519646824, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 92.8, "se": 1.6387002835275906, "unit": "percent", "own_run": true, "date": "2026-05-22" }, { "id": "epoch:gpqa_diamond:gemini-3.5-flash_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_low", "level": "low", "settings": "close", "score": 88.89, "se": 2.239078763821682, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:gpqa_diamond:gemini-3.5-flash_minimal", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_minimal", "level": "minimal", "settings": "close", "score": 86.36, "se": 2.445015597318985, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:gpqa_diamond:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 95.39, "se": 1.401136318210652, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:gpqa_diamond:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 66.92, "se": 2.821440093855908, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:gpqa_diamond:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 65.85, "se": 2.7051500965295916, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:gpqa_diamond:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 37.72, "se": 2.4068925424620136, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:gpqa_diamond:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 75, "se": 2.2767295427113803, "unit": "percent", "own_run": true, "date": "2025-10-30" }, { "id": "epoch:gpqa_diamond:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 71.65, "se": 2.3192741878083396, "unit": "percent", "own_run": true, "date": "2025-08-07" }, { "id": "epoch:gpqa_diamond:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 71.72, "se": 3.2087795587867514, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:gpqa_diamond:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 89.9, "se": 2.1469735576055355, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:gpqa_diamond:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 84.85, "se": 2.554565042660364, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:gpqa_diamond:gpt-5.4-2026-03-05_medium", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_medium", "level": "medium", "settings": "mismatch", "score": 88.89, "se": 2.239078763821682, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:gpqa_diamond:gpt-5.4-2026-03-05_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_none", "level": "none", "settings": "exact", "score": 74.75, "se": 3.095405547036587, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:gpqa_diamond:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 93.3, "se": 1.7999999999999998, "unit": "percent", "own_run": true, "date": "2026-03-06" }, { "id": "epoch:gpqa_diamond:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 83.59, "se": 2.2274469189069883, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:gpqa_diamond:gpt-5.4-mini-2026-03-17_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_none", "level": "none", "settings": "exact", "score": 64.14, "se": 3.416903640391528, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:gpqa_diamond:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 86.87, "se": 2.406315641682249, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:gpqa_diamond:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 78.47, "se": 2.44, "unit": "percent", "own_run": true, "date": "2026-04-14" }, { "id": "epoch:gpqa_diamond:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 72.22, "se": 3.1911782267135482, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:gpqa_diamond:gpt-5.4-nano-2026-03-17_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_none", "level": "none", "settings": "exact", "score": 55.56, "se": 3.5402943770953677, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:gpqa_diamond:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 90.66, "se": 1.7532658377721708, "unit": "percent", "own_run": true, "date": "2026-05-05" }, { "id": "epoch:gpqa_diamond:gpt-5.5_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-5.5", "source_model": "gpt-5.5_none", "level": "none", "settings": "exact", "score": 77.27, "se": 2.9857515673386437, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:gpqa_diamond:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 95.77, "se": 1.3696812097020312, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:gpqa_diamond:mistral-large-2402", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 38.76, "se": 2.4776598535417285, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:gpqa_diamond:mistral-large-2407", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 49.02, "se": 2.591884098865697, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:gpqa_diamond:mistral-large-2411", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 51.33, "se": 2.6887818295509085, "unit": "percent", "own_run": true, "date": "2025-02-25" }, { "id": "epoch:gpqa_diamond:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 59.53, "se": 2.824697229654645, "unit": "percent", "own_run": true, "date": "2025-05-07" }, { "id": "epoch:gpqa_diamond:mistral-small-2501", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "mistral-small-2603", "source_model": "mistral-small-2501", "settings": "exact", "score": 45.3, "se": 2.4794688677057852, "unit": "percent", "own_run": true, "date": "2025-01-30" }, { "id": "epoch:gpqa_diamond:mistral-small-2503", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "mistral-small-2603", "source_model": "mistral-small-2503", "settings": "exact", "score": 47.47, "se": 2.761649514001224, "unit": "percent", "own_run": true, "date": "2025-03-18" }, { "id": "epoch:gpqa_diamond:openai/gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "gpt-oss-120b", "source_model": "openai/gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 75.76, "se": 2.7437163443883765, "unit": "percent", "own_run": true, "date": "2025-12-11" }, { "id": "epoch:gpqa_diamond:qwen3.5-9B", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B", "settings": "exact", "score": 78.98, "se": 2.365568030958479, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:gpqa_diamond:qwen3.5-9B_none", "source": "epoch", "benchmark": "epoch:gpqa_diamond", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B_none", "level": "none", "settings": "exact", "score": 78.91, "se": 2.388848562991196, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:gsm8k_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:gsm8k_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 91.3, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:gso_external:claude-opus-4-8_unknown", "source": "epoch", "benchmark": "epoch:gso_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_unknown", "level": "unknown", "settings": "exact", "score": 47.06, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:gso_external:claude-sonnet-5_unknown", "source": "epoch", "benchmark": "epoch:gso_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_unknown", "level": "unknown", "settings": "exact", "score": 37.25, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:gso_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:gso_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 22.55, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:gso_external:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:gso_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 25.49, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:gso_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:gso_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 31.37, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:gso_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:gso_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 40.2, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:hle_external:claude-fable-5-1_xhigh", "source": "epoch", "benchmark": "epoch:hle_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_xhigh", "level": "xhigh", "settings": "mismatch", "score": 46.5, "se": 2, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:hle_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:hle_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 46.44, "se": 1.96, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:hle_external:gemini-3.8-flash_unknown", "source": "epoch", "benchmark": "epoch:hle_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_unknown", "level": "unknown", "settings": "exact", "score": 44.52, "se": 1.96, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:hle_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:hle_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 5.4, "se": 0.89, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:hle_external:gpt-5-mini-2025-08-07_unknown", "source": "epoch", "benchmark": "epoch:hle_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_unknown", "level": "unknown", "settings": "exact", "score": 19.44, "se": 1.5500000000000003, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:hle_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:hle_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 36.24, "se": 1.8800000000000001, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:hle_external:gpt-6-astra_unknown", "source": "epoch", "benchmark": "epoch:hle_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_unknown", "level": "unknown", "settings": "exact", "score": 54.8, "se": 1.94, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:hle_external:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:hle_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 4.52, "se": 0.8100000000000002, "unit": "percent", "own_run": false, "date": "2025-05-07" }, { "id": "epoch:lech_mazur_writing_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:lech_mazur_writing_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 6.72, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:lech_mazur_writing_external:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:lech_mazur_writing_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 8.31, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:lech_mazur_writing_external:gpt-oss-120b", "source": "epoch", "benchmark": "epoch:lech_mazur_writing_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 7.71, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:lech_mazur_writing_external:mistral-large-2407", "source": "epoch", "benchmark": "epoch:lech_mazur_writing_external", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 6.9, "unit": "percent", "own_run": false, "date": "2024-07-24" }, { "id": "epoch:lech_mazur_writing_external:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:lech_mazur_writing_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 7.73, "unit": "percent", "own_run": false, "date": "2025-05-07" }, { "id": "epoch:lmca_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 17.52, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:lmca_external:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 65.09, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:lmca_external:claude-fable-5-1_low", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_low", "level": "low", "settings": "close", "score": 63.46, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:lmca_external:claude-fable-5-1_medium", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_medium", "level": "medium", "settings": "mismatch", "score": 65.32, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:lmca_external:claude-fable-5-1_unknown", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_unknown", "level": "unknown", "settings": "exact", "score": 65.46, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:lmca_external:claude-fable-5-1_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_xhigh", "level": "xhigh", "settings": "mismatch", "score": 65.06, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:lmca_external:claude-haiku-4-5-20251001_unknown", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_unknown", "level": "unknown", "settings": "exact", "score": 30.94, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:lmca_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 57.51, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:lmca_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 68.23, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:lmca_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 63.67, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:lmca_external:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 62.58, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:lmca_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 63.3, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:lmca_external:claude-opus-5_medium", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-5", "source_model": "claude-opus-5_medium", "level": "medium", "settings": "mismatch", "score": 62.93, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:lmca_external:claude-opus-5_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-opus-5", "source_model": "claude-opus-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 64.45, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:lmca_external:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 46.51, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:lmca_external:claude-sonnet-5_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_high", "level": "high", "settings": "mismatch", "score": 50.02, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:lmca_external:claude-sonnet-5_low", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_low", "level": "low", "settings": "close", "score": 49.65, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:lmca_external:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 49.3, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:lmca_external:claude-sonnet-5_medium", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_medium", "level": "medium", "settings": "mismatch", "score": 49.71, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:lmca_external:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 48.81, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:lmca_external:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 41.73, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:lmca_external:deepseek-v4-flash_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash_max", "level": "max", "settings": "mismatch", "score": 35.88, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:lmca_external:deepseek-v4-pro-0813_unknown", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_unknown", "level": "unknown", "settings": "exact", "score": 45.45, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:lmca_external:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 41.2, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:lmca_external:deepseek-v4.1-flash_unknown", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash_unknown", "level": "unknown", "settings": "exact", "score": 46.97, "unit": "percent", "own_run": false, "date": "2026-09-09" }, { "id": "epoch:lmca_external:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 53.82, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:lmca_external:gemini-3.1-pro-preview_low", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_low", "level": "low", "settings": "close", "score": 51.18, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:lmca_external:gemini-3.1-pro-preview_medium", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_medium", "level": "medium", "settings": "close", "score": 52.8, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:lmca_external:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 37.6, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:lmca_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 47.15, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:lmca_external:gemini-3.8-flash_low", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_low", "level": "low", "settings": "close", "score": 51.16, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:lmca_external:gemini-3.8-flash_medium", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_medium", "level": "medium", "settings": "mismatch", "score": 52.9, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:lmca_external:gemini-3.8-flash_unknown", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_unknown", "level": "unknown", "settings": "exact", "score": 51.55, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:lmca_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 25.61, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:lmca_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 21.12, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:lmca_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 10.37, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:lmca_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 34.2, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:lmca_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 51.97, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:lmca_external:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 40.79, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:lmca_external:gpt-5.4-nano-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 36.86, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:lmca_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 54.27, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:lmca_external:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 63.84, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:lmca_external:gpt-6-astra_low", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_low", "level": "low", "settings": "close", "score": 62.19, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:lmca_external:gpt-6-astra_medium", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_medium", "level": "medium", "settings": "mismatch", "score": 62.93, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:lmca_external:gpt-6-astra_unknown", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_unknown", "level": "unknown", "settings": "exact", "score": 64.11, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:lmca_external:gpt-6-astra_xhigh", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_xhigh", "level": "xhigh", "settings": "mismatch", "score": 64.37, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:lmca_external:gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 22.14, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:lmca_external:mistral-large-2512", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "mistral-large-2512", "source_model": "mistral-large-2512", "settings": "exact", "score": 16.7, "unit": "percent", "own_run": false, "date": "2025-12-02" }, { "id": "epoch:lmca_external:mistral-medium-2508", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 19.05, "unit": "percent", "own_run": false, "date": "2025-08-12" }, { "id": "epoch:lmca_external:mistral-medium-2604", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2604", "settings": "exact", "score": 26.06, "unit": "percent", "own_run": false, "date": "2026-04-28" }, { "id": "epoch:lmca_external:mistral-small-2603", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "mistral-small-2603", "source_model": "mistral-small-2603", "settings": "exact", "score": 20.61, "unit": "percent", "own_run": false, "date": "2026-03-16" }, { "id": "epoch:lmca_external:qwen3.5-9B", "source": "epoch", "benchmark": "epoch:lmca_external", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B", "settings": "exact", "score": 24.47, "unit": "percent", "own_run": false, "date": "2026-02-24" }, { "id": "epoch:math_level_5:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 41.6, "se": 1.1816670264886684, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:math_level_5:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 86.91, "se": 0.7747551687214772, "unit": "percent", "own_run": true, "date": "2025-10-16" }, { "id": "epoch:math_level_5:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 96.36, "se": 0.515811348923114, "unit": "percent", "own_run": true, "date": "2025-10-22" }, { "id": "epoch:math_level_5:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 83.01, "se": 0.8490146044318322, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:math_level_5:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 87.29, "se": 0.7298484758774334, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:math_level_5:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 52.63, "se": 1.1275275696871734, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:math_level_5:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 97.85, "se": 0.3369422414145465, "unit": "percent", "own_run": true, "date": "2025-10-30" }, { "id": "epoch:math_level_5:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 96.79, "se": 0.3853475747353661, "unit": "percent", "own_run": true, "date": "2025-08-20" }, { "id": "epoch:math_level_5:mistral-large-2402", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 24.46, "se": 0.892334786643176, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:math_level_5:mistral-large-2407", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 44.82, "se": 1.0703488686415494, "unit": "percent", "own_run": true, "date": "2025-01-27" }, { "id": "epoch:math_level_5:mistral-large-2411", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 50.28, "se": 1.1469451623480957, "unit": "percent", "own_run": true, "date": "2025-02-25" }, { "id": "epoch:math_level_5:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 81.63, "se": 0.8995651651009511, "unit": "percent", "own_run": true, "date": "2025-05-07" }, { "id": "epoch:math_level_5:mistral-small-2501", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "mistral-small-2603", "source_model": "mistral-small-2501", "settings": "exact", "score": 44.82, "se": 1.0854927233993668, "unit": "percent", "own_run": true, "date": "2025-01-30" }, { "id": "epoch:math_level_5:mistral-small-2503", "source": "epoch", "benchmark": "epoch:math_level_5", "model": "mistral-small-2603", "source_model": "mistral-small-2503", "settings": "exact", "score": 46.77, "se": 1.1995700377965737, "unit": "percent", "own_run": true, "date": "2025-03-18" }, { "id": "epoch:metr_time_horizons_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:metr_time_horizons_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 77.04, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:metr_time_horizons_external:gpt-5.4-2026-03-05_unknown", "source": "epoch", "benchmark": "epoch:metr_time_horizons_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_unknown", "level": "unknown", "settings": "exact", "score": 74.34, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:metr_time_horizons_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:metr_time_horizons_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 74.34, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:metr_time_horizons_external:gpt-oss-120b", "source": "epoch", "benchmark": "epoch:metr_time_horizons_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 56.63, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:mirrorcode:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:mirrorcode", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 73.3, "se": 9.17, "unit": "percent", "own_run": true, "date": "2026-09-10" }, { "id": "epoch:mirrorcode:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:mirrorcode", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 77.4, "se": 8.309999999999999, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:mirrorcode:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:mirrorcode", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 8.89, "se": 4.561939004190449, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:mirrorcode:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:mirrorcode", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 15.56, "se": 7.7094504410037406, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:mirrorcode:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:mirrorcode", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 10, "se": 6.042179781166438, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:mirrorcode:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:mirrorcode", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 46.7, "se": 11.86, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:mmlu_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:mmlu_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 86.3, "se": 0.29, "n": 14042, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:mmlu_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:mmlu_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 76.7, "se": 0.36, "n": 14042, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:mmlu_external:mistral-large-2402", "source": "epoch", "benchmark": "epoch:mmlu_external", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 68.8, "se": 0.39, "n": 14042, "unit": "percent", "own_run": false, "date": "2024-02-26" }, { "id": "epoch:mmlu_external:mistral-large-2407", "source": "epoch", "benchmark": "epoch:mmlu_external", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 80, "se": 0.34, "n": 14042, "unit": "percent", "own_run": false, "date": "2024-07-24" }, { "id": "epoch:mmlu_external:mistral-small-2402", "source": "epoch", "benchmark": "epoch:mmlu_external", "model": "mistral-small-2603", "source_model": "mistral-small-2402", "settings": "exact", "score": 68.7, "se": 0.39, "n": 14042, "unit": "percent", "own_run": false, "date": "2024-02-26" }, { "id": "epoch:mystery_game_puzzles:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 58, "se": 4.9604496374885825, "unit": "percent", "own_run": true, "date": "2026-09-01" }, { "id": "epoch:mystery_game_puzzles:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 36, "se": 4.824181513244218, "unit": "percent", "own_run": true, "date": "2026-07-25" }, { "id": "epoch:mystery_game_puzzles:claude-opus-4-8_xhigh", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_xhigh", "level": "xhigh", "settings": "mismatch", "score": 31, "se": 4.648231987117317, "unit": "percent", "own_run": true, "date": "2026-07-26" }, { "id": "epoch:mystery_game_puzzles:claude-opus-5", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-opus-5", "source_model": "claude-opus-5", "settings": "exact", "score": 37, "se": 4.8523658709390975, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:mystery_game_puzzles:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 71, "se": 4.560480215720688, "unit": "percent", "own_run": true, "date": "2026-09-29" }, { "id": "epoch:mystery_game_puzzles:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 59, "se": 4.943110704237103, "unit": "percent", "own_run": true, "date": "2026-07-25" }, { "id": "epoch:mystery_game_puzzles:claude-sonnet-4-6", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 14, "se": 3.4873508801977695, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:mystery_game_puzzles:claude-sonnet-4-6_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_low", "level": "low", "settings": "close", "score": 16, "se": 3.684529491774706, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:mystery_game_puzzles:claude-sonnet-5", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5", "settings": "exact", "score": 16, "se": 3.6845294917747067, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:mystery_game_puzzles:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 35, "se": 4.793724854411023, "unit": "percent", "own_run": true, "date": "2026-07-28" }, { "id": "epoch:mystery_game_puzzles:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 34, "se": 4.760952285695233, "unit": "percent", "own_run": true, "date": "2026-08-05" }, { "id": "epoch:mystery_game_puzzles:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 43, "se": 4.97569851956243, "unit": "percent", "own_run": true, "date": "2026-08-19" }, { "id": "epoch:mystery_game_puzzles:deepseek-v4-pro_none", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_none", "level": "none", "settings": "exact", "score": 17, "se": 3.7752516806863685, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 34, "se": 4.760952285695233, "unit": "percent", "own_run": true, "date": "2026-07-27" }, { "id": "epoch:mystery_game_puzzles:gemini-3.1-pro-preview_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_low", "level": "low", "settings": "close", "score": 29, "se": 4.560480215720688, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:mystery_game_puzzles:gemini-3.1-pro-preview_medium", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_medium", "level": "medium", "settings": "close", "score": 32, "se": 4.688261722621506, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:mystery_game_puzzles:gemini-3.5-flash-lite_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_low", "level": "low", "settings": "close", "score": 19, "se": 3.9427724440366254, "unit": "percent", "own_run": true, "date": "2026-08-05" }, { "id": "epoch:mystery_game_puzzles:gemini-3.5-flash-lite_minimal", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_minimal", "level": "minimal", "settings": "close", "score": 12, "se": 3.2659863237109037, "unit": "percent", "own_run": true, "date": "2026-08-05" }, { "id": "epoch:mystery_game_puzzles:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 32, "se": 4.688261722621506, "unit": "percent", "own_run": true, "date": "2026-07-27" }, { "id": "epoch:mystery_game_puzzles:gemini-3.5-flash_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_low", "level": "low", "settings": "close", "score": 28, "se": 4.51260859854213, "unit": "percent", "own_run": true, "date": "2026-08-05" }, { "id": "epoch:mystery_game_puzzles:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 47, "se": 5.016135580465917, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:mystery_game_puzzles:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 7, "se": 2.564323999762428, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 12, "se": 3.2659863237109037, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 5, "se": 2.1904291355759034, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 4, "se": 1.9694638556693238, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 10, "se": 3.015113445777636, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 17, "se": 3.7752516806863685, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-2026-03-05_medium", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_medium", "level": "medium", "settings": "mismatch", "score": 28, "se": 4.512608598542129, "unit": "percent", "own_run": true, "date": "2026-08-28" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-2026-03-05_none", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_none", "level": "none", "settings": "exact", "score": 16, "se": 3.684529491774706, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 37, "se": 4.8523658709390975, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-mini-2026-03-17_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_low", "level": "low", "settings": "close", "score": 8, "se": 2.7265992434429074, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-mini-2026-03-17_medium", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_medium", "level": "medium", "settings": "mismatch", "score": 7, "se": 2.564323999762428, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-mini-2026-03-17_none", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_none", "level": "none", "settings": "exact", "score": 11, "se": 3.1446603773522006, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 5, "se": 2.1904291355759034, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 3, "se": 1.7144660799776528, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-nano-2026-03-17_medium", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_medium", "level": "medium", "settings": "mismatch", "score": 6, "se": 2.3868325657594203, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.4-nano-2026-03-17_none", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_none", "level": "none", "settings": "exact", "score": 9, "se": 2.8762349126466145, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 52, "se": 5.021167315686783, "unit": "percent", "own_run": true, "date": "2026-07-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 28, "se": 4.51260859854213, "unit": "percent", "own_run": true, "date": "2026-08-28" }, { "id": "epoch:mystery_game_puzzles:gpt-5.5_none", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.5", "source_model": "gpt-5.5_none", "level": "none", "settings": "exact", "score": 18, "se": 3.8612291966536914, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 56, "se": 4.988876515698588, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:mystery_game_puzzles:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 84, "se": 3.6845294917747067, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:mystery_game_puzzles:gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 0, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:mystery_game_puzzles:gpt-oss-120b_medium", "source": "epoch", "benchmark": "epoch:mystery_game_puzzles", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_medium", "level": "medium", "settings": "mismatch", "score": 2, "se": 1.4070529413628967, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:os_world_external:claude-sonnet-4-6", "source": "epoch", "benchmark": "epoch:os_world_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 72.1, "se": 2.33, "n": 369, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:osworld_2_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 18.52, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:osworld_2_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 28.98, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:osworld_2_external:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 22.32, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:osworld_2_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 31.43, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:osworld_2_external:claude-opus-5_medium", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_medium", "level": "medium", "settings": "mismatch", "score": 25.33, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:osworld_2_external:claude-opus-5_xhigh", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-opus-5", "source_model": "claude-opus-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 30.21, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:osworld_2_external:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 8.3, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:osworld_2_external:claude-sonnet-4-6_medium", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_medium", "level": "medium", "settings": "mismatch", "score": 9.3, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:osworld_2_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:osworld_2_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 13, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:otis_mock_aime_2024_2025:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 5.14, "se": 2.3611111111111107, "unit": "percent", "own_run": true, "date": "2025-02-25" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 100, "unit": "percent", "own_run": true, "date": "2026-09-01" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 35.83, "se": 6.2386260142321825, "unit": "percent", "own_run": true, "date": "2025-10-16" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 66.67, "se": 7.106690545187014, "unit": "percent", "own_run": true, "date": "2025-10-22" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-4-8_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_low", "level": "low", "settings": "close", "score": 97.78, "se": 2.2222222222222223, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 98.33, "se": 1.4101901870444153, "unit": "percent", "own_run": true, "date": "2026-06-07" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-4-8_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_none", "level": "none", "settings": "exact", "score": 84.44, "se": 5.463890236888291, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-5", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-5", "source_model": "claude-opus-5", "settings": "exact", "score": 97.78, "se": 2.2222222222222223, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 100, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-5_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-5", "source_model": "claude-opus-5_low", "level": "low", "settings": "close", "score": 93.33, "se": 3.760507165451774, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 98.89, "se": 1.1111111111111112, "unit": "percent", "own_run": true, "date": "2026-07-24" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-sonnet-4-6_32K", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_32K", "level": "thinking", "settings": "mismatch", "score": 85.79, "se": 4.539927553202047, "unit": "percent", "own_run": true, "date": "2026-02-20" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 75.56, "se": 6.478835438717, "unit": "percent", "own_run": true, "date": "2026-07-13" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 71.11, "se": 6.832943242540508, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-sonnet-4-6_medium", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_medium", "level": "medium", "settings": "mismatch", "score": 82.22, "se": 5.763774795025092, "unit": "percent", "own_run": true, "date": "2026-07-13" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 80, "se": 6.030226891555273, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 94.72, "se": 2.2644349282618412, "unit": "percent", "own_run": true, "date": "2026-07-01" }, { "id": "epoch:otis_mock_aime_2024_2025:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 94.44, "se": 2.0854366486931895, "unit": "percent", "own_run": true, "date": "2026-08-02" }, { "id": "epoch:otis_mock_aime_2024_2025:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 98.61, "se": 0.990980650017605, "unit": "percent", "own_run": true, "date": "2026-08-18" }, { "id": "epoch:otis_mock_aime_2024_2025:deepseek-v4-pro_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_high", "level": "high", "settings": "mismatch", "score": 95.56, "se": 3.1067790907534745, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 96.67, "se": 2.002208376727653, "unit": "percent", "own_run": true, "date": "2026-06-17" }, { "id": "epoch:otis_mock_aime_2024_2025:deepseek-v4-pro_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_none", "level": "none", "settings": "exact", "score": 46.67, "se": 7.521014330903548, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 95.6, "se": 3.1, "unit": "percent", "own_run": true, "date": "2026-02-20" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 95.56, "se": 3.1067790907534736, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 71.11, "se": 6.83294324254051, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash-lite_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_low", "level": "low", "settings": "close", "score": 60, "se": 7.385489458759962, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash-lite_minimal", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_minimal", "level": "minimal", "settings": "close", "score": 51.11, "se": 7.535922203472521, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 95.56, "se": 2.6696111689702042, "unit": "percent", "own_run": true, "date": "2026-05-25" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_low", "level": "low", "settings": "close", "score": 88.89, "se": 4.737793696791343, "unit": "percent", "own_run": true, "date": "2026-08-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash_minimal", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_minimal", "level": "minimal", "settings": "close", "score": 80, "se": 6.030226891555273, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:otis_mock_aime_2024_2025:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 98.89, "se": 0.8724007409153889, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 38.33, "se": 6.324056203789373, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 44.72, "se": 6.148017573965347, "unit": "percent", "own_run": true, "date": "2025-04-14" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 6.94, "se": 2.73963714244758, "unit": "percent", "own_run": true, "date": "2025-07-30" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 86.67, "se": 4.286165199944133, "unit": "percent", "own_run": true, "date": "2025-10-30" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 78.33, "se": 4.790431329560243, "unit": "percent", "own_run": true, "date": "2025-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5-mini-2025-08-07_minimal", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_minimal", "level": "minimal", "settings": "close", "score": 55.56, "se": 7.4911095829249135, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 97.78, "se": 2.2222222222222223, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-2026-03-05_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_low", "level": "low", "settings": "close", "score": 84.44, "se": 5.463890236888291, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-2026-03-05_medium", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_medium", "level": "medium", "settings": "mismatch", "score": 95.56, "se": 3.1067790907534736, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-2026-03-05_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_none", "level": "none", "settings": "exact", "score": 57.78, "se": 7.446027270295804, "unit": "percent", "own_run": true, "date": "2026-07-15" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 95.3, "se": 3.2, "unit": "percent", "own_run": true, "date": "2026-03-06" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 87.22, "se": 4.250833416666831, "unit": "percent", "own_run": true, "date": "2026-04-15" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-mini-2026-03-17_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_none", "level": "none", "settings": "exact", "score": 26.67, "se": 6.666666666666668, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 88.89, "se": 4.737793696791343, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 87.78, "se": 3.94, "unit": "percent", "own_run": true, "date": "2026-04-14" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-nano-2026-03-17_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_low", "level": "low", "settings": "close", "score": 68.89, "se": 6.979205927323111, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.4-nano-2026-03-17_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_none", "level": "none", "settings": "exact", "score": 46.67, "se": 7.521014330903548, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.5_low", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.5", "source_model": "gpt-5.5_low", "level": "low", "settings": "close", "score": 84.44, "se": 5.463890236888291, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-5.5_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-5.5", "source_model": "gpt-5.5_none", "level": "none", "settings": "exact", "score": 57.78, "se": 7.446027270295804, "unit": "percent", "own_run": true, "date": "2026-08-07" }, { "id": "epoch:otis_mock_aime_2024_2025:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 100, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:otis_mock_aime_2024_2025:mistral-large-2402", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1.94, "se": 0.9489457497216071, "unit": "percent", "own_run": true, "date": "2025-02-25" }, { "id": "epoch:otis_mock_aime_2024_2025:mistral-large-2407", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 8.47, "se": 2.4742642797709364, "unit": "percent", "own_run": true, "date": "2025-02-25" }, { "id": "epoch:otis_mock_aime_2024_2025:mistral-large-2411", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 7.78, "se": 2.4662778950157507, "unit": "percent", "own_run": true, "date": "2025-02-25" }, { "id": "epoch:otis_mock_aime_2024_2025:mistral-medium-2505", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 32.22, "se": 5.965611798482264, "unit": "percent", "own_run": true, "date": "2025-05-07" }, { "id": "epoch:otis_mock_aime_2024_2025:mistral-small-2501", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "mistral-small-2603", "source_model": "mistral-small-2501", "settings": "exact", "score": 5.28, "se": 2.589584594604903, "unit": "percent", "own_run": true, "date": "2025-03-18" }, { "id": "epoch:otis_mock_aime_2024_2025:mistral-small-2503", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "mistral-small-2603", "source_model": "mistral-small-2503", "settings": "exact", "score": 5.83, "se": 2.26830377599695, "unit": "percent", "own_run": true, "date": "2025-03-18" }, { "id": "epoch:otis_mock_aime_2024_2025:openai/gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "gpt-oss-120b", "source_model": "openai/gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 88.89, "se": 4.463343655613273, "unit": "percent", "own_run": true, "date": "2025-12-11" }, { "id": "epoch:otis_mock_aime_2024_2025:qwen3.5-9B", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B", "settings": "exact", "score": 42.78, "se": 6.533819469304575, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:otis_mock_aime_2024_2025:qwen3.5-9B_none", "source": "epoch", "benchmark": "epoch:otis_mock_aime_2024_2025", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B_none", "level": "none", "settings": "exact", "score": 61.67, "se": 6.003996985174109, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:piqa_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:piqa_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 88.7, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:posttrainbench_external:claude-opus-4-8_high", "source": "epoch", "benchmark": "epoch:posttrainbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_high", "level": "high", "settings": "mismatch", "score": 33.84, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:posttrainbench_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:posttrainbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 32.9, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:posttrainbench_external:claude-opus-5_unknown", "source": "epoch", "benchmark": "epoch:posttrainbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_unknown", "level": "unknown", "settings": "exact", "score": 35.04, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:posttrainbench_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:posttrainbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 21.99, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:posttrainbench_external:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:posttrainbench_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 19, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:posttrainbench_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:posttrainbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 27.23, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:proofbench_external:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 100, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:proofbench_external:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 69, "se": 4.648, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:proofbench_external:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 100, "unit": "percent", "own_run": false, "date": "2026-09-22" }, { "id": "epoch:proofbench_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 99, "se": 4.163, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:proofbench_external:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 45, "se": 4.975, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:proofbench_external:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 77, "se": 4.761, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:proofbench_external:deepseek-v4-flash-0731_unknown", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_unknown", "level": "unknown", "settings": "exact", "score": 56, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:proofbench_external:deepseek-v4-pro-0813_unknown", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_unknown", "level": "unknown", "settings": "exact", "score": 50, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:proofbench_external:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 16, "se": 3, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:proofbench_external:deepseek-v4.1-flash_unknown", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash_unknown", "level": "unknown", "settings": "exact", "score": 54, "unit": "percent", "own_run": false, "date": "2026-09-09" }, { "id": "epoch:proofbench_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 26, "se": 4.386, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:proofbench_external:gemini-3.5-flash-lite", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 13, "se": 3.38, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:proofbench_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 31, "se": 4.56, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:proofbench_external:gemini-3.8-flash_unknown", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_unknown", "level": "unknown", "settings": "exact", "score": 48, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:proofbench_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 9, "se": 2.862, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:proofbench_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 56, "se": 4.964, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:proofbench_external:gpt-5.4-mini-2026-03-17_xhigh", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_xhigh", "level": "xhigh", "settings": "mismatch", "score": 21, "se": 4.073, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:proofbench_external:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 5, "se": 2.179, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:proofbench_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 50, "se": 5, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:proofbench_external:gpt-6-astra_unknown", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_unknown", "level": "unknown", "settings": "exact", "score": 99, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:proofbench_external:grok-4.7_unknown", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "grok-4.7", "source_model": "grok-4.7_unknown", "level": "unknown", "settings": "exact", "score": 34, "unit": "percent", "own_run": false, "date": "2026-09-21" }, { "id": "epoch:proofbench_external:mistral-medium-2604", "source": "epoch", "benchmark": "epoch:proofbench_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2604", "settings": "exact", "score": 9, "se": 3.015, "unit": "percent", "own_run": false, "date": "2026-04-28" }, { "id": "epoch:rli_external:claude-fable-5-1_unknown", "source": "epoch", "benchmark": "epoch:rli_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_unknown", "level": "unknown", "settings": "exact", "score": 17.92, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:rli_external:claude-opus-4-8_unknown", "source": "epoch", "benchmark": "epoch:rli_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_unknown", "level": "unknown", "settings": "exact", "score": 8.33, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:rli_external:gpt-5.5_unknown", "source": "epoch", "benchmark": "epoch:rli_external", "model": "gpt-5.5", "source_model": "gpt-5.5_unknown", "level": "unknown", "settings": "exact", "score": 6.25, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:rli_external:gpt-6-astra_unknown", "source": "epoch", "benchmark": "epoch:rli_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_unknown", "level": "unknown", "settings": "exact", "score": 20.83, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:simplebench_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 19.9, "se": 2.82, "n": 200, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:simplebench_external:claude-opus-4-8_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_unknown", "level": "unknown", "settings": "exact", "score": 64.8, "se": 3.38, "n": 200, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:simplebench_external:claude-opus-5_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "claude-opus-5", "source_model": "claude-opus-5_unknown", "level": "unknown", "settings": "exact", "score": 80.6, "se": 2.8, "n": 200, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:simplebench_external:claude-sonnet-5_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_unknown", "level": "unknown", "settings": "exact", "score": 60.6, "se": 3.46, "n": 200, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:simplebench_external:deepseek-v4-flash-0731_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_unknown", "level": "unknown", "settings": "exact", "score": 61.1, "se": 3.45, "n": 200, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:simplebench_external:deepseek-v4-flash_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash_unknown", "level": "unknown", "settings": "exact", "score": 46.3, "se": 3.53, "n": 200, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:simplebench_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 79.6, "se": 2.85, "n": 200, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:simplebench_external:gemini-3.5-flash_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_unknown", "level": "unknown", "settings": "exact", "score": 76.7, "se": 2.99, "n": 200, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:simplebench_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 27, "se": 3.14, "n": 200, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:simplebench_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 10.7, "se": 2.19, "n": 200, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:simplebench_external:gpt-5.5_unknown", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_unknown", "level": "unknown", "settings": "exact", "score": 69, "se": 3.27, "n": 200, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:simplebench_external:gpt-oss-120b", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 22.1, "se": 2.93, "n": 200, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:simplebench_external:mistral-large-2407", "source": "epoch", "benchmark": "epoch:simplebench_external", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 22.5, "se": 2.95, "n": 200, "unit": "percent", "own_run": false, "date": "2024-07-24" }, { "id": "epoch:simpleqa_verified:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 70.8, "se": 1.4385511563477433, "unit": "percent", "own_run": true, "date": "2026-09-01" }, { "id": "epoch:simpleqa_verified:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 13.2, "se": 1.0709373963527966, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:simpleqa_verified:claude-haiku-4-5-20251001_32K", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_32K", "level": "thinking", "settings": "close", "score": 12.6, "se": 1.0499249222407965, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:claude-opus-4-8_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_max", "level": "max", "settings": "mismatch", "score": 53, "se": 1.5790799515836722, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:claude-opus-5-5_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5_max", "level": "max", "settings": "mismatch", "score": 72.2, "se": 1.4174516461485274, "unit": "percent", "own_run": true, "date": "2026-09-22" }, { "id": "epoch:simpleqa_verified:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 59.9, "se": 1.550610974549837, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:simpleqa_verified:claude-sonnet-4-6_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_high", "level": "high", "settings": "mismatch", "score": 35.5, "se": 1.5139491543780594, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:simpleqa_verified:claude-sonnet-4-6_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_max", "level": "max", "settings": "mismatch", "score": 32.8, "se": 1.4853842487270312, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:claude-sonnet-5_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_max", "level": "max", "settings": "mismatch", "score": 33.7, "se": 1.495508791865346, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:claude-sonnet-5_xhigh", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 32.9, "se": 1.4865395385928355, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 33.63, "se": 1.495531772489948, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 52.91, "se": 1.5808394665706598, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 46.99, "se": 1.5806515332355318, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:gemini-3.1-pro-preview_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview_high", "level": "high", "settings": "close", "score": 73.5, "se": 1.3963164754810065, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:simpleqa_verified:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 66.2, "se": 1.4965960710224384, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 69.7, "se": 1.4539683710535192, "unit": "percent", "own_run": true, "date": "2026-09-02" }, { "id": "epoch:simpleqa_verified:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 31.1, "se": 1.4645596385722692, "unit": "percent", "own_run": true, "date": "2026-08-31" }, { "id": "epoch:simpleqa_verified:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 12.7, "se": 1.0534798620855645, "unit": "percent", "own_run": true, "date": "2026-08-31" }, { "id": "epoch:simpleqa_verified:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 8.3, "se": 0.8728527206074758, "unit": "percent", "own_run": true, "date": "2026-08-31" }, { "id": "epoch:simpleqa_verified:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 21.6, "se": 1.3019735539307762, "unit": "percent", "own_run": true, "date": "2026-08-10" }, { "id": "epoch:simpleqa_verified:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 45.1, "se": 1.5743152379585443, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 29.4, "se": 1.4414290540008265, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 11.7, "se": 1.0169287802713345, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 63, "se": 1.5275252316519465, "unit": "percent", "own_run": true, "date": "2026-08-27" }, { "id": "epoch:simpleqa_verified:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:simpleqa_verified", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 75.6, "se": 1.3588548437881387, "unit": "percent", "own_run": true, "date": "2026-08-30" }, { "id": "epoch:surface_evolver_bench_external:claude-opus-4-8_high", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_high", "level": "high", "settings": "mismatch", "score": 87.5, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:surface_evolver_bench_external:claude-opus-4-8_none", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_none", "level": "none", "settings": "exact", "score": 68.13, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:surface_evolver_bench_external:claude-sonnet-5_medium", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_medium", "level": "medium", "settings": "mismatch", "score": 60, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:surface_evolver_bench_external:deepseek-v4-pro_high", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_high", "level": "high", "settings": "mismatch", "score": 40, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:surface_evolver_bench_external:deepseek-v4.1-flash_high", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash_high", "level": "high", "settings": "mismatch", "score": 46.25, "unit": "percent", "own_run": false, "date": "2026-09-09" }, { "id": "epoch:surface_evolver_bench_external:gemini-3.5-flash_medium", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_medium", "level": "medium", "settings": "mismatch", "score": 58.13, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:surface_evolver_bench_external:gemini-3.8-flash_high", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash_high", "level": "high", "settings": "mismatch", "score": 76.88, "unit": "percent", "own_run": false, "date": "2026-09-02" }, { "id": "epoch:surface_evolver_bench_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 88.13, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:surface_evolver_bench_external:gpt-5.5_medium", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_medium", "level": "medium", "settings": "mismatch", "score": 81.25, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:surface_evolver_bench_external:gpt-oss-120b", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 25, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:surface_evolver_bench_external:mistral-medium-2604", "source": "epoch", "benchmark": "epoch:surface_evolver_bench_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2604", "settings": "exact", "score": 26.88, "unit": "percent", "own_run": false, "date": "2026-04-28" }, { "id": "epoch:swe_bench_verified:claude-sonnet-4-6", "source": "epoch", "benchmark": "epoch:swe_bench_verified", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 75.21, "se": 1.9648182701390322, "unit": "percent", "own_run": true, "date": "2026-02-21" }, { "id": "epoch:swe_bench_verified:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:swe_bench_verified", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 77.64, "se": 1.897828052158487, "unit": "percent", "own_run": true, "date": "2026-06-18" }, { "id": "epoch:swe_bench_verified:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:swe_bench_verified", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 79.34, "se": 1.8422420250424887, "unit": "percent", "own_run": true, "date": "2026-06-01" }, { "id": "epoch:swe_bench_verified:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:swe_bench_verified", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 48.54, "se": 2.2835863746433964, "unit": "percent", "own_run": true, "date": "2026-02-08" }, { "id": "epoch:swe_bench_verified:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:swe_bench_verified", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 64.67, "se": 2.1749601139671304, "unit": "percent", "own_run": true, "date": "2026-02-01" }, { "id": "epoch:swe_bench_verified:gpt-5.4-2026-03-05_high", "source": "epoch", "benchmark": "epoch:swe_bench_verified", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_high", "level": "high", "settings": "mismatch", "score": 76.86, "se": 1.9189407003691976, "unit": "percent", "own_run": true, "date": "2026-03-06" }, { "id": "epoch:terminalbench_external:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 13.9, "se": 2.7, "n": 89, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:terminalbench_external:claude-haiku-4-5-20251001_unknown", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_unknown", "level": "unknown", "settings": "exact", "score": 13.93, "se": 2.7329168928, "n": 89, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:terminalbench_external:claude-sonnet-4-6", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 53.4, "se": 2.8, "n": 89, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:terminalbench_external:claude-sonnet-4-6_unknown", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_unknown", "level": "unknown", "settings": "exact", "score": 53.43, "se": 2.8300280743, "n": 89, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:terminalbench_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 59.36, "se": 4.2009785316, "n": 89, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:terminalbench_external:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 22.2, "se": 2.6, "n": 89, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:terminalbench_external:gpt-5-mini-2025-08-07_unknown", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_unknown", "level": "unknown", "settings": "exact", "score": 22.25, "se": 2.6242804005, "n": 89, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:terminalbench_external:gpt-5.4-2026-03-05_unknown", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_unknown", "level": "unknown", "settings": "exact", "score": 81.8, "se": 2, "n": 89, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:terminalbench_external:gpt-5.5_unknown", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gpt-5.5", "source_model": "gpt-5.5_unknown", "level": "unknown", "settings": "exact", "score": 66.07, "se": 2.4915582762, "n": 89, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:terminalbench_external:gpt-oss-120b", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 14.2, "se": 2.3, "n": 89, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:terminalbench_external:gpt-oss-120b_unknown", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_unknown", "level": "unknown", "settings": "exact", "score": 14.16, "se": 2.2673522558, "n": 89, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:terminalbench_external:qwen3.5-9B", "source": "epoch", "benchmark": "epoch:terminalbench_external", "model": "qwen3.5-9b", "source_model": "qwen3.5-9B", "settings": "exact", "score": 9.21, "se": 2.3513565589, "n": 89, "unit": "percent", "own_run": false, "date": "2026-02-24" }, { "id": "epoch:vpct_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:vpct_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 34, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:vpct_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:vpct_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 39, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:vpct_external:gpt-5-mini-2025-08-07_medium", "source": "epoch", "benchmark": "epoch:vpct_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_medium", "level": "medium", "settings": "mismatch", "score": 40.2, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:weirdml_external:Llama-3.3-70B-Instruct", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "llama-3.3-70b-instruct-turbo", "source_model": "Llama-3.3-70B-Instruct", "settings": "exact", "score": 14.44, "se": 0.011523640365438143, "unit": "percent", "own_run": false, "date": "2024-12-06" }, { "id": "epoch:weirdml_external:claude-fable-5-1_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_high", "level": "high", "settings": "mismatch", "score": 92.3, "se": 0.001376337560603623, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:weirdml_external:claude-fable-5-1_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1_max", "level": "max", "settings": "mismatch", "score": 92.9, "se": 0.001280965433748779, "unit": "percent", "own_run": false, "date": "2026-09-01" }, { "id": "epoch:weirdml_external:claude-haiku-4-5-20251001", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 45.4, "se": 0.011362920132010245, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:weirdml_external:claude-haiku-4-5-20251001_16K", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001_16K", "level": "thinking", "settings": "close", "score": 44.15, "se": 0.010276696227397039, "unit": "percent", "own_run": false, "date": "2025-10-15" }, { "id": "epoch:weirdml_external:claude-opus-4-8_medium", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_medium", "level": "medium", "settings": "mismatch", "score": 76.04, "se": 0.01646215855604908, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:weirdml_external:claude-opus-4-8_none", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_none", "level": "none", "settings": "exact", "score": 70.45, "se": 0.013620999100200573, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:weirdml_external:claude-opus-4-8_xhigh", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8_xhigh", "level": "xhigh", "settings": "mismatch", "score": 82.89, "se": 0.009958824837467326, "unit": "percent", "own_run": false, "date": "2026-05-28" }, { "id": "epoch:weirdml_external:claude-opus-5", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-opus-5", "source_model": "claude-opus-5", "settings": "exact", "score": 86.34, "se": 0.01453204125517409, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:weirdml_external:claude-opus-5_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-opus-5", "source_model": "claude-opus-5_high", "level": "high", "settings": "mismatch", "score": 91.59, "se": 0.0017927171677920716, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:weirdml_external:claude-opus-5_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-opus-5", "source_model": "claude-opus-5_max", "level": "max", "settings": "mismatch", "score": 91.78, "se": 0.003098361045836742, "unit": "percent", "own_run": false, "date": "2026-07-24" }, { "id": "epoch:weirdml_external:claude-sonnet-4-6_medium", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6_medium", "level": "medium", "settings": "mismatch", "score": 66.07, "se": 0.029806185543827163, "unit": "percent", "own_run": false, "date": "2026-02-17" }, { "id": "epoch:weirdml_external:claude-sonnet-5_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5_high", "level": "high", "settings": "mismatch", "score": 68.78, "se": 0.017696329549524787, "unit": "percent", "own_run": false, "date": "2026-06-30" }, { "id": "epoch:weirdml_external:deepseek-v4-flash-0731_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_high", "level": "high", "settings": "mismatch", "score": 57.05, "se": 0.012292173470515064, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:weirdml_external:deepseek-v4-flash-0731_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731_max", "level": "max", "settings": "mismatch", "score": 62.96, "se": 0.01825242529226991, "unit": "percent", "own_run": false, "date": "2026-07-31" }, { "id": "epoch:weirdml_external:deepseek-v4-flash_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash_high", "level": "high", "settings": "mismatch", "score": 43.76, "se": 0.009118142469320882, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:weirdml_external:deepseek-v4-flash_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-flash", "source_model": "deepseek-v4-flash_max", "level": "max", "settings": "mismatch", "score": 45.63, "se": 0.005552935999080915, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:weirdml_external:deepseek-v4-pro-0813_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813_max", "level": "max", "settings": "mismatch", "score": 66.22, "se": 0.01638262150844637, "unit": "percent", "own_run": false, "date": "2026-08-13" }, { "id": "epoch:weirdml_external:deepseek-v4-pro_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_high", "level": "high", "settings": "mismatch", "score": 46.53, "se": 0.010879277129526622, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:weirdml_external:deepseek-v4-pro_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro_max", "level": "max", "settings": "mismatch", "score": 48.9, "se": 0.017538854527333746, "unit": "percent", "own_run": false, "date": "2026-04-24" }, { "id": "epoch:weirdml_external:gemini-3.1-pro-preview", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 72.07, "se": 0.017632957273620634, "unit": "percent", "own_run": false, "date": "2026-02-19" }, { "id": "epoch:weirdml_external:gemini-3.5-flash-lite_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite_high", "level": "high", "settings": "mismatch", "score": 39, "se": 0.017156164116084945, "unit": "percent", "own_run": false, "date": "2026-07-21" }, { "id": "epoch:weirdml_external:gemini-3.5-flash_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash_high", "level": "high", "settings": "mismatch", "score": 62.64, "se": 0.024091415851094065, "unit": "percent", "own_run": false, "date": "2026-05-19" }, { "id": "epoch:weirdml_external:gpt-4.1-2025-04-14", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 39.04, "se": 0.010299257007316829, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:weirdml_external:gpt-4.1-mini-2025-04-14", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 37.61, "se": 0.011099148435005509, "unit": "percent", "own_run": false, "date": "2025-04-14" }, { "id": "epoch:weirdml_external:gpt-4o-mini-2024-07-18", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 11.76, "se": 0.013958141397025036, "unit": "percent", "own_run": false, "date": "2024-07-18" }, { "id": "epoch:weirdml_external:gpt-5-mini-2025-08-07_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5-mini", "source_model": "gpt-5-mini-2025-08-07_high", "level": "high", "settings": "mismatch", "score": 52.67, "se": 0.014642503026075516, "unit": "percent", "own_run": false, "date": "2025-08-07" }, { "id": "epoch:weirdml_external:gpt-5.4-2026-03-05_none", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_none", "level": "none", "settings": "exact", "score": 57.44, "se": 0.01493713713906097, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:weirdml_external:gpt-5.4-2026-03-05_xhigh", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.4", "source_model": "gpt-5.4-2026-03-05_xhigh", "level": "xhigh", "settings": "mismatch", "score": 77.7, "se": 0.015646383228370683, "unit": "percent", "own_run": false, "date": "2026-03-05" }, { "id": "epoch:weirdml_external:gpt-5.4-mini-2026-03-17_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 60.3, "se": 0.016867371616663178, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:weirdml_external:gpt-5.4-mini-2026-03-17_none", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-2026-03-17_none", "level": "none", "settings": "exact", "score": 37.9, "se": 0.00993267029964524, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:weirdml_external:gpt-5.4-nano-2026-03-17_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_high", "level": "high", "settings": "mismatch", "score": 49.23, "se": 0.01592769681566432, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:weirdml_external:gpt-5.4-nano-2026-03-17_none", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-2026-03-17_none", "level": "none", "settings": "exact", "score": 37.96, "se": 0.011700544258476444, "unit": "percent", "own_run": false, "date": "2026-03-17" }, { "id": "epoch:weirdml_external:gpt-5.5_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.5", "source_model": "gpt-5.5_high", "level": "high", "settings": "mismatch", "score": 83.9, "se": 0.010857064112429189, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:weirdml_external:gpt-5.5_none", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.5", "source_model": "gpt-5.5_none", "level": "none", "settings": "exact", "score": 67.15, "se": 0.017253756008593506, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:weirdml_external:gpt-5.5_xhigh", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-5.5", "source_model": "gpt-5.5_xhigh", "level": "xhigh", "settings": "mismatch", "score": 84.91, "se": 0.01091535447188698, "unit": "percent", "own_run": false, "date": "2026-04-23" }, { "id": "epoch:weirdml_external:gpt-6-astra_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_high", "level": "high", "settings": "mismatch", "score": 92.87, "se": 0.001648119724718335, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:weirdml_external:gpt-6-astra_max", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_max", "level": "max", "settings": "mismatch", "score": 93.28, "se": 0.0015446475430902452, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:weirdml_external:gpt-6-astra_promax", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-6-astra", "source_model": "gpt-6-astra_promax", "settings": "exact", "score": 93.57, "se": 0.0026473963528335824, "unit": "percent", "own_run": false, "date": "2026-09-03" }, { "id": "epoch:weirdml_external:gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 48.17, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:weirdml_external:gpt-oss-120b_medium", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b_medium", "level": "medium", "settings": "mismatch", "score": 41.88, "se": 0.01318182235022303, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "epoch:weirdml_external:mistral-medium-2508", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 33.13, "se": 0.012017965528137121, "unit": "percent", "own_run": false, "date": "2025-08-12" }, { "id": "epoch:weirdml_external:mistral-medium-2604", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2604", "settings": "exact", "score": 43.71, "se": 0.010624798303565263, "unit": "percent", "own_run": false, "date": "2026-04-28" }, { "id": "epoch:weirdml_external:openai/gpt-oss-120b_high", "source": "epoch", "benchmark": "epoch:weirdml_external", "model": "gpt-oss-120b", "source_model": "openai/gpt-oss-120b_high", "level": "high", "settings": "mismatch", "score": 48.17, "se": 0.017207317818265386, "unit": "percent", "own_run": false, "date": "2025-08-05" }, { "id": "livebench:category/Agentic Coding:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 66.06, "se": 5.58, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 50.51, "se": 5.89, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 71.72, "se": 5.31, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 65.35, "se": 5.61, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 65.2, "se": 5.61, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 42.63, "se": 5.83, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 59.39, "se": 5.79, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 37.63, "se": 5.71, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 46.77, "se": 5.88, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 42.63, "se": 5.83, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 54.95, "se": 5.86, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 77.27, "se": 4.94, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 44.14, "se": 5.85, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 48.99, "se": 5.89, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 45.25, "se": 5.87, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 54.24, "se": 5.87, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 41.67, "se": 5.81, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 46.77, "se": 5.88, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 53.84, "se": 5.88, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 53.99, "se": 5.87, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 57.32, "se": 5.83, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Agentic Coding:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/Agentic Coding", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 53.99, "se": 5.87, "n": 72, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 86.38, "se": 3.17, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 81.83, "se": 3.56, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 89.25, "se": 2.86, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 89.25, "se": 2.86, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 81.45, "se": 3.59, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 79.27, "se": 3.75, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 80.68, "se": 3.65, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 69.23, "se": 4.27, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 74.98, "se": 4, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 69.99, "se": 4.24, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 77.16, "se": 3.88, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 80.04, "se": 3.7, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 76.45, "se": 3.92, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 78.18, "se": 3.82, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 76.07, "se": 3.94, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 72.49, "se": 4.13, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 71.62, "se": 4.17, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70.84, "se": 4.2, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 77.54, "se": 3.86, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 82.15, "se": 3.54, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 80.36, "se": 3.67, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Coding:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/Coding", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 77.16, "se": 3.88, "n": 117, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 80.28, "se": 3.25, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 66.03, "se": 3.87, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 80.31, "se": 3.25, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 79.8, "se": 3.28, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 74.55, "se": 3.56, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 77.95, "se": 3.39, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 71.74, "se": 3.68, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 68.02, "se": 3.81, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 79.33, "se": 3.31, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 74.54, "se": 3.56, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 79.24, "se": 3.31, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 79.25, "se": 3.31, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 78.54, "se": 3.35, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 64.86, "se": 3.9, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 53.25, "se": 4.07, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 54.01, "se": 4.07, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70.79, "se": 3.71, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 67.64, "se": 3.82, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 79.31, "se": 3.31, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 81.58, "se": 3.17, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 82.97, "se": 3.07, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Data Analysis:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/Data Analysis", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 76.88, "se": 3.44, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 72.99, "se": 3.14, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 72.03, "se": 3.17, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 65.74, "se": 3.36, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 67.05, "se": 3.32, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 63.77, "se": 3.4, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 63.22, "se": 3.41, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/IF", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 63.86, "se": 3.4, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/IF", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 63.14, "se": 3.41, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/IF", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 65.52, "se": 3.36, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/IF", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 62.35, "se": 3.43, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/IF", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 67.7, "se": 3.31, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/IF", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 70.03, "se": 3.24, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 79.1, "se": 2.88, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 75.6, "se": 3.04, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 67.24, "se": 3.32, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 81.41, "se": 2.75, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 59.8, "se": 3.47, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 67.2, "se": 3.32, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70.22, "se": 3.23, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70.73, "se": 3.22, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/IF", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 75.58, "se": 3.04, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/IF:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/IF", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 75.28, "se": 3.05, "n": 200, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 89.5, "se": 2.5, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 79.66, "se": 3.29, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 86.27, "se": 2.81, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 85.5, "se": 2.88, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 88.69, "se": 2.59, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 76.1, "se": 3.48, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Language", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 74.97, "se": 3.54, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/Language", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 70.12, "se": 3.74, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/Language", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 79.18, "se": 3.32, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/Language", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 78.13, "se": 3.38, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/Language", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 82.07, "se": 3.13, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/Language", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 81.19, "se": 3.19, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 85.38, "se": 2.89, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 84.58, "se": 2.95, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 71.82, "se": 3.67, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 87.79, "se": 2.67, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70.95, "se": 3.71, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 62.51, "se": 3.95, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 82.63, "se": 3.09, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 87.36, "se": 2.71, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/Language", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 89.43, "se": 2.51, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Language:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/Language", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 80.14, "se": 3.26, "n": 150, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 97.01, "se": 0.88, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 94.32, "se": 1.19, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 97.08, "se": 0.87, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 96.8, "se": 0.9, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 95.73, "se": 1.04, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 86.99, "se": 1.73, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 92.94, "se": 1.32, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 79.65, "se": 2.07, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 86.79, "se": 1.74, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 90.68, "se": 1.49, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 95.09, "se": 1.11, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 93.29, "se": 1.29, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 91.04, "se": 1.47, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 88.24, "se": 1.65, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 73.74, "se": 2.26, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 91.56, "se": 1.43, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 78.46, "se": 2.11, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 90.98, "se": 1.47, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 94.15, "se": 1.21, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 95.86, "se": 1.02, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 96.81, "se": 0.9, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Mathematics:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/Mathematics", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 95.68, "se": 1.04, "n": 379, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 91.69, "se": 1.94, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 89.19, "se": 2.18, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 92.15, "se": 1.89, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 90.65, "se": 2.05, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 91.21, "se": 1.99, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 84.77, "se": 2.53, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 88.69, "se": 2.23, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 70.58, "se": 3.21, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 86.63, "se": 2.39, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 82.69, "se": 2.66, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 85.84, "se": 2.45, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 86.69, "se": 2.39, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 84, "se": 2.58, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 82, "se": 2.7, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 60.19, "se": 3.44, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 89.29, "se": 2.18, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 71.32, "se": 3.18, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 81.1, "se": 2.75, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 88.12, "se": 2.28, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 89.65, "se": 2.14, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 92.65, "se": 1.84, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:category/Reasoning:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:category/Reasoning", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 82.65, "se": 2.66, "n": 202, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 99, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 99, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 99.01, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 76, "se": 4.27, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 97, "se": 1.71, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 95, "se": 2.18, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 99, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 97, "se": 1.71, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 97, "se": 1.71, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/AMPS_Hard:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/AMPS_Hard", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 98, "se": 1.4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 82.61, "se": 5.59, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 84.78, "se": 5.3, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 86.96, "se": 4.97, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 86.96, "se": 4.97, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 82.61, "se": 5.59, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 78.26, "se": 6.08, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 78.26, "se": 6.08, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 65.22, "se": 7.02, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 73.91, "se": 6.47, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 69.57, "se": 6.78, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 78.26, "se": 6.08, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 82.61, "se": 5.59, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 78.26, "se": 6.08, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 76.09, "se": 6.29, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 76.09, "se": 6.29, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 71.74, "se": 6.64, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 75.56, "se": 6.34, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 67.39, "se": 6.91, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 80.44, "se": 5.85, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 82.61, "se": 5.59, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 80.44, "se": 5.85, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_completion:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/code_completion", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 78.26, "se": 6.08, "n": 46, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 90.14, "se": 3.54, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 78.87, "se": 4.84, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 91.55, "se": 3.3, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 91.55, "se": 3.3, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 80.28, "se": 4.72, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 80.28, "se": 4.72, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 83.1, "se": 4.45, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 73.24, "se": 5.25, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 76.06, "se": 5.06, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 70.42, "se": 5.42, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 76.06, "se": 5.06, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 77.47, "se": 4.96, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 74.65, "se": 5.16, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 80.28, "se": 4.72, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 76.06, "se": 5.06, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 73.24, "se": 5.25, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 67.69, "se": 5.55, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 74.29, "se": 5.19, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 74.65, "se": 5.16, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 81.69, "se": 4.59, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 80.28, "se": 4.72, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/code_generation:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/code_generation", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 76.06, "se": 5.06, "n": 71, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 99.33, "se": 1.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 99.33, "se": 1.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 99.33, "se": 1.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 99.33, "se": 1.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 99.33, "se": 1.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 99.33, "se": 1.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/connections", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 95.5, "se": 2.93, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/connections", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 89.5, "se": 4.34, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/connections", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 97.33, "se": 2.28, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/connections", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/connections", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/connections", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 92, "se": 3.84, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98.98, "se": 1.42, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 94.39, "se": 3.25, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/connections", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/connections:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/connections", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 90.56, "se": 4.27, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 52.91, "se": 7.28, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 90.56, "se": 4.27, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 90.37, "se": 4.3, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 77.57, "se": 6.08, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 92.68, "se": 3.8, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 74.74, "se": 6.34, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 59.92, "se": 7.15, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 89.44, "se": 4.48, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 75.42, "se": 6.28, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 90.23, "se": 4.33, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 88.72, "se": 4.61, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 85.24, "se": 5.17, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 47.99, "se": 7.29, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 21.1, "se": 5.95, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 20.61, "se": 5.9, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 62.97, "se": 7.04, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 54.3, "se": 7.27, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 86.19, "se": 5.03, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 88.82, "se": 4.6, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 90.56, "se": 4.27, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/consecutive_events:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/consecutive_events", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 85.73, "se": 5.1, "n": 47, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 99, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 89, "se": 3.13, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 0.5, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 99, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 97, "se": 1.71, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 90, "se": 3, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 88, "se": 3.25, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 37, "se": 4.83, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 65, "se": 4.77, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 78, "se": 4.14, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 94, "se": 2.37, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 90, "se": 3, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 78, "se": 4.14, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 68, "se": 4.66, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 26, "se": 4.39, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 80, "se": 4, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 37, "se": 4.83, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 82.83, "se": 3.77, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 93, "se": 2.55, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 99, "se": 0.99, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 100, "se": 0.5, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/integrals_with_game:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/integrals_with_game", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 96, "se": 1.96, "n": 100, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 68.18, "se": 9.93, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 68.18, "se": 9.93, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 81.82, "se": 8.22, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 72.73, "se": 9.5, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 77.27, "se": 8.93, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 54.55, "se": 10.62, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 68.18, "se": 9.93, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 54.55, "se": 10.62, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 63.64, "se": 10.26, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 54.55, "se": 10.62, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 68.18, "se": 9.93, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 81.82, "se": 8.22, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 59.09, "se": 10.48, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 63.64, "se": 10.26, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 59.09, "se": 10.48, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 72.73, "se": 9.5, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 50, "se": 10.66, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 63.64, "se": 10.26, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 68.18, "se": 9.93, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 63.64, "se": 10.26, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 63.64, "se": 10.26, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/javascript:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/javascript", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 63.64, "se": 10.26, "n": 22, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 86, "se": 4.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 78, "se": 5.86, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 86, "se": 4.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 80, "se": 5.66, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 86, "se": 4.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 70, "se": 6.48, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 86, "se": 4.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 64, "se": 6.79, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 70, "se": 6.48, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 64, "se": 6.79, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 66, "se": 6.7, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 68, "se": 6.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 72, "se": 6.35, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 74, "se": 6.2, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 68, "se": 6.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 60, "se": 6.93, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 64, "se": 6.79, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 68, "se": 6.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 76, "se": 6.04, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 88, "se": 4.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/logic_with_navigation:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/logic_with_navigation", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 66, "se": 6.7, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 98.04, "se": 1.37, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 94.12, "se": 2.33, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 94.12, "se": 2.33, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 95.1, "se": 2.14, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 96.08, "se": 1.92, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 96.08, "se": 1.92, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 96.08, "se": 1.92, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 96.08, "se": 1.92, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 95.1, "se": 2.14, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 92.16, "se": 2.66, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 95.1, "se": 2.14, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 94.12, "se": 2.33, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 95.1, "se": 2.14, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 94.12, "se": 2.33, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 96.08, "se": 1.92, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/math_comp:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/math_comp", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 97.06, "se": 1.67, "n": 102, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 92.97, "se": 2.91, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 92.23, "se": 3.05, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 93.25, "se": 2.86, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 92.15, "se": 3.06, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 92.8, "se": 2.95, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 87.86, "se": 3.72, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 90.67, "se": 3.31, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 86.53, "se": 3.89, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 89.08, "se": 3.55, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 90.62, "se": 3.32, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 91.29, "se": 3.21, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 89.08, "se": 3.55, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 92.1, "se": 3.07, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 91.88, "se": 3.11, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 81.8, "se": 4.4, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 92.16, "se": 3.06, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 85.73, "se": 3.99, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 87.98, "se": 3.71, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 91.47, "se": 3.18, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 91.35, "se": 3.2, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 92.17, "se": 3.06, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/olympiad:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/olympiad", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 91.65, "se": 3.15, "n": 77, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 71.52, "se": 6.38, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 67.87, "se": 6.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 64.27, "se": 6.78, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 64.68, "se": 6.76, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 65.73, "se": 6.71, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 59.4, "se": 6.94, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 61.25, "se": 6.89, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 62.38, "se": 6.85, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 61.7, "se": 6.87, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 56.22, "se": 7.02, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 68.63, "se": 6.56, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 70.73, "se": 6.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 79.87, "se": 5.67, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 75.52, "se": 6.08, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 65, "se": 6.75, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 80, "se": 5.66, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 58.88, "se": 6.96, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 66.65, "se": 6.67, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 63.43, "se": 6.81, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 71.73, "se": 6.37, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 78.25, "se": 5.83, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/paraphrase:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/paraphrase", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 78.05, "se": 5.85, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 75.17, "se": 6.11, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 61.65, "se": 6.88, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 77.48, "se": 5.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 77.16, "se": 5.94, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 74.73, "se": 6.15, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 56.98, "se": 7, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 53.41, "se": 7.05, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 46.87, "se": 7.06, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 58.19, "se": 6.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 56.39, "se": 7.01, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 64.22, "se": 6.78, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 61.57, "se": 6.88, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 74.13, "se": 6.19, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 67.75, "se": 6.61, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 49.46, "se": 7.07, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 73.38, "se": 6.25, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 43.88, "se": 7.02, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 39.13, "se": 6.9, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 65.9, "se": 6.7, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 74.09, "se": 6.2, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 86.29, "se": 4.86, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/plot_unscrambling:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/plot_unscrambling", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 72.42, "se": 6.32, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 70, "se": 10.25, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 70, "se": 10.25, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 70, "se": 10.25, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 75, "se": 9.68, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/python", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 60, "se": 10.95, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/python", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 35, "se": 10.67, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/python", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/python", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/python", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/python", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 90, "se": 6.71, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/python", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/python", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/python", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/python", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/python", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 55, "se": 11.12, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/python", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/python", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 50, "se": 11.18, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/python", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 55, "se": 11.12, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/python", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 65, "se": 10.67, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/python:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/python", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 55, "se": 11.12, "n": 20, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 70.17, "se": 6.47, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 62, "se": 6.86, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 58.67, "se": 6.96, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 63, "se": 6.83, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 61.58, "se": 6.88, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 62.93, "se": 6.83, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 60.65, "se": 6.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 54.05, "se": 7.05, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 58.37, "se": 6.97, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 55.93, "se": 7.02, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 61.57, "se": 6.88, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 67.07, "se": 6.65, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 71.05, "se": 6.41, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 69.1, "se": 6.53, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 68.02, "se": 6.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 75.4, "se": 6.09, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 54.07, "se": 7.05, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 62.75, "se": 6.84, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70, "se": 6.48, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 66.32, "se": 6.68, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 70.53, "se": 6.45, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/simplify:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/simplify", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 69.97, "se": 6.48, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 96, "se": 2.77, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 90, "se": 4.24, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 94, "se": 3.36, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 96, "se": 2.77, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 96, "se": 2.77, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 90, "se": 4.24, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 88, "se": 4.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/spatial:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/spatial", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 94, "se": 3.36, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 78.1, "se": 5.85, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 80.6, "se": 5.59, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 70.92, "se": 6.42, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 71.18, "se": 6.41, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 61.32, "se": 6.89, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 68.9, "se": 6.55, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 62.32, "se": 6.85, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 67.27, "se": 6.64, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 70.5, "se": 6.45, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 67.75, "se": 6.61, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 74.93, "se": 6.13, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 72.68, "se": 6.3, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 77.25, "se": 5.93, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 75.2, "se": 6.11, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 67.47, "se": 6.63, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 82.75, "se": 5.34, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 66.63, "se": 6.67, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 67.65, "se": 6.62, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 79.02, "se": 5.76, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 72.55, "se": 6.31, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 73.98, "se": 6.2, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/story_generation:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/story_generation", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 74.92, "se": 6.13, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 72.17, "se": 6.34, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 77.67, "se": 5.89, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 69.1, "se": 6.53, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 69.33, "se": 6.52, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 66.43, "se": 6.68, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 61.65, "se": 6.88, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 71.22, "se": 6.4, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 68.85, "se": 6.55, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 71.5, "se": 6.38, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 69.5, "se": 6.51, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 65.67, "se": 6.71, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 69.65, "se": 6.5, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 88.23, "se": 4.56, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 82.58, "se": 5.36, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 68.47, "se": 6.57, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 87.5, "se": 4.68, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 59.63, "se": 6.94, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 71.77, "se": 6.37, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 68.42, "se": 6.57, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 72.32, "se": 6.33, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 79.55, "se": 5.7, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/summarize:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/summarize", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 78.17, "se": 5.84, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 56.15, "se": 6.88, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 51.08, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 52.33, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 51, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 51.96, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 45.08, "se": 6.9, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 42.44, "se": 6.85, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 46.12, "se": 6.91, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 48.54, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 48.19, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 49.46, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 49.04, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 52.35, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 50.5, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 40.62, "se": 6.81, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 45.33, "se": 6.9, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 49.39, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 50.59, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 51.75, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 55.9, "se": 6.89, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 58.37, "se": 6.84, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablejoin:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/tablejoin", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 44.92, "se": 6.9, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 94.12, "se": 3.29, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 94.12, "se": 3.29, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 94.12, "se": 3.29, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 96.08, "se": 2.72, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 96.08, "se": 2.72, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 96.08, "se": 2.72, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98.04, "se": 1.94, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/tablereformat:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/tablereformat", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 100, "se": 0.98, "n": 51, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 84.62, "se": 5, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 84.62, "se": 5, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 78.85, "se": 5.66, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 73.08, "se": 6.15, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 73.08, "se": 6.15, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 86.54, "se": 4.73, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 84.62, "se": 5, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 50, "se": 6.93, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 76.92, "se": 5.84, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 76.92, "se": 5.84, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 80.77, "se": 5.47, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 88.46, "se": 4.43, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 84.62, "se": 5, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 84.62, "se": 5, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/theory_of_mind:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/theory_of_mind", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 84.62, "se": 5, "n": 52, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 60, "se": 8.94, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 33.33, "se": 8.61, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 63.33, "se": 8.8, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 53.33, "se": 9.11, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 43.33, "se": 9.05, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 23.33, "se": 7.72, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 50, "se": 9.13, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 23.33, "se": 7.72, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 26.67, "se": 8.07, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 23.33, "se": 7.72, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 46.67, "se": 9.11, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 60, "se": 8.94, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 23.33, "se": 7.72, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 33.33, "se": 8.61, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 26.67, "se": 8.07, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 40, "se": 8.94, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 20, "se": 7.3, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 26.67, "se": 8.07, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 43.33, "se": 9.05, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 43.33, "se": 9.05, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 43.33, "se": 9.05, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typescript:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/typescript", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 43.33, "se": 9.05, "n": 30, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 94, "se": 3.36, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 78, "se": 5.86, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 80, "se": 5.66, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 92, "se": 3.84, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 72, "se": 6.35, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/typos", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 76, "se": 6.04, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/typos", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 74, "se": 6.2, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/typos", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/typos", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 80, "se": 5.66, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/typos", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/typos", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 86, "se": 4.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 74, "se": 6.2, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 90, "se": 4.24, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 70, "se": 6.48, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 54, "se": 7.05, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 88, "se": 4.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/typos", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 82, "se": 5.43, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/typos:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/typos", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 70, "se": 6.48, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-fable-5-1-max-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-fable-5-1", "source_model": "claude-fable-5-1-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-opus-4-8-max-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-opus-5-5-max-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-opus-5-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-opus-5-5", "source_model": "claude-opus-5-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-opus-5-max-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-opus-5", "source_model": "claude-opus-5-max-effort", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-sonnet-4-6-thinking-auto-medium-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6-thinking-auto-medium-effort", "level": "medium", "settings": "mismatch", "score": 96, "se": 2.77, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:claude-sonnet-5-xhigh-effort", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-xhigh-effort", "level": "xhigh", "settings": "mismatch", "score": 88, "se": 4.6, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:deepseek-v4-flash", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 49.25, "se": 7.07, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:deepseek-v4-flash-0731", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-0731", "settings": "exact", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:deepseek-v4-pro", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 92, "se": 3.84, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:deepseek-v4-pro-0813", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro-0813", "settings": "exact", "score": 96.75, "se": 2.51, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:deepseek-v4.1-flash-max", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gemini-3.1-pro-preview-high", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview-high", "level": "high", "settings": "close", "score": 85.25, "se": 5.01, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gemini-3.5-flash-high", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 77.25, "se": 5.93, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gemini-3.5-flash-lite-high", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite-high", "level": "high", "settings": "mismatch", "score": 32.75, "se": 6.64, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gemini-3.8-flash-high", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 98.25, "se": 1.85, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gpt-5.4-mini-xhigh", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-xhigh", "level": "xhigh", "settings": "mismatch", "score": 60.37, "se": 6.92, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gpt-5.4-nano-xhigh", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-xhigh", "level": "xhigh", "settings": "mismatch", "score": 97.62, "se": 2.16, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gpt-5.4-xhigh", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gpt-5.4", "source_model": "gpt-5.4-xhigh", "level": "xhigh", "settings": "mismatch", "score": 98, "se": 1.98, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gpt-5.5-xhigh", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gpt-5.5", "source_model": "gpt-5.5-xhigh", "level": "xhigh", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:gpt-6-astra-max", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 100, "se": 1, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "livebench:task/zebra_puzzle:grok-4.7-xhigh", "source": "livebench", "benchmark": "livebench:task/zebra_puzzle", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 86, "se": 4.91, "n": 50, "unit": "percent", "own_run": true, "date": "2026-06-25" }, { "id": "lmarena:agent/overall:GPT 5.5", "source": "lmarena", "benchmark": "lmarena:agent/overall", "model": "gpt-5.5", "source_model": "GPT 5.5", "settings": "exact", "score": 0.01, "se": 0, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:agent/overall:Gemini 3.1 Pro Preview", "source": "lmarena", "benchmark": "lmarena:agent/overall", "model": "gemini-3.1-pro-preview", "source_model": "Gemini 3.1 Pro Preview", "settings": "exact", "score": -0.07, "se": 0, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:agent/overall:Mistral Medium 3.5", "source": "lmarena", "benchmark": "lmarena:agent/overall", "model": "mistral-medium-3-5", "source_model": "Mistral Medium 3.5", "settings": "exact", "score": -0.12, "se": 0.01, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:document/overall:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1512.58, "se": 7.75, "n": 1403, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1419.97, "se": 2.99, "n": 34677, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1464.07, "se": 3.55, "n": 12128, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1474.79, "se": 3.52, "n": 11551, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1516.27, "se": 3.58, "n": 8794, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1482.24, "se": 3, "n": 57813, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1466.14, "se": 3.9, "n": 7411, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1443.92, "se": 2.67, "n": 49457, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1461.4, "se": 4.72, "n": 4061, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1463.27, "se": 4.01, "n": 6500, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1471, "se": 3.08, "n": 33331, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1485.72, "se": 3.18, "n": 21279, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1484.48, "se": 3.25, "n": 20944, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:document/overall:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:document/overall", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1467.52, "se": 7.28, "n": 1621, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:search/overall:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:search/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1204.3, "se": 3.27, "n": 70998, "unit": "rating", "own_run": false, "date": "2026-08-24" }, { "id": "lmarena:text_style_control/chinese:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1573.11, "se": 10.97, "n": 841, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1431.37, "se": 3.37, "n": 10012, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1526.95, "se": 4.93, "n": 4395, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1526.19, "se": 4.93, "n": 4342, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1548.16, "se": 5, "n": 4653, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1548.13, "se": 6.89, "n": 2239, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1571.58, "se": 15.62, "n": 353, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1508.99, "se": 4.85, "n": 4519, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1495.58, "se": 5.39, "n": 3488, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1470.4, "se": 5.82, "n": 3003, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1489.6, "se": 5.49, "n": 3402, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1523.83, "se": 12.01, "n": 640, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1528.35, "se": 3.81, "n": 8292, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1515.12, "se": 5.39, "n": 3613, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1483.13, "se": 6.16, "n": 2726, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1517.34, "se": 5.51, "n": 3431, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1535.37, "se": 7.15, "n": 1970, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1423.17, "se": 5.45, "n": 3265, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1384.22, "se": 6.14, "n": 2530, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1316.62, "se": 4.72, "n": 5777, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1404.34, "se": 7.24, "n": 1920, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1493.68, "se": 5.01, "n": 4362, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1507.17, "se": 5.1, "n": 4143, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1473.54, "se": 5.21, "n": 3924, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1421.96, "se": 5.35, "n": 3838, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1526.43, "se": 5.01, "n": 4466, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1516.19, "se": 4.92, "n": 4493, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1514.38, "se": 11.55, "n": 631, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1375.54, "se": 6.54, "n": 2243, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1506.65, "se": 13.3, "n": 484, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1293.2, "se": 5.4, "n": 3664, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1210.24, "se": 5.54, "n": 7840, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1304.36, "se": 5.5, "n": 3933, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1297.21, "se": 6.97, "n": 1914, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1434.06, "se": 4.16, "n": 5954, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1386.12, "se": 6.73, "n": 2024, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1426.83, "se": 3.78, "n": 7115, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1445.09, "se": 12.23, "n": 620, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/chinese:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/chinese", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1355.02, "se": 9.11, "n": 1074, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1532.47, "se": 6.54, "n": 2254, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1480.83, "se": 2.09, "n": 36871, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1530.24, "se": 2.96, "n": 17933, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1533.3, "se": 2.98, "n": 17275, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1533.7, "se": 3.23, "n": 15273, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1529.88, "se": 4.01, "n": 7234, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1538.02, "se": 9.74, "n": 918, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1528.69, "se": 2.87, "n": 19706, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1518.92, "se": 3.26, "n": 12249, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1483.9, "se": 3.18, "n": 15253, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1500.38, "se": 3.02, "n": 17135, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1529.07, "se": 6.37, "n": 2314, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1521.27, "se": 2.38, "n": 33447, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1506.91, "se": 3.13, "n": 13673, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1500.99, "se": 3.55, "n": 9763, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1505.91, "se": 3.2, "n": 13480, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1530.8, "se": 4.12, "n": 6660, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1456.1, "se": 3.39, "n": 9191, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1432.98, "se": 3.85, "n": 6801, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1347.98, "se": 3.35, "n": 10926, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1431.19, "se": 4.32, "n": 5481, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1511.2, "se": 2.96, "n": 18903, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1520.5, "se": 2.99, "n": 17560, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1494.66, "se": 3.01, "n": 17208, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1460.1, "se": 3.03, "n": 17640, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1510.53, "se": 2.88, "n": 20019, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1518.56, "se": 2.92, "n": 19151, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1539.04, "se": 6.85, "n": 2010, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1389.67, "se": 3.94, "n": 6374, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1492.11, "se": 8.6, "n": 1238, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1345.74, "se": 3.35, "n": 8717, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1295.75, "se": 4.34, "n": 10418, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1353.76, "se": 4.02, "n": 7589, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1345.96, "se": 4.54, "n": 4212, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1469.27, "se": 2.48, "n": 19911, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1432.64, "se": 4.09, "n": 5826, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1453.92, "se": 2.34, "n": 21502, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1478.63, "se": 5.54, "n": 3281, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/coding:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/coding", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1412.14, "se": 5.12, "n": 3326, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1481.64, "se": 6.56, "n": 2506, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1387.4, "se": 2.46, "n": 24017, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1463.47, "se": 3.34, "n": 12743, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1469.43, "se": 3.36, "n": 12699, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1472.35, "se": 3.52, "n": 12964, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1469.54, "se": 4.43, "n": 6376, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1514.98, "se": 10.78, "n": 908, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1449.38, "se": 3.34, "n": 12357, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1435.75, "se": 3.74, "n": 9284, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1407.77, "se": 3.85, "n": 8570, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1446.13, "se": 3.72, "n": 9736, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1437.83, "se": 7.87, "n": 1626, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1480.34, "se": 2.8, "n": 22144, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1463.77, "se": 3.7, "n": 9378, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1435.4, "se": 4.12, "n": 7360, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1469.1, "se": 3.79, "n": 9356, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1484.4, "se": 4.71, "n": 5335, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1402.38, "se": 3.93, "n": 6595, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1349.05, "se": 4.5, "n": 5057, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1293.93, "se": 3.58, "n": 10484, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1323.71, "se": 5.34, "n": 3490, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1434.78, "se": 3.48, "n": 11321, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1442.43, "se": 3.55, "n": 11118, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1401.56, "se": 3.61, "n": 10803, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1336.07, "se": 3.71, "n": 10384, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1448.03, "se": 3.36, "n": 13393, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1451.39, "se": 3.37, "n": 13155, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1448.27, "se": 7.45, "n": 1875, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1277.82, "se": 5.08, "n": 3869, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1432.9, "se": 8.83, "n": 1293, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1285.34, "se": 3.61, "n": 8016, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1211.09, "se": 4.82, "n": 9056, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1287.42, "se": 4.43, "n": 6692, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1276.19, "se": 4.64, "n": 4439, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1373.37, "se": 2.94, "n": 13942, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1364.29, "se": 4.92, "n": 4038, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1378.25, "se": 2.84, "n": 13972, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1400.31, "se": 7.03, "n": 2067, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/creative_writing:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/creative_writing", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1323.22, "se": 6.63, "n": 2124, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1497.32, "se": 4.65, "n": 4547, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1430.47, "se": 1.71, "n": 62057, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1482.53, "se": 2.55, "n": 28511, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1489.61, "se": 2.56, "n": 27760, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1494.35, "se": 2.74, "n": 23826, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1490.03, "se": 3.36, "n": 11474, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1503.47, "se": 7.81, "n": 1455, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1482.49, "se": 2.4, "n": 31876, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1472.54, "se": 2.79, "n": 19077, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1444.64, "se": 2.7, "n": 23047, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1467.34, "se": 2.61, "n": 25236, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1489.09, "se": 5.54, "n": 2994, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1490.07, "se": 2.05, "n": 53494, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1478.51, "se": 2.66, "n": 20335, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1461.07, "se": 3.08, "n": 14370, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1477.76, "se": 2.75, "n": 19815, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1497.3, "se": 3.58, "n": 9791, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1422.81, "se": 2.47, "n": 23221, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1396.39, "se": 2.79, "n": 17984, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1328.18, "se": 2.21, "n": 37729, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1397.12, "se": 3.22, "n": 11403, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1468.2, "se": 2.5, "n": 30767, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1476.35, "se": 2.53, "n": 29230, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1450.26, "se": 2.53, "n": 29081, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1412.56, "se": 2.56, "n": 28828, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1482.57, "se": 2.44, "n": 32168, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1484.26, "se": 2.45, "n": 31480, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1486.85, "se": 5.26, "n": 3462, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1361.07, "se": 3.01, "n": 12953, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1444.62, "se": 6.48, "n": 2213, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1343.68, "se": 2.22, "n": 29195, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1266.2, "se": 2.97, "n": 36796, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1336.91, "se": 2.62, "n": 24863, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1321.4, "se": 2.77, "n": 16289, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1426.81, "se": 2.05, "n": 32833, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1402.15, "se": 3.04, "n": 15060, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1424.97, "se": 1.84, "n": 40711, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1443.91, "se": 4.34, "n": 5633, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/english:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/english", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1375.33, "se": 3.59, "n": 7505, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1518.57, "se": 4.38, "n": 8341, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1401.17, "se": 1.71, "n": 106626, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1484.54, "se": 2.6, "n": 48591, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1493.58, "se": 2.6, "n": 47760, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1505.98, "se": 2.63, "n": 44123, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1502.39, "se": 3.32, "n": 21378, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1523.96, "se": 6.97, "n": 2817, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1481.78, "se": 2.39, "n": 53736, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1467.06, "se": 2.8, "n": 33900, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1431.56, "se": 2.71, "n": 39244, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1461.5, "se": 2.62, "n": 43583, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1481.34, "se": 4.9, "n": 5931, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1501.42, "se": 2.08, "n": 91214, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1486.85, "se": 2.64, "n": 36162, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1458.14, "se": 3.01, "n": 26492, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1486.97, "se": 2.8, "n": 35092, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1511.07, "se": 3.52, "n": 18352, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1403.11, "se": 2.61, "n": 35501, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1358.74, "se": 3.01, "n": 27366, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1260.89, "se": 2.72, "n": 43349, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1365.91, "se": 3.21, "n": 19411, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1470.87, "se": 2.46, "n": 52114, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1484.26, "se": 2.51, "n": 49573, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1448.53, "se": 2.5, "n": 48675, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1388.27, "se": 2.57, "n": 47894, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1486.07, "se": 2.47, "n": 54140, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1491.87, "se": 2.47, "n": 52700, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1488.07, "se": 4.73, "n": 6385, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1312.74, "se": 3.14, "n": 21350, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1440.5, "se": 5.85, "n": 4109, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1261.61, "se": 2.59, "n": 36655, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1141.56, "se": 3.55, "n": 41750, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1254.18, "se": 3.17, "n": 28276, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1236.88, "se": 3.52, "n": 18034, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1400.76, "se": 2.06, "n": 57718, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1366.25, "se": 3.31, "n": 22666, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1392.99, "se": 1.88, "n": 68275, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1420.72, "se": 4.32, "n": 8921, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/exclude_ties:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/exclude_ties", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1318.23, "se": 3.76, "n": 12127, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1519.26, "se": 9.89, "n": 965, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1451.44, "se": 3.09, "n": 12987, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1516.58, "se": 3.98, "n": 7328, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1525.25, "se": 4.02, "n": 7004, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1542.36, "se": 4.36, "n": 6572, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1529.72, "se": 5.93, "n": 3073, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1538.97, "se": 14.22, "n": 448, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1506.29, "se": 4.04, "n": 7355, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1511.75, "se": 4.54, "n": 5051, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1457.05, "se": 4.48, "n": 5386, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1479.66, "se": 4.3, "n": 5915, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1511.36, "se": 9.78, "n": 926, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1508.55, "se": 3.27, "n": 12526, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1492.29, "se": 4.42, "n": 5500, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1469.22, "se": 5.03, "n": 4111, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1487.46, "se": 4.44, "n": 5528, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1524.79, "se": 5.8, "n": 2925, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1405.36, "se": 6, "n": 2589, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1383.41, "se": 6.66, "n": 2008, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1284.94, "se": 5.55, "n": 3549, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1400.98, "se": 9.01, "n": 1200, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1492.04, "se": 4.21, "n": 6703, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1515.26, "se": 4.27, "n": 6334, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1480.17, "se": 4.22, "n": 6296, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1437.48, "se": 4.3, "n": 6357, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1511.44, "se": 3.98, "n": 7633, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1512.4, "se": 4, "n": 7379, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1513.04, "se": 10.46, "n": 854, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1356.91, "se": 8.18, "n": 1368, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1458.63, "se": 12.2, "n": 580, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1297.53, "se": 5.73, "n": 2904, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1229.15, "se": 6.99, "n": 2940, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1301.7, "se": 6.42, "n": 2505, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1270.68, "se": 7.68, "n": 1510, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1427.35, "se": 3.79, "n": 7439, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1378.81, "se": 6.97, "n": 1777, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1410.15, "se": 3.9, "n": 6519, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1434.51, "se": 8.79, "n": 1205, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/expert:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/expert", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1334.62, "se": 10.08, "n": 837, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1521.82, "se": 16.94, "n": 360, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1429.64, "se": 5.88, "n": 3968, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1505.1, "se": 7.43, "n": 2192, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1508.45, "se": 7.45, "n": 2153, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1508.42, "se": 8.29, "n": 1721, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1495.66, "se": 11.2, "n": 853, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1481.57, "se": 7.29, "n": 2277, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1483.76, "se": 8.76, "n": 1453, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1448.61, "se": 7.67, "n": 1999, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1484.99, "se": 7.47, "n": 2208, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1447.81, "se": 20.22, "n": 237, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1501.08, "se": 5.93, "n": 3955, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1492.14, "se": 8.62, "n": 1496, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1480.45, "se": 9.94, "n": 1067, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1481.53, "se": 8.8, "n": 1471, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1508.57, "se": 11.85, "n": 742, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1420.87, "se": 11.46, "n": 731, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1407.61, "se": 13.03, "n": 580, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1337.27, "se": 12.54, "n": 649, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1407.91, "se": 15.4, "n": 412, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1485.64, "se": 7.27, "n": 2414, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1499.64, "se": 7.28, "n": 2394, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1476.06, "se": 7.49, "n": 2230, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1429.51, "se": 7.68, "n": 2098, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1501.34, "se": 7.14, "n": 2511, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1495.82, "se": 7.11, "n": 2517, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1498, "se": 21.9, "n": 193, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1357.19, "se": 14.49, "n": 454, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1325.87, "se": 12.86, "n": 569, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1284.31, "se": 12.25, "n": 1049, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1327.47, "se": 15.11, "n": 441, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1373.45, "se": 20.55, "n": 204, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1432.28, "se": 7.3, "n": 2211, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1389.27, "se": 13.66, "n": 495, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1434.16, "se": 7.31, "n": 2105, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1437.39, "se": 15.29, "n": 453, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/french:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/french", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1361.87, "se": 19.74, "n": 226, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1488.48, "se": 20.55, "n": 209, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1404.77, "se": 6.94, "n": 2370, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1473.66, "se": 9.57, "n": 1080, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1491.36, "se": 9.64, "n": 1061, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1486.92, "se": 10.66, "n": 902, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1504.89, "se": 14.24, "n": 484, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1455.66, "se": 9.12, "n": 1221, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1471.01, "se": 11.81, "n": 707, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1424.64, "se": 10.81, "n": 843, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1468.53, "se": 9.9, "n": 1001, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1494.4, "se": 7.52, "n": 2017, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1481.95, "se": 11.37, "n": 760, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1472.27, "se": 11.97, "n": 637, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1494.8, "se": 11.63, "n": 744, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1465.13, "se": 14.77, "n": 380, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1409.31, "se": 8.49, "n": 1279, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1392.06, "se": 9.49, "n": 1049, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1301.6, "se": 7.9, "n": 1641, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1385.9, "se": 12.53, "n": 590, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1449.35, "se": 10.09, "n": 1004, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1476.22, "se": 9.7, "n": 1092, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1442.88, "se": 10.39, "n": 929, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1393.81, "se": 10.62, "n": 935, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1489.41, "se": 9.77, "n": 1110, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1471.13, "se": 9.79, "n": 1054, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1328.58, "se": 11.7, "n": 670, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1289.91, "se": 8.69, "n": 1221, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1243.66, "se": 8.88, "n": 1690, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1302.4, "se": 9.08, "n": 1230, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1284.73, "se": 11.56, "n": 648, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1408.84, "se": 8.99, "n": 1288, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1393.26, "se": 9.61, "n": 990, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1400.65, "se": 7.66, "n": 1739, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1445.29, "se": 22.25, "n": 175, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/german:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/german", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1356.14, "se": 15.01, "n": 426, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1521.51, "se": 4.03, "n": 6630, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1440.7, "se": 1.56, "n": 87995, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1505.25, "se": 2.33, "n": 43049, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1512.86, "se": 2.33, "n": 42198, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1515.88, "se": 2.37, "n": 38320, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1514.54, "se": 2.97, "n": 18450, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1533.24, "se": 6.17, "n": 2437, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1503.4, "se": 2.19, "n": 46186, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1490.7, "se": 2.5, "n": 29825, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1458.77, "se": 2.43, "n": 34708, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1480.17, "se": 2.34, "n": 38514, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1498.28, "se": 4.33, "n": 5282, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1507.28, "se": 1.87, "n": 78890, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1494.76, "se": 2.35, "n": 32041, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1473.61, "se": 2.69, "n": 23471, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1494.59, "se": 2.49, "n": 31179, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1515.06, "se": 3.14, "n": 16172, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1432.35, "se": 2.51, "n": 21907, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1402.33, "se": 2.91, "n": 16023, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1310.92, "se": 2.72, "n": 18256, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1403.99, "se": 3.11, "n": 12795, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1486.84, "se": 2.28, "n": 44662, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1496.56, "se": 2.31, "n": 42060, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1468.69, "se": 2.29, "n": 41566, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1421.19, "se": 2.32, "n": 41268, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1499.27, "se": 2.23, "n": 47241, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1501.16, "se": 2.24, "n": 45509, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1501.07, "se": 4.36, "n": 5254, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1361.94, "se": 2.88, "n": 14616, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1461.97, "se": 5.24, "n": 3405, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1320.67, "se": 2.55, "n": 16920, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1262.41, "se": 3.55, "n": 17187, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1319.93, "se": 3.23, "n": 12682, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1313.32, "se": 3.59, "n": 6954, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1431.66, "se": 1.84, "n": 48181, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1400.93, "se": 3.05, "n": 13589, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1426.89, "se": 1.7, "n": 52794, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1445.62, "se": 3.92, "n": 7637, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1374.01, "se": 3.53, "n": 7719, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1518.45, "se": 6.02, "n": 2671, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1452.56, "se": 2.06, "n": 38941, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1507.04, "se": 2.91, "n": 19326, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1516.25, "se": 2.91, "n": 18643, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1514.98, "se": 3.17, "n": 15454, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1512.51, "se": 3.99, "n": 7369, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1532.58, "se": 9.84, "n": 914, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1506.33, "se": 2.81, "n": 21295, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1495.62, "se": 3.22, "n": 12593, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1463.2, "se": 3.09, "n": 16167, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1484.96, "se": 2.99, "n": 17811, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1506.7, "se": 6.82, "n": 2000, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1506.49, "se": 2.36, "n": 35600, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1490.1, "se": 3.12, "n": 13633, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1476.29, "se": 3.6, "n": 9488, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1490.11, "se": 3.19, "n": 13165, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1511.55, "se": 4.15, "n": 6337, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1438.13, "se": 3.24, "n": 10380, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1412.93, "se": 3.73, "n": 7642, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1319.13, "se": 3.29, "n": 11352, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1409.39, "se": 4.29, "n": 5569, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1486.54, "se": 2.87, "n": 21016, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1493.07, "se": 2.9, "n": 19663, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1466.21, "se": 2.9, "n": 19794, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1429.29, "se": 2.93, "n": 19875, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1500.17, "se": 2.81, "n": 21917, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1498.95, "se": 2.82, "n": 21165, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1506.98, "se": 6.65, "n": 2127, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1365.98, "se": 3.93, "n": 6440, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1461.46, "se": 8.29, "n": 1327, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1336.68, "se": 3.23, "n": 9521, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1273.3, "se": 4.24, "n": 11296, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1332.91, "se": 3.94, "n": 7818, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1322.88, "se": 4.4, "n": 4452, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1439.22, "se": 2.49, "n": 20054, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1409.29, "se": 3.99, "n": 6347, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1438.01, "se": 2.29, "n": 23104, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1457.11, "se": 5.17, "n": 3835, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/hard_prompts_english:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/hard_prompts_english", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1394.54, "se": 5.03, "n": 3423, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1486.72, "se": 6.66, "n": 2102, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1419.47, "se": 2.28, "n": 27698, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1486.71, "se": 3.25, "n": 12693, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1489.3, "se": 3.27, "n": 12641, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1486.45, "se": 3.52, "n": 11158, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1484.03, "se": 4.56, "n": 5350, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1492.86, "se": 11.29, "n": 716, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1481.67, "se": 3.15, "n": 14051, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1464.85, "se": 3.66, "n": 8766, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1438.24, "se": 3.56, "n": 10421, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1459.98, "se": 3.43, "n": 11388, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1478.25, "se": 7.84, "n": 1494, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1477.12, "se": 2.61, "n": 23673, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1468.3, "se": 3.55, "n": 9311, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1451.38, "se": 4.1, "n": 6515, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1462.36, "se": 3.64, "n": 9024, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1480.8, "se": 4.72, "n": 4637, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1415.02, "se": 3.6, "n": 8216, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1378.96, "se": 4.14, "n": 5993, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1310.68, "se": 3.97, "n": 7765, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1380.61, "se": 4.62, "n": 4815, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1474.12, "se": 3.25, "n": 13849, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1481.66, "se": 3.31, "n": 13081, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1459.75, "se": 3.31, "n": 12836, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1405.35, "se": 3.37, "n": 12568, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1485.52, "se": 3.23, "n": 14079, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1489.52, "se": 3.24, "n": 13729, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1477.49, "se": 7.8, "n": 1539, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1351.71, "se": 4.29, "n": 5345, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1430.07, "se": 9.46, "n": 996, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1303.59, "se": 3.87, "n": 6619, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1219.93, "se": 5.21, "n": 6461, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1290.09, "se": 4.78, "n": 5167, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1293.76, "se": 5.47, "n": 3021, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1410.05, "se": 2.79, "n": 14812, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1386.88, "se": 4.45, "n": 5080, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1406.75, "se": 2.54, "n": 17840, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1430.41, "se": 6.54, "n": 2254, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_business_and_management_and_financial_operations:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_business_and_management_and_financial_operations", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1362.62, "se": 5.61, "n": 2920, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1475.45, "se": 6.11, "n": 2818, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1388.38, "se": 2.24, "n": 30588, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1455.12, "se": 3.1, "n": 15932, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1459.32, "se": 3.11, "n": 16083, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1464.49, "se": 3.3, "n": 15468, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1460.13, "se": 4.09, "n": 7558, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1497.79, "se": 9.84, "n": 1013, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1445.98, "se": 3.1, "n": 15902, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1432.76, "se": 3.44, "n": 11592, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1404.32, "se": 3.54, "n": 11204, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1434.66, "se": 3.38, "n": 12786, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1450.31, "se": 7.22, "n": 1944, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1468.87, "se": 2.59, "n": 27899, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1454.23, "se": 3.38, "n": 11810, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1433.11, "se": 3.78, "n": 9138, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1456.98, "se": 3.43, "n": 11933, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1470.03, "se": 4.32, "n": 6468, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1400.76, "se": 3.43, "n": 8950, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1348.13, "se": 3.9, "n": 6699, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1284.3, "se": 3.35, "n": 11283, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1348.28, "se": 4.61, "n": 4814, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1432.11, "se": 3.2, "n": 14789, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1441.18, "se": 3.28, "n": 14136, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1408.94, "se": 3.31, "n": 13882, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1351.33, "se": 3.38, "n": 13668, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1447.34, "se": 3.08, "n": 17172, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1448.17, "se": 3.11, "n": 16713, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1457.17, "se": 6.88, "n": 2156, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1285.83, "se": 4.21, "n": 5440, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1413.49, "se": 8.38, "n": 1404, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1285.28, "se": 3.24, "n": 9686, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1213.07, "se": 4.59, "n": 9175, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1290.6, "se": 4.22, "n": 7002, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1271.61, "se": 4.26, "n": 5130, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1371.81, "se": 2.68, "n": 17264, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1355.48, "se": 4.2, "n": 5570, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1373.95, "se": 2.54, "n": 17975, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1395.92, "se": 6.14, "n": 2687, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_entertainment_and_sports_and_media:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_entertainment_and_sports_and_media", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1324.34, "se": 5.28, "n": 3116, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1494.34, "se": 10.26, "n": 918, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1415.48, "se": 3.31, "n": 10700, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1496.86, "se": 4.51, "n": 5472, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1495.33, "se": 4.53, "n": 5385, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1503.15, "se": 4.85, "n": 4946, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1502.46, "se": 6.54, "n": 2454, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1490.35, "se": 16.68, "n": 336, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1484.31, "se": 4.46, "n": 5693, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1477.73, "se": 5.19, "n": 3876, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1447.68, "se": 5.03, "n": 4138, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1474.42, "se": 4.87, "n": 4501, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1451.56, "se": 11.7, "n": 676, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1499.81, "se": 3.56, "n": 9969, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1478.47, "se": 5.12, "n": 4039, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1454.16, "se": 5.88, "n": 3012, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1484.51, "se": 5.31, "n": 3775, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1492.91, "se": 6.7, "n": 2233, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1436.85, "se": 5.42, "n": 3220, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1396.4, "se": 6.2, "n": 2428, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1346.6, "se": 5.56, "n": 3866, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1395.47, "se": 7.55, "n": 1679, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1475.68, "se": 4.61, "n": 5371, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1490.23, "se": 4.65, "n": 5233, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1456.87, "se": 4.75, "n": 5040, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1399.85, "se": 4.81, "n": 4961, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1485.5, "se": 4.4, "n": 5915, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1494.79, "se": 4.51, "n": 5614, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1490.16, "se": 11.37, "n": 728, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1339.33, "se": 7.07, "n": 1862, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1459.66, "se": 13.3, "n": 510, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1342.15, "se": 5.52, "n": 3150, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1268.29, "se": 6.91, "n": 3306, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1338.16, "se": 6.3, "n": 2735, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1325.95, "se": 7.61, "n": 1593, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1421, "se": 4.15, "n": 6002, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1409.45, "se": 6.59, "n": 2107, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1416.87, "se": 3.87, "n": 6690, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1437.71, "se": 10.58, "n": 891, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_legal_and_government:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_legal_and_government", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1369.88, "se": 8.74, "n": 1138, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1521.46, "se": 7.24, "n": 1842, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1430.25, "se": 2.42, "n": 23254, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1499.3, "se": 3.39, "n": 10657, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1503.8, "se": 3.43, "n": 10329, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1510.75, "se": 3.65, "n": 9955, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1510.53, "se": 4.73, "n": 4741, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1520.12, "se": 11.71, "n": 659, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1499.99, "se": 3.33, "n": 11646, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1489.64, "se": 3.87, "n": 7453, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1456.23, "se": 3.77, "n": 8412, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1481.28, "se": 3.65, "n": 9159, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1485.95, "se": 8.43, "n": 1285, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1510.72, "se": 2.71, "n": 20185, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1490.07, "se": 3.72, "n": 7925, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1478, "se": 4.29, "n": 5756, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1496.18, "se": 3.83, "n": 7699, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1515.72, "se": 5.03, "n": 4040, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1424.97, "se": 3.54, "n": 8439, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1384.71, "se": 4.04, "n": 6468, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1326.51, "se": 3.41, "n": 11641, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1397.89, "se": 4.88, "n": 4187, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1478.51, "se": 3.43, "n": 11077, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1487.6, "se": 3.49, "n": 10640, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1464.82, "se": 3.52, "n": 10499, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1415.18, "se": 3.58, "n": 10512, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1496.46, "se": 3.34, "n": 11802, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1497.9, "se": 3.36, "n": 11611, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1492.71, "se": 8.05, "n": 1442, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1361.22, "se": 4.52, "n": 4799, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1457.91, "se": 9.74, "n": 961, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1340.02, "se": 3.34, "n": 9329, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1250.49, "se": 4.6, "n": 10875, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1330.56, "se": 4.21, "n": 7485, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1314.55, "se": 4.46, "n": 4869, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1428.8, "se": 2.9, "n": 12899, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1409.28, "se": 4.29, "n": 5601, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1424.91, "se": 2.69, "n": 15252, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1435.68, "se": 7.12, "n": 1940, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_life_and_physical_and_social_science:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_life_and_physical_and_social_science", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1367.19, "se": 5.67, "n": 2875, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1524.58, "se": 12.1, "n": 638, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1432.68, "se": 3.8, "n": 7566, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1488.58, "se": 5.39, "n": 3589, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1506.58, "se": 5.47, "n": 3431, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1535.64, "se": 5.82, "n": 3263, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1515.34, "se": 7.95, "n": 1573, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1514.86, "se": 17.43, "n": 258, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1484.26, "se": 5.18, "n": 3986, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1485.25, "se": 6.22, "n": 2465, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1434.16, "se": 5.84, "n": 2973, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1459.61, "se": 5.55, "n": 3289, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1495.48, "se": 12.7, "n": 507, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1484.31, "se": 4.15, "n": 6773, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1482.47, "se": 6, "n": 2833, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1445.67, "se": 6.93, "n": 2000, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1486.93, "se": 6.2, "n": 2652, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1509.48, "se": 8.26, "n": 1421, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1373.41, "se": 5.44, "n": 3032, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1363.66, "se": 5.97, "n": 2448, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1285.7, "se": 3.79, "n": 7963, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1406.47, "se": 8.52, "n": 1268, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1471.85, "se": 5.33, "n": 3726, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1498.66, "se": 5.42, "n": 3544, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1451.87, "se": 5.53, "n": 3424, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1431.05, "se": 5.52, "n": 3481, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1498.6, "se": 5.19, "n": 4064, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1505.79, "se": 5.17, "n": 3991, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1496.6, "se": 13.86, "n": 483, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1384.06, "se": 7.58, "n": 1511, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1447.34, "se": 15.46, "n": 335, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1301.67, "se": 4.17, "n": 5222, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1242.27, "se": 4.89, "n": 7028, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1298.03, "se": 4.4, "n": 5745, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1283.52, "se": 5.17, "n": 3037, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1414.44, "se": 4.94, "n": 4128, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1353.47, "se": 6.33, "n": 2166, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1401.36, "se": 4.36, "n": 5059, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1438.67, "se": 11.68, "n": 652, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_mathematical:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_mathematical", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1351.98, "se": 9.36, "n": 978, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1516.98, "se": 10.62, "n": 866, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1425.01, "se": 3.56, "n": 9633, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1497.09, "se": 4.84, "n": 4735, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1499.09, "se": 4.81, "n": 4736, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1503.59, "se": 5.07, "n": 4384, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1504.8, "se": 6.98, "n": 2143, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1537.92, "se": 18.48, "n": 274, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1488.15, "se": 4.55, "n": 5444, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1480.87, "se": 5.69, "n": 3266, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1459.29, "se": 5.36, "n": 3690, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1478.03, "se": 5.17, "n": 4057, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1498.19, "se": 13.32, "n": 537, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1506.03, "se": 3.72, "n": 9058, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1484.3, "se": 5.49, "n": 3532, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1469.44, "se": 6.32, "n": 2583, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1486.31, "se": 5.6, "n": 3400, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1508.04, "se": 7.63, "n": 1737, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1432.04, "se": 6, "n": 2760, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1395.37, "se": 6.53, "n": 2306, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1334.4, "se": 5.82, "n": 3439, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1406.77, "se": 8.25, "n": 1431, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1465.93, "se": 4.76, "n": 5044, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1475.01, "se": 4.86, "n": 4837, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1450.11, "se": 4.92, "n": 4767, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1413.33, "se": 4.95, "n": 4795, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1485.76, "se": 4.65, "n": 5343, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1475.63, "se": 4.61, "n": 5237, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1465.71, "se": 12.1, "n": 627, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1367.68, "se": 7.69, "n": 1617, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1477.8, "se": 15.21, "n": 374, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1342.3, "se": 6.09, "n": 2726, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1247.63, "se": 6.96, "n": 3434, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1322.94, "se": 6.86, "n": 2423, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1317.59, "se": 8.56, "n": 1230, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1430.41, "se": 4.34, "n": 5541, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1419.11, "se": 7.03, "n": 1962, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1432.98, "se": 4.15, "n": 6000, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1421.07, "se": 10.85, "n": 845, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_medicine_and_healthcare:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_medicine_and_healthcare", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1389.41, "se": 9.08, "n": 1125, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1528.59, "se": 5.13, "n": 3738, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1463.87, "se": 1.81, "n": 53874, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1517.72, "se": 2.65, "n": 25504, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1523.61, "se": 2.65, "n": 24909, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1524.53, "se": 2.87, "n": 21921, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1515.88, "se": 3.49, "n": 10426, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1531.34, "se": 7.88, "n": 1404, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1514.19, "se": 2.54, "n": 27743, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1503.63, "se": 2.87, "n": 17596, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1470.73, "se": 2.81, "n": 21330, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1491.63, "se": 2.69, "n": 23653, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1517.2, "se": 5.38, "n": 3253, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1512.42, "se": 2.12, "n": 47818, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1501.19, "se": 2.75, "n": 19202, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1487.94, "se": 3.11, "n": 14144, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1496.1, "se": 2.85, "n": 18676, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1516.97, "se": 3.62, "n": 9512, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1442.93, "se": 2.76, "n": 16363, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1418.08, "se": 3.13, "n": 12352, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1338.07, "se": 2.84, "n": 17793, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1415.69, "se": 3.55, "n": 8866, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1498.51, "se": 2.63, "n": 26922, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1506.5, "se": 2.64, "n": 25420, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1485.19, "se": 2.66, "n": 24995, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1449.91, "se": 2.7, "n": 24926, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1505.93, "se": 2.56, "n": 28296, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1510.19, "se": 2.57, "n": 27435, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1521.08, "se": 5.48, "n": 3120, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1385.38, "se": 3.28, "n": 10168, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1480.4, "se": 6.8, "n": 1984, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1337.82, "se": 2.75, "n": 14822, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1270.22, "se": 3.72, "n": 17516, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1343.11, "se": 3.42, "n": 12143, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1333.49, "se": 3.74, "n": 6815, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1455.61, "se": 2.15, "n": 29566, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1421.71, "se": 3.35, "n": 10517, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1439.31, "se": 1.97, "n": 33857, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1469.27, "se": 4.75, "n": 4579, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_software_and_it_services:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_software_and_it_services", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1399.44, "se": 3.95, "n": 5878, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1492.77, "se": 5.75, "n": 3159, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1399.83, "se": 2.11, "n": 34618, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1467.6, "se": 2.99, "n": 17278, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1474.56, "se": 2.98, "n": 17150, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1482.35, "se": 3.19, "n": 16518, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1479.17, "se": 3.95, "n": 8098, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1519.01, "se": 9.31, "n": 1159, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1455.04, "se": 2.9, "n": 17594, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1452.3, "se": 3.3, "n": 12341, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1415.53, "se": 3.34, "n": 12571, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1450.85, "se": 3.23, "n": 14147, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1450.97, "se": 6.85, "n": 2085, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1480.26, "se": 2.46, "n": 30825, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1473.34, "se": 3.25, "n": 12533, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1441.85, "se": 3.65, "n": 9627, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1471.77, "se": 3.31, "n": 12680, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1486.23, "se": 4.19, "n": 6807, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1405.35, "se": 3.16, "n": 11253, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1360.49, "se": 3.56, "n": 8602, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1313.82, "se": 2.87, "n": 18531, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1351.61, "se": 4.1, "n": 6099, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1458.29, "se": 3.02, "n": 16695, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1464.49, "se": 3.05, "n": 16332, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1423.23, "se": 3.09, "n": 15829, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1362.49, "se": 3.17, "n": 15389, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1466.43, "se": 2.97, "n": 18510, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1469.06, "se": 2.98, "n": 18157, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1465.17, "se": 6.51, "n": 2369, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1309.37, "se": 3.92, "n": 6590, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1426.82, "se": 7.87, "n": 1579, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1291.65, "se": 2.88, "n": 13769, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1225.85, "se": 3.95, "n": 15192, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1303.19, "se": 3.51, "n": 12224, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1298.29, "se": 3.67, "n": 7416, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1389.11, "se": 2.52, "n": 19588, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1370.47, "se": 3.88, "n": 7137, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1386.2, "se": 2.34, "n": 21837, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1405.11, "se": 5.77, "n": 2987, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/industry_writing_and_literature_and_language:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/industry_writing_and_literature_and_language", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1329.09, "se": 4.97, "n": 3730, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1497.68, "se": 5.51, "n": 3380, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1414.99, "se": 1.92, "n": 45457, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1478.51, "se": 2.73, "n": 23069, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1490.43, "se": 2.72, "n": 22488, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1496.99, "se": 2.95, "n": 20700, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1492.69, "se": 3.62, "n": 9911, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1514.42, "se": 8.3, "n": 1337, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1474.33, "se": 2.65, "n": 23644, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1466.87, "se": 2.97, "n": 15961, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1427.94, "se": 2.97, "n": 17914, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1452.39, "se": 2.85, "n": 20002, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1477.9, "se": 5.74, "n": 2880, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1479.34, "se": 2.22, "n": 41331, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1465, "se": 2.87, "n": 17201, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1442.6, "se": 3.26, "n": 12701, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1465.5, "se": 2.95, "n": 16877, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1487.18, "se": 3.75, "n": 8909, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1403.29, "se": 2.94, "n": 13210, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1372.9, "se": 3.34, "n": 9956, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1293.1, "se": 2.48, "n": 26705, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1373.26, "se": 3.87, "n": 6953, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1460.12, "se": 2.75, "n": 23320, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1471.13, "se": 2.79, "n": 21840, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1433.91, "se": 2.78, "n": 21666, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1386.82, "se": 2.82, "n": 21423, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1475.13, "se": 2.69, "n": 24750, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1477.72, "se": 2.71, "n": 23963, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1473.1, "se": 5.89, "n": 2708, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1323.37, "se": 3.61, "n": 7804, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1441.89, "se": 6.93, "n": 1875, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1292.61, "se": 2.58, "n": 18745, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1237.86, "se": 3.39, "n": 21532, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1299.46, "se": 2.91, "n": 18321, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1294.74, "se": 3.11, "n": 10971, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1400.45, "se": 2.28, "n": 25339, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1365.93, "se": 3.67, "n": 7889, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1394.74, "se": 2.12, "n": 27530, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1421.21, "se": 5.11, "n": 3820, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/instruction_following:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/instruction_following", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1337.69, "se": 4.55, "n": 4350, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1547.28, "se": 23.86, "n": 194, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1360.91, "se": 8.66, "n": 1584, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1457.57, "se": 11.17, "n": 870, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1454.6, "se": 11.3, "n": 876, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1507.28, "se": 11.49, "n": 940, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1499.6, "se": 15.21, "n": 474, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1439.3, "se": 11.49, "n": 793, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1442.53, "se": 12.66, "n": 665, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1398.13, "se": 12.69, "n": 647, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1457.43, "se": 12.21, "n": 715, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1498.78, "se": 9.23, "n": 1396, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1476.66, "se": 12.53, "n": 668, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1444.2, "se": 13.73, "n": 570, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1463.44, "se": 12.96, "n": 666, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1503.24, "se": 15.54, "n": 442, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1346.38, "se": 9.5, "n": 1118, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1329.18, "se": 10.12, "n": 986, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1250.47, "se": 7.93, "n": 1793, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1364.51, "se": 14.87, "n": 447, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1445, "se": 12.33, "n": 681, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1478.95, "se": 12.71, "n": 671, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1408.76, "se": 13.04, "n": 618, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1372.91, "se": 13.47, "n": 603, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1478.2, "se": 11.85, "n": 801, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1503.72, "se": 11.12, "n": 899, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1420.71, "se": 25.6, "n": 163, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1326.92, "se": 12.93, "n": 568, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1212.35, "se": 9.63, "n": 1156, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1079.11, "se": 11.73, "n": 945, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1233.55, "se": 9.18, "n": 1249, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1213.58, "se": 11.76, "n": 629, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1380.12, "se": 11.1, "n": 858, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1325.33, "se": 10.29, "n": 891, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1357.68, "se": 9.38, "n": 1115, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/japanese:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/japanese", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1286.46, "se": 13.4, "n": 510, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1366.53, "se": 7.18, "n": 2239, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1446.82, "se": 9.95, "n": 1047, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1449.78, "se": 10.08, "n": 1079, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1492.91, "se": 10.19, "n": 1095, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1497.66, "se": 13.31, "n": 547, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1433.69, "se": 9.53, "n": 1147, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1433.53, "se": 11.31, "n": 781, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1394.07, "se": 10.7, "n": 897, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1418.4, "se": 9.7, "n": 1106, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1456.85, "se": 7.52, "n": 2129, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1437.64, "se": 10.18, "n": 951, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1425.22, "se": 12.01, "n": 689, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1444.57, "se": 11.05, "n": 839, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1452.55, "se": 15.38, "n": 445, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1370.35, "se": 9.64, "n": 1041, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1346.45, "se": 11.14, "n": 806, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1238.11, "se": 10.56, "n": 1087, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1321.5, "se": 12.74, "n": 648, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1426.46, "se": 9.84, "n": 1158, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1434.49, "se": 10.14, "n": 1106, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1392.6, "se": 10.33, "n": 985, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1343.65, "se": 10.27, "n": 1032, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1429.78, "se": 9.62, "n": 1179, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1445.06, "se": 9.47, "n": 1186, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1266.42, "se": 12.46, "n": 668, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1214.84, "se": 11.31, "n": 854, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1108.4, "se": 11.98, "n": 1090, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1226.92, "se": 12.49, "n": 751, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1259.1, "se": 15.39, "n": 408, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1357.82, "se": 9.04, "n": 1333, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1317.1, "se": 12.25, "n": 632, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1350.78, "se": 7.41, "n": 1969, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1393.24, "se": 22.1, "n": 188, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/korean:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/korean", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1271.03, "se": 16.38, "n": 359, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1511.39, "se": 4.81, "n": 4705, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1436.32, "se": 1.93, "n": 53867, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1499.75, "se": 2.63, "n": 30720, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1502.03, "se": 2.63, "n": 30035, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1504.8, "se": 2.78, "n": 27765, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1499.61, "se": 3.35, "n": 13516, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1527.81, "se": 7.62, "n": 1708, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1492.39, "se": 2.56, "n": 31045, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1481.83, "se": 2.83, "n": 21121, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1447.41, "se": 2.84, "n": 23332, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1471.65, "se": 2.72, "n": 25921, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1478.38, "se": 5.21, "n": 3644, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1498.89, "se": 2.18, "n": 53616, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1481.48, "se": 2.74, "n": 22623, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1461.62, "se": 3.08, "n": 16906, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1480.45, "se": 2.8, "n": 22475, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1507.93, "se": 3.53, "n": 11720, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1426.9, "se": 3.27, "n": 9890, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1389.88, "se": 3.83, "n": 7035, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1326.18, "se": 3.63, "n": 8986, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1370.25, "se": 4.19, "n": 5857, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1477.36, "se": 2.66, "n": 29904, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1481.33, "se": 2.7, "n": 28408, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1446.73, "se": 2.69, "n": 27712, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1402.6, "se": 2.74, "n": 27687, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1486.64, "se": 2.55, "n": 33347, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1487.33, "se": 2.58, "n": 32011, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1485.58, "se": 5.29, "n": 3656, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1324.42, "se": 3.96, "n": 6520, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1450.54, "se": 6.42, "n": 2342, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1313.38, "se": 3.41, "n": 8311, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1248.09, "se": 5.07, "n": 6217, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1304.8, "se": 4.44, "n": 5885, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1305.5, "se": 4.91, "n": 3607, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1416.95, "se": 2.24, "n": 29819, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1392.36, "se": 4.11, "n": 5890, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1408.29, "se": 2.17, "n": 27737, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1429.62, "se": 4.65, "n": 5091, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/longer_query:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/longer_query", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1358.76, "se": 5.14, "n": 3400, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1517.46, "se": 12.98, "n": 523, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1400.23, "se": 3.65, "n": 7930, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1476.25, "se": 5.64, "n": 3049, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1493.88, "se": 5.75, "n": 2918, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1522.49, "se": 6.18, "n": 2703, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1514.83, "se": 8.43, "n": 1310, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1510.21, "se": 19.84, "n": 202, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1463.24, "se": 5.14, "n": 3775, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1473.61, "se": 6.76, "n": 2018, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1424.75, "se": 6.02, "n": 2686, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1444.76, "se": 5.68, "n": 3052, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1502.04, "se": 14.29, "n": 408, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1487.87, "se": 4.17, "n": 6277, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1496.34, "se": 6.41, "n": 2331, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1441.85, "se": 7.6, "n": 1572, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1481.48, "se": 6.62, "n": 2165, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1516.35, "se": 8.87, "n": 1196, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1374.56, "se": 5.25, "n": 3203, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1353.43, "se": 5.69, "n": 2657, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1275.35, "se": 3.6, "n": 9319, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1405.04, "se": 7.94, "n": 1445, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1459.91, "se": 5.27, "n": 3550, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1491.4, "se": 5.41, "n": 3379, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1438.82, "se": 5.48, "n": 3364, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1424.73, "se": 5.53, "n": 3259, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1499.39, "se": 5.39, "n": 3493, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1493.24, "se": 5.31, "n": 3472, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1490, "se": 14.15, "n": 436, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1381.12, "se": 7, "n": 1769, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1444.85, "se": 17.37, "n": 288, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1295.83, "se": 3.98, "n": 5763, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1244.45, "se": 4.64, "n": 7987, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1288.08, "se": 4.17, "n": 6664, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1282.09, "se": 4.78, "n": 3574, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1401.5, "se": 4.64, "n": 4486, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1348.21, "se": 6.22, "n": 2207, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1395.9, "se": 3.97, "n": 5962, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1428.35, "se": 12.31, "n": 565, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/math:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/math", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1337.98, "se": 8.96, "n": 1036, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1486.89, "se": 8.05, "n": 1474, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1424.96, "se": 2.37, "n": 24983, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1494.54, "se": 3.43, "n": 11326, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1497.7, "se": 3.43, "n": 11116, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1484.29, "se": 3.75, "n": 9144, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1485.96, "se": 4.92, "n": 4327, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1496.84, "se": 13.39, "n": 496, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1481.56, "se": 3.28, "n": 12712, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1473.54, "se": 3.87, "n": 7677, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1453.21, "se": 3.72, "n": 9354, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1474.99, "se": 3.58, "n": 10361, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1462.28, "se": 8.9, "n": 1178, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1496.48, "se": 2.74, "n": 20777, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1480.81, "se": 3.79, "n": 7997, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1464.55, "se": 4.36, "n": 5773, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1478.24, "se": 3.92, "n": 7565, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1501.17, "se": 5.23, "n": 3796, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1430.17, "se": 3.51, "n": 8731, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1393.42, "se": 3.9, "n": 6999, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1311.05, "se": 3.44, "n": 12279, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1372.45, "se": 4.79, "n": 4394, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1480.21, "se": 3.35, "n": 12667, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1493.04, "se": 3.39, "n": 11980, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1465.49, "se": 3.41, "n": 11875, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1413.3, "se": 3.46, "n": 11795, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1484.17, "se": 3.37, "n": 12388, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1487.42, "se": 3.43, "n": 11837, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1488.7, "se": 8.88, "n": 1172, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1326.76, "se": 4.47, "n": 4887, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1424.72, "se": 10.56, "n": 816, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1316.53, "se": 3.41, "n": 9058, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1226.38, "se": 4.7, "n": 9521, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1296.5, "se": 4.05, "n": 8296, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1293.48, "se": 4.59, "n": 4438, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1419.64, "se": 2.91, "n": 13359, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1402.87, "se": 4.21, "n": 5717, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1407.21, "se": 2.64, "n": 16397, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1432.88, "se": 7.15, "n": 1924, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/multi_turn:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/multi_turn", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1364.93, "se": 5.49, "n": 2993, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1495.41, "se": 3.97, "n": 6689, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1393.87, "se": 1.55, "n": 80633, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1463.57, "se": 2.38, "n": 36195, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1469.37, "se": 2.38, "n": 35807, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1481.93, "se": 2.44, "n": 34543, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1480.87, "se": 3.01, "n": 16873, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1496.19, "se": 6.09, "n": 2470, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1457.49, "se": 2.21, "n": 38948, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1448.21, "se": 2.54, "n": 26108, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1423.04, "se": 2.52, "n": 28821, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1444.66, "se": 2.41, "n": 32313, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1456.96, "se": 4.38, "n": 5122, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1478.98, "se": 1.9, "n": 67720, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1468.24, "se": 2.42, "n": 27639, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1443.75, "se": 2.74, "n": 20960, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1467.39, "se": 2.53, "n": 26915, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1483.6, "se": 3.16, "n": 15030, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1402.52, "se": 2.37, "n": 26742, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1361.24, "se": 2.69, "n": 20559, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1300.74, "se": 2.34, "n": 30968, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1378.06, "se": 2.94, "n": 15201, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1456.58, "se": 2.34, "n": 36984, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1468.07, "se": 2.37, "n": 35416, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1437.24, "se": 2.39, "n": 34229, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1386.06, "se": 2.4, "n": 33472, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1466.07, "se": 2.29, "n": 38725, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1473.59, "se": 2.31, "n": 37831, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1462.15, "se": 4.41, "n": 5098, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1337.24, "se": 2.74, "n": 17047, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1435.28, "se": 5.21, "n": 3428, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1283.53, "se": 2.27, "n": 25174, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1206.32, "se": 3.2, "n": 25640, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1284.11, "se": 2.72, "n": 20596, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1278.06, "se": 3.01, "n": 11784, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1397.61, "se": 1.82, "n": 46144, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1369.87, "se": 2.87, "n": 17548, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1388.79, "se": 1.67, "n": 54258, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1406.34, "se": 4.16, "n": 6124, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/non_english:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/non_english", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1333.34, "se": 3.25, "n": 9838, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1501.25, "se": 3.32, "n": 11241, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1413.56, "se": 1.24, "n": 142743, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1474.52, "se": 1.97, "n": 64711, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1481.29, "se": 1.98, "n": 63577, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1491.19, "se": 1.94, "n": 58380, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1489.08, "se": 2.55, "n": 28351, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1503.98, "se": 4.93, "n": 3932, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1472.13, "se": 1.8, "n": 70828, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1461.83, "se": 2.15, "n": 45191, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1435.88, "se": 2.06, "n": 51869, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1457.72, "se": 1.99, "n": 57549, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1472.77, "se": 3.61, "n": 8125, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1486.96, "se": 1.54, "n": 121225, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1477.07, "se": 1.99, "n": 47981, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1454.11, "se": 2.29, "n": 35335, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1476.2, "se": 2.12, "n": 46735, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1494.09, "se": 2.6, "n": 24828, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1414.83, "se": 1.9, "n": 50007, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1382.75, "se": 2.21, "n": 38549, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1317.67, "se": 1.81, "n": 68697, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1389.63, "se": 2.38, "n": 26645, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1464.86, "se": 1.89, "n": 67753, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1475.18, "se": 1.92, "n": 64648, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1447.05, "se": 1.92, "n": 63311, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1401.13, "se": 1.93, "n": 62300, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1476.73, "se": 1.89, "n": 70894, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1481.34, "se": 1.9, "n": 69312, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1476.18, "se": 3.58, "n": 8565, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1351.7, "se": 2.23, "n": 30039, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1442.45, "se": 4.17, "n": 5675, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1317.79, "se": 1.79, "n": 54371, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1242.47, "se": 2.4, "n": 62436, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1314.37, "se": 2.01, "n": 45459, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1305.54, "se": 2.27, "n": 28073, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1413.72, "se": 1.48, "n": 78986, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1387.43, "se": 2.41, "n": 32616, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1407.96, "se": 1.33, "n": 94995, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1426.93, "se": 3.27, "n": 11757, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/overall:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/overall", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1356.46, "se": 2.67, "n": 17353, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1486.44, "se": 22.54, "n": 164, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1368.79, "se": 5.58, "n": 3370, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1483.52, "se": 9.1, "n": 1067, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1487.57, "se": 9.45, "n": 1049, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1502.24, "se": 10.82, "n": 838, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1501.64, "se": 14.34, "n": 431, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1471.94, "se": 8.13, "n": 1415, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1440.3, "se": 11.94, "n": 641, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1438.4, "se": 9.54, "n": 1031, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1466.26, "se": 9.25, "n": 1123, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1493.78, "se": 6.5, "n": 2330, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1510.76, "se": 11.24, "n": 770, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1451.43, "se": 13.28, "n": 491, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1497.04, "se": 11.92, "n": 651, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1493.89, "se": 15.85, "n": 350, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1427.89, "se": 5.95, "n": 3051, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1379.15, "se": 6.28, "n": 2664, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1277.4, "se": 34.3, "n": 65, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1399.02, "se": 8.27, "n": 1559, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1471.67, "se": 8.58, "n": 1305, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1481.59, "se": 8.61, "n": 1320, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1462.06, "se": 9.15, "n": 1191, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1401.55, "se": 9.47, "n": 1130, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1471.9, "se": 9.01, "n": 1199, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1479.61, "se": 8.99, "n": 1171, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1331.67, "se": 7.45, "n": 1736, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1319.05, "se": 8.6, "n": 1450, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1213.26, "se": 30.48, "n": 170, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1415.59, "se": 7.25, "n": 1772, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1391.79, "se": 6, "n": 2879, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1400.68, "se": 5.54, "n": 3323, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1418.21, "se": 19.39, "n": 234, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/polish:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/polish", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1341.59, "se": 7.58, "n": 1729, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1514.39, "se": 8.3, "n": 1431, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1400.59, "se": 2.72, "n": 15791, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1485.73, "se": 3.97, "n": 7046, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1494.77, "se": 3.95, "n": 7090, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1497.21, "se": 4.15, "n": 6959, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1492.47, "se": 5.52, "n": 3368, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-opus-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-high", "level": "high", "settings": "mismatch", "score": 1509.57, "se": 12.91, "n": 511, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1465.66, "se": 3.77, "n": 7967, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1469.99, "se": 4.53, "n": 5053, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1432.88, "se": 4.34, "n": 5745, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1460.06, "se": 4.2, "n": 6311, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1483.14, "se": 9.08, "n": 1153, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1500.68, "se": 3.12, "n": 13675, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1488.93, "se": 4.42, "n": 5395, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1460.91, "se": 5.01, "n": 4024, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1491.8, "se": 4.5, "n": 5330, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1507.62, "se": 5.64, "n": 3131, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1412.15, "se": 4.82, "n": 3787, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1367.52, "se": 5.39, "n": 3086, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1309.73, "se": 3.82, "n": 8268, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1377.95, "se": 6.81, "n": 1859, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1469.67, "se": 3.92, "n": 7652, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1486.83, "se": 4.02, "n": 7202, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1450.8, "se": 4.07, "n": 6970, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1388.65, "se": 4.1, "n": 6881, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1478.78, "se": 3.94, "n": 7406, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1480.24, "se": 3.94, "n": 7335, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1467.51, "se": 9.22, "n": 1103, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1337.33, "se": 6.32, "n": 2132, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1444.33, "se": 10.78, "n": 768, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1294.74, "se": 4.08, "n": 5707, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1237.9, "se": 5.4, "n": 5462, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1305.13, "se": 4.5, "n": 5748, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1295.99, "se": 5.16, "n": 3258, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1408.6, "se": 3.29, "n": 9469, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1385.7, "se": 5.59, "n": 2753, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1404.28, "se": 3.21, "n": 9710, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1410.55, "se": 8.75, "n": 1243, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/russian:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/russian", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1352.04, "se": 8.17, "n": 1228, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1489.36, "se": 15.43, "n": 359, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1428.97, "se": 5.42, "n": 4117, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1466.15, "se": 7.48, "n": 1901, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1483.13, "se": 7.66, "n": 1811, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1476.94, "se": 8.16, "n": 1633, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1485.33, "se": 11.2, "n": 780, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1466.85, "se": 7.17, "n": 2176, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1448.98, "se": 8.95, "n": 1243, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:deepseek-v4-flash", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "deepseek-flash", "source_model": "deepseek-v4-flash", "settings": "exact", "score": 1432.19, "se": 8.24, "n": 1577, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1457.15, "se": 7.85, "n": 1724, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1483.67, "se": 19.48, "n": 245, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1481.85, "se": 5.59, "n": 3849, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1471.1, "se": 8.63, "n": 1385, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1451.76, "se": 10.26, "n": 927, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1467.33, "se": 8.98, "n": 1254, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1475.99, "se": 10.9, "n": 776, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1410.5, "se": 9.34, "n": 1128, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1368.49, "se": 10.8, "n": 803, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1308.88, "se": 12.19, "n": 741, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1371.69, "se": 11, "n": 852, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1460.6, "se": 7.34, "n": 2041, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1454.52, "se": 7.3, "n": 2071, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1434.69, "se": 7.65, "n": 1914, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1392.23, "se": 7.67, "n": 1919, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1459.12, "se": 7.34, "n": 2035, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1470.05, "se": 7.45, "n": 1936, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1438.88, "se": 19.4, "n": 243, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:gpt-oss-120b", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "gpt-oss-120b", "source_model": "gpt-oss-120b", "settings": "exact", "score": 1367.81, "se": 10.88, "n": 852, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1434.93, "se": 22.32, "n": 169, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:llama-3.3-70b-instruct", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "llama-3.3-70b-instruct-turbo", "source_model": "llama-3.3-70b-instruct", "settings": "exact", "score": 1305.15, "se": 12.31, "n": 635, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-large-2402", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-large-2512", "source_model": "mistral-large-2402", "settings": "exact", "score": 1260.12, "se": 12.8, "n": 886, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-large-2407", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-large-2512", "source_model": "mistral-large-2407", "settings": "exact", "score": 1286.91, "se": 13.32, "n": 584, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-large-2411", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-large-2512", "source_model": "mistral-large-2411", "settings": "exact", "score": 1307.95, "se": 23.16, "n": 153, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1406.53, "se": 6.79, "n": 2331, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1378.76, "se": 13.01, "n": 563, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1401.02, "se": 6.25, "n": 2855, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1393.79, "se": 16.2, "n": 390, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:text_style_control/spanish:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:text_style_control/spanish", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1366, "se": 15.03, "n": 419, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:vision/captioning:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1343.62, "se": 24.5, "n": 137, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/captioning:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1207.79, "se": 15.82, "n": 422, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/captioning:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1173.24, "se": 16.42, "n": 395, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/captioning:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1216.03, "se": 18.59, "n": 283, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/captioning:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1094.68, "se": 25.05, "n": 160, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/captioning:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1145.97, "se": 15.39, "n": 405, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/captioning:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/captioning", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1059.24, "se": 24.8, "n": 178, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1365.39, "se": 19.79, "n": 222, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1345.41, "se": 10.1, "n": 1110, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1342.72, "se": 10.49, "n": 1001, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1404.98, "se": 10.97, "n": 906, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1306.86, "se": 8.63, "n": 1740, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1332.98, "se": 11.71, "n": 781, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1364.39, "se": 6.94, "n": 3094, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1380.42, "se": 12.05, "n": 755, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1309.75, "se": 15.3, "n": 415, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1379.01, "se": 11.9, "n": 726, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1227.55, "se": 8.6, "n": 1974, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1186, "se": 9.05, "n": 1778, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1033.93, "se": 19.34, "n": 954, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1214.15, "se": 9.5, "n": 1585, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1335.6, "se": 9.56, "n": 1352, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1355.08, "se": 8.52, "n": 1760, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1285.26, "se": 8.82, "n": 1661, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1232.37, "se": 8.76, "n": 1740, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1357.7, "se": 8.72, "n": 1697, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1358.8, "se": 9.1, "n": 1491, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1224.05, "se": 14.37, "n": 493, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1161.26, "se": 12.86, "n": 598, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1190.09, "se": 8.25, "n": 2171, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1242.89, "se": 19.11, "n": 290, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/chinese:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/chinese", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1162.68, "se": 11.93, "n": 847, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1222.9, "se": 8.29, "n": 1432, "unit": "rating", "own_run": false, "date": "2026-01-09" }, { "id": "lmarena:vision/creative_writing:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1164.89, "se": 8.92, "n": 1276, "unit": "rating", "own_run": false, "date": "2026-01-09" }, { "id": "lmarena:vision/creative_writing:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1181.31, "se": 9.27, "n": 1108, "unit": "rating", "own_run": false, "date": "2026-01-09" }, { "id": "lmarena:vision/creative_writing:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1168.64, "se": 12.57, "n": 518, "unit": "rating", "own_run": false, "date": "2026-01-09" }, { "id": "lmarena:vision/creative_writing:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1223.06, "se": 8.38, "n": 1365, "unit": "rating", "own_run": false, "date": "2026-01-09" }, { "id": "lmarena:vision/creative_writing:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1185.31, "se": 14.3, "n": 442, "unit": "rating", "own_run": false, "date": "2026-01-09" }, { "id": "lmarena:vision/creative_writing_vision:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1321.6, "se": 15.74, "n": 425, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1289.65, "se": 8.7, "n": 1370, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1300.75, "se": 8.64, "n": 1368, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1312.53, "se": 8.59, "n": 1406, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1273.86, "se": 7.75, "n": 1704, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1250.97, "se": 10.1, "n": 1005, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1324.52, "se": 5.87, "n": 3095, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1290.88, "se": 10.05, "n": 1017, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1254.45, "se": 11.95, "n": 685, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1310.6, "se": 10.32, "n": 952, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1271.16, "se": 19.36, "n": 252, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1209.55, "se": 10.02, "n": 1283, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1181.96, "se": 10.53, "n": 1192, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1180.31, "se": 10.93, "n": 1017, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1272.97, "se": 8.33, "n": 1450, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1278.84, "se": 7.35, "n": 1912, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1220.75, "se": 7.88, "n": 1633, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1157.53, "se": 8.49, "n": 1537, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1285.15, "se": 7.17, "n": 1958, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1305.08, "se": 7.45, "n": 1860, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1273.26, "se": 18.08, "n": 313, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1217.14, "se": 11.8, "n": 737, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1145.92, "se": 14.36, "n": 480, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1199.69, "se": 8.78, "n": 1550, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1200.21, "se": 16.73, "n": 352, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/creative_writing_vision:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/creative_writing_vision", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1180.42, "se": 14.75, "n": 466, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1342.26, "se": 9.93, "n": 965, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1303.54, "se": 4.89, "n": 4859, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1314.76, "se": 4.9, "n": 4827, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1349.27, "se": 5.33, "n": 3856, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1305.32, "se": 4.33, "n": 7655, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1298.12, "se": 5.58, "n": 3356, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1311.7, "se": 3.59, "n": 11970, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1320.67, "se": 5.81, "n": 3106, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1287.25, "se": 7.59, "n": 1782, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1323.2, "se": 5.74, "n": 3147, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1344.74, "se": 11.73, "n": 700, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1215.12, "se": 6.62, "n": 2760, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1182.23, "se": 6.95, "n": 2368, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1203, "se": 7.53, "n": 1935, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1316.07, "se": 4.67, "n": 6232, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1326.59, "se": 4.24, "n": 7742, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1265.84, "se": 4.43, "n": 6942, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1210.81, "se": 4.55, "n": 6861, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1326.07, "se": 4.31, "n": 7297, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1315.76, "se": 4.41, "n": 6739, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1299.77, "se": 11.75, "n": 686, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1238.88, "se": 7.15, "n": 1993, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1155.16, "se": 8.11, "n": 1426, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1175.28, "se": 5.36, "n": 3742, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1255.24, "se": 8.15, "n": 1545, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/diagram:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/diagram", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1125.82, "se": 9.09, "n": 1167, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1314.12, "se": 8.22, "n": 1470, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1292.99, "se": 4.73, "n": 7214, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1290.71, "se": 4.69, "n": 7122, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1316.49, "se": 5.04, "n": 5423, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1298.17, "se": 4.29, "n": 11561, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1275.14, "se": 5.16, "n": 4895, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1287.05, "se": 3.69, "n": 17823, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1306.1, "se": 5.36, "n": 4385, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1266.2, "se": 6.62, "n": 2503, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1300.49, "se": 5.36, "n": 4366, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1319.49, "se": 10.6, "n": 888, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1209.34, "se": 4.61, "n": 19026, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1190.4, "se": 4.84, "n": 19165, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1072.27, "se": 5.49, "n": 10721, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1215.57, "se": 5.34, "n": 13439, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1289.83, "se": 4.52, "n": 9608, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1300.47, "se": 4.2, "n": 11487, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1244.02, "se": 4.3, "n": 10706, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1201.19, "se": 4.39, "n": 10363, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1294.56, "se": 4.27, "n": 10729, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1289.76, "se": 4.3, "n": 10137, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1281.01, "se": 10.07, "n": 961, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1235.71, "se": 6.45, "n": 2725, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1171.61, "se": 5.42, "n": 5320, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1179.08, "se": 4.29, "n": 19034, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1240.79, "se": 6.99, "n": 2321, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/english:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/english", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1161.54, "se": 5.78, "n": 5379, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1196.69, "se": 23.41, "n": 158, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1330.55, "se": 14.95, "n": 378, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1207.25, "se": 16.35, "n": 392, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1186.72, "se": 16.63, "n": 358, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1225.69, "se": 18.09, "n": 282, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1251.02, "se": 23.68, "n": 134, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1227.25, "se": 23.11, "n": 155, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1212.4, "se": 20.82, "n": 182, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1143.1, "se": 21.73, "n": 175, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1224.16, "se": 27.33, "n": 115, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1218.49, "se": 24.63, "n": 127, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1140.45, "se": 20.48, "n": 200, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1146.81, "se": 15.83, "n": 439, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/entity_recognition:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/entity_recognition", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1151.11, "se": 21.4, "n": 207, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1341.37, "se": 12.41, "n": 582, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1327.72, "se": 6.65, "n": 2205, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1337.23, "se": 6.97, "n": 2014, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1344.2, "se": 7.84, "n": 1567, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1310.74, "se": 5.44, "n": 3716, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1318.02, "se": 8.54, "n": 1326, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1317.38, "se": 4.4, "n": 6011, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1321.42, "se": 8.32, "n": 1366, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1295.82, "se": 11.43, "n": 719, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1329.57, "se": 8.67, "n": 1244, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1336.43, "se": 15.5, "n": 366, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1248.36, "se": 8.34, "n": 1481, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1214, "se": 8.71, "n": 1314, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1209.58, "se": 9.88, "n": 977, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1308.84, "se": 5.94, "n": 3099, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1337.03, "se": 5.47, "n": 3698, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1285.53, "se": 5.59, "n": 3512, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1225.7, "se": 5.72, "n": 3454, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1330.97, "se": 5.67, "n": 3368, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1333.53, "se": 5.72, "n": 3258, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1331.09, "se": 15.3, "n": 367, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1216.17, "se": 11.25, "n": 783, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1165.46, "se": 10.53, "n": 813, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1165.39, "se": 6.78, "n": 2152, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1229.61, "se": 11.21, "n": 806, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/homework:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/homework", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1174.37, "se": 12.01, "n": 616, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1282.17, "se": 11.46, "n": 761, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1280.49, "se": 11.56, "n": 726, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1297.18, "se": 11.29, "n": 767, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1248.9, "se": 9.8, "n": 1061, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1263.72, "se": 13.18, "n": 564, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1324.01, "se": 7.41, "n": 1805, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1312.31, "se": 14.22, "n": 493, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1280.13, "se": 16.82, "n": 318, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1325.15, "se": 14.42, "n": 480, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1196.01, "se": 10.54, "n": 1366, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1181.7, "se": 10.53, "n": 1404, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1205.37, "se": 11.39, "n": 1047, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1262.46, "se": 10.67, "n": 848, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1285.63, "se": 9.73, "n": 1055, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1232.95, "se": 10.7, "n": 871, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1159.81, "se": 10.72, "n": 924, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1275.6, "se": 9.69, "n": 1033, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1296.65, "se": 9.85, "n": 986, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1216.26, "se": 15.87, "n": 376, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1144.13, "se": 15.38, "n": 442, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1167.67, "se": 9.65, "n": 1503, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1218.28, "se": 19.98, "n": 239, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/humor:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/humor", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1139.24, "se": 15.04, "n": 481, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1326.43, "se": 5.92, "n": 2764, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1297.88, "se": 3.42, "n": 13097, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1302.48, "se": 3.42, "n": 12783, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1329.35, "se": 3.72, "n": 10485, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1295.85, "se": 3.13, "n": 19885, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1289, "se": 3.75, "n": 8914, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1303.19, "se": 2.6, "n": 31402, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1313.61, "se": 3.83, "n": 8293, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1278.83, "se": 4.76, "n": 4854, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1315.5, "se": 3.85, "n": 8359, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1329.18, "se": 7.33, "n": 1819, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1218.02, "se": 4.53, "n": 11366, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1187.68, "se": 4.8, "n": 10477, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1198.59, "se": 5.25, "n": 8003, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1298.77, "se": 3.36, "n": 16478, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1310.98, "se": 2.96, "n": 20521, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1252.86, "se": 3.18, "n": 18305, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1204.31, "se": 3.25, "n": 18264, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1308.12, "se": 3.04, "n": 18975, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1304.29, "se": 3.08, "n": 17580, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1292.32, "se": 7.06, "n": 1924, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1229.8, "se": 4.6, "n": 5260, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1153.75, "se": 5.33, "n": 4569, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1174.93, "se": 3.7, "n": 13480, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1232.24, "se": 5.31, "n": 3914, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/ocr:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/ocr", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1134.01, "se": 5.69, "n": 4085, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1318.93, "se": 5.21, "n": 4050, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1287.93, "se": 3.35, "n": 18583, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1294.96, "se": 3.33, "n": 18020, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1320.86, "se": 3.55, "n": 15304, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1282.75, "se": 3.15, "n": 27465, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1275.4, "se": 3.59, "n": 12722, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1296.21, "se": 2.68, "n": 45204, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1310.49, "se": 3.65, "n": 11983, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1266.09, "se": 4.34, "n": 7200, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1307.97, "se": 3.64, "n": 12124, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1312.02, "se": 6.2, "n": 2711, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-4.1-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-4.1", "source_model": "gpt-4.1-2025-04-14", "settings": "exact", "score": 1210.31, "se": 3.4, "n": 41027, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-4.1-mini-2025-04-14", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-4.1-mini", "source_model": "gpt-4.1-mini-2025-04-14", "settings": "exact", "score": 1181.29, "se": 3.81, "n": 40235, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-4o-mini-2024-07-18", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-4o-mini", "source_model": "gpt-4o-mini-2024-07-18", "settings": "exact", "score": 1065.48, "se": 4.23, "n": 17341, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5-mini", "source_model": "gpt-5-mini-high", "level": "high", "settings": "mismatch", "score": 1201.96, "se": 3.98, "n": 28971, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1291.91, "se": 3.32, "n": 22854, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5.4-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5.4", "source_model": "gpt-5.4-high", "level": "high", "settings": "mismatch", "score": 1302.17, "se": 3, "n": 28881, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1244.85, "se": 3.18, "n": 25775, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5.4-nano-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5.4-nano", "source_model": "gpt-5.4-nano-high", "level": "high", "settings": "mismatch", "score": 1196.06, "se": 3.24, "n": 25685, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5.5", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5.5", "source_model": "gpt-5.5", "settings": "exact", "score": 1297.26, "se": 3.03, "n": 26870, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-5.5-high", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-5.5", "source_model": "gpt-5.5-high", "level": "high", "settings": "mismatch", "score": 1292.16, "se": 3.07, "n": 25170, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1280.05, "se": 6.02, "n": 2869, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1221.46, "se": 4.19, "n": 7880, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:mistral-medium-2505", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2505", "settings": "exact", "score": 1156.86, "se": 4.21, "n": 10810, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:mistral-medium-2508", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-2508", "settings": "exact", "score": 1172.29, "se": 3.15, "n": 41647, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1222.37, "se": 4.85, "n": 5391, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:vision/overall:mistral-small-2506", "source": "lmarena", "benchmark": "lmarena:vision/overall", "model": "mistral-small-2603", "source_model": "mistral-small-2506", "settings": "exact", "score": 1141.55, "se": 4.63, "n": 10848, "unit": "rating", "own_run": false, "date": "2026-09-28" }, { "id": "lmarena:webdev/image_to_webdev:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1710.39, "se": 10.88, "n": 964, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1665.15, "se": 6.78, "n": 2490, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1530.05, "se": 4.63, "n": 7726, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1542.49, "se": 4.6, "n": 5572, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1478.31, "se": 4.56, "n": 7615, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1496.9, "se": 7.32, "n": 1855, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1448.36, "se": 8.11, "n": 1491, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1499.96, "se": 6.2, "n": 2622, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1445.47, "se": 6.81, "n": 2433, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1733.34, "se": 10.67, "n": 1085, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/image_to_webdev:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:webdev/image_to_webdev", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1362.25, "se": 9.16, "n": 1316, "unit": "rating", "own_run": false, "date": "2026-09-13" }, { "id": "lmarena:webdev/overall:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1750.67, "se": 4.94, "n": 6137, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1329.36, "se": 2.49, "n": 29175, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1534.11, "se": 2.88, "n": 17646, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1556.02, "se": 2.93, "n": 18685, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1659.97, "se": 2.88, "n": 21088, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1693.68, "se": 3.3, "n": 16747, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-opus-5.5-max", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-max", "level": "max", "settings": "mismatch", "score": 1817.84, "se": 8.42, "n": 1976, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1521.58, "se": 2.62, "n": 22931, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1539.52, "se": 3.17, "n": 13915, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:deepseek-v4-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-high", "level": "high", "settings": "mismatch", "score": 1581, "se": 4.71, "n": 5040, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1446.22, "se": 3.22, "n": 14032, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1620.01, "se": 5.33, "n": 3958, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1446.24, "se": 2.57, "n": 24319, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1499.09, "se": 3.69, "n": 9495, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1439.71, "se": 20.69, "n": 242, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1489.91, "se": 3.39, "n": 12259, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1582.66, "se": 4.04, "n": 8357, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1394.87, "se": 5.47, "n": 3743, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1396.89, "se": 3.32, "n": 11992, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1789.11, "se": 5.33, "n": 5918, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1636.27, "se": 6.13, "n": 3062, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1229.84, "se": 12.99, "n": 840, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/overall:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:webdev/overall", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1263.11, "se": 7.53, "n": 2318, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1682.08, "se": 13.34, "n": 706, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1317.37, "se": 4.69, "n": 10655, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1524.73, "se": 6.59, "n": 2399, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1545.32, "se": 7.09, "n": 2225, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1651.31, "se": 6.65, "n": 2847, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1681.12, "se": 8.17, "n": 1977, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1526.79, "se": 6.05, "n": 2899, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1524.07, "se": 7.87, "n": 1754, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:deepseek-v4-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-high", "level": "high", "settings": "mismatch", "score": 1557.78, "se": 13.05, "n": 607, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1434.04, "se": 8.12, "n": 1498, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1609.41, "se": 13.3, "n": 618, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1489.6, "se": 6.25, "n": 2618, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1513.33, "se": 9.73, "n": 1141, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1522.25, "se": 8.74, "n": 1342, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1620.28, "se": 9.9, "n": 1176, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1439.2, "se": 14.93, "n": 453, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1420.05, "se": 8.51, "n": 1436, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1779.94, "se": 13, "n": 931, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1621.74, "se": 15.38, "n": 496, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1238.52, "se": 13.31, "n": 840, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-html:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-html", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1309.29, "se": 20.94, "n": 260, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1797.48, "se": 6.06, "n": 4260, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1325.67, "se": 3.02, "n": 16094, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1535.5, "se": 3.29, "n": 12693, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1556.24, "se": 3.33, "n": 12166, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1680.33, "se": 3.39, "n": 14146, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1716.18, "se": 3.88, "n": 11094, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-opus-5.5-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-max", "level": "max", "settings": "mismatch", "score": 1866.55, "se": 10.54, "n": 1408, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1511.32, "se": 2.88, "n": 18208, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1541.89, "se": 3.64, "n": 9747, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:deepseek-v4-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-high", "level": "high", "settings": "mismatch", "score": 1584.08, "se": 5.67, "n": 3452, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1439.98, "se": 3.61, "n": 9673, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1630.22, "se": 6.55, "n": 2581, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1437.96, "se": 2.85, "n": 18375, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1494.62, "se": 4.25, "n": 6877, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1417.38, "se": 26.51, "n": 151, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1480.52, "se": 3.84, "n": 8647, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1575.44, "se": 4.77, "n": 5439, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1387.18, "se": 6.54, "n": 2684, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1386.19, "se": 3.62, "n": 9873, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1810.94, "se": 6.38, "n": 4085, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1660.2, "se": 7.65, "n": 1982, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev-react:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:webdev/webdev-react", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1253.1, "se": 8.24, "n": 1974, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-fable-5.1-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-fable-5-1", "source_model": "claude-fable-5.1-max", "level": "max", "settings": "mismatch", "score": 1750.67, "se": 4.94, "n": 6137, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-haiku-4-5-20251001", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-haiku-4-5", "source_model": "claude-haiku-4-5-20251001", "settings": "exact", "score": 1329.36, "se": 2.49, "n": 29175, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-opus-4-8", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8", "settings": "exact", "score": 1534.11, "se": 2.88, "n": 17646, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-opus-4-8-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-opus-4-8", "source_model": "claude-opus-4-8-high", "level": "high", "settings": "mismatch", "score": 1556.02, "se": 2.93, "n": 18685, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-opus-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-opus-5", "source_model": "claude-opus-5-high", "level": "high", "settings": "mismatch", "score": 1659.97, "se": 2.88, "n": 21088, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-opus-5-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-opus-5", "source_model": "claude-opus-5-max", "level": "max", "settings": "mismatch", "score": 1693.68, "se": 3.3, "n": 16747, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-opus-5.5-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-opus-5-5", "source_model": "claude-opus-5.5-max", "level": "max", "settings": "mismatch", "score": 1817.84, "se": 8.42, "n": 1976, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-sonnet-4-6", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-sonnet-4-6", "source_model": "claude-sonnet-4-6", "settings": "exact", "score": 1521.58, "se": 2.62, "n": 22931, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:claude-sonnet-5-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "claude-sonnet-5", "source_model": "claude-sonnet-5-high", "level": "high", "settings": "mismatch", "score": 1539.52, "se": 3.17, "n": 13915, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:deepseek-v4-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "deepseek-flash", "source_model": "deepseek-v4-flash-high", "level": "high", "settings": "mismatch", "score": 1581, "se": 4.71, "n": 5040, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:deepseek-v4-pro", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "deepseek-v4-pro", "source_model": "deepseek-v4-pro", "settings": "exact", "score": 1446.22, "se": 3.22, "n": 14032, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:deepseek-v4.1-flash-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "deepseek-flash", "source_model": "deepseek-v4.1-flash-max", "level": "max", "settings": "mismatch", "score": 1620.01, "se": 5.33, "n": 3958, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gemini-3.1-pro-preview", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gemini-3.1-pro-preview", "source_model": "gemini-3.1-pro-preview", "settings": "exact", "score": 1446.24, "se": 2.57, "n": 24319, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gemini-3.5-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-high", "level": "high", "settings": "mismatch", "score": 1499.09, "se": 3.69, "n": 9495, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gemini-3.5-flash-lite", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gemini-3.5-flash-lite", "source_model": "gemini-3.5-flash-lite", "settings": "exact", "score": 1439.71, "se": 20.69, "n": 242, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gemini-3.5-flash-medium", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gemini-3.5-flash", "source_model": "gemini-3.5-flash-medium", "level": "medium", "settings": "mismatch", "score": 1489.91, "se": 3.39, "n": 12259, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gemini-3.8-flash-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gemini-3.8-flash", "source_model": "gemini-3.8-flash-high", "level": "high", "settings": "mismatch", "score": 1582.66, "se": 4.04, "n": 8357, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gpt-5.4", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gpt-5.4", "source_model": "gpt-5.4", "settings": "exact", "score": 1394.87, "se": 5.47, "n": 3743, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gpt-5.4-mini-high", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gpt-5.4-mini", "source_model": "gpt-5.4-mini-high", "level": "high", "settings": "mismatch", "score": 1396.89, "se": 3.32, "n": 11992, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:gpt-6-astra-max", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "gpt-6-astra", "source_model": "gpt-6-astra-max", "level": "max", "settings": "mismatch", "score": 1789.11, "se": 5.33, "n": 5918, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:grok-4.7-xhigh", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "grok-4.7", "source_model": "grok-4.7-xhigh", "level": "xhigh", "settings": "close", "score": 1636.27, "se": 6.13, "n": 3062, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:mistral-large-3", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "mistral-large-2512", "source_model": "mistral-large-3", "settings": "exact", "score": 1229.84, "se": 12.99, "n": 840, "unit": "rating", "own_run": false, "date": "2026-09-30" }, { "id": "lmarena:webdev/webdev:mistral-medium-3.5", "source": "lmarena", "benchmark": "lmarena:webdev/webdev", "model": "mistral-medium-3-5", "source_model": "mistral-medium-3.5", "settings": "exact", "score": 1263.11, "se": 7.53, "n": 2318, "unit": "rating", "own_run": false, "date": "2026-09-30" } ], "task_scores": [ { "model": "gemini-3.8-flash", "task": "writing", "band": "medium", "score": 83.9, "se": 3.13, "evidence": 2.8788, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gemini-3.8-flash-high", "score": 87.79, "se": 2.67, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gemini-3.8-flash-high", "score": 80, "se": 5.66, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.8-flash-high", "score": 1487.18, "se": 3.75, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "writing", "band": "medium", "score": 80.51, "se": 3.43, "evidence": 2.716, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:claude-fable-5-1-max-effort", "score": 89.5, "se": 2.5, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:claude-fable-5-1-max-effort", "score": 71.52, "se": 6.38, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-fable-5.1-max", "score": 1497.68, "se": 5.51, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "writing", "band": "medium", "score": 83.84, "se": 3.17, "evidence": 2.6149, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gpt-6-astra-max", "score": 89.43, "se": 2.51, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gpt-6-astra-max", "score": 78.25, "se": 5.83, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-6-astra-max", "score": 1473.1, "se": 5.89, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "writing", "band": "medium", "score": 82.62, "se": 3.18, "evidence": 2.5373, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gemini-3.1-pro-preview-high", "score": 85.38, "se": 2.89, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gemini-3.1-pro-preview-high", "score": 79.87, "se": 5.67, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.1-pro-preview", "score": 1479.34, "se": 2.22, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "writing", "band": "medium", "score": 77.21, "se": 3.6, "evidence": 2.3657, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:claude-opus-5-max-effort", "score": 88.69, "se": 2.59, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:claude-opus-5-max-effort", "score": 65.73, "se": 6.71, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-opus-5-high", "score": 1496.99, "se": 2.95, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "writing", "band": "medium", "score": 79.55, "se": 3.46, "evidence": 2.1771, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gpt-5.5-xhigh", "score": 87.36, "se": 2.71, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gpt-5.5-xhigh", "score": 71.73, "se": 6.37, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.5", "score": 1475.13, "se": 2.69, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "writing", "band": "medium", "score": 75.27, "se": 3.67, "evidence": 2.177, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:claude-opus-5-5-max-effort", "score": 86.27, "se": 2.81, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:claude-opus-5-5-max-effort", "score": 64.27, "se": 6.78, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-opus-5.5-high", "score": 1514.42, "se": 8.3, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "writing", "band": "medium", "score": 80.05, "se": 3.38, "evidence": 1.8648, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gemini-3.5-flash-high", "score": 84.58, "se": 2.95, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gemini-3.5-flash-high", "score": 75.52, "se": 6.08, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-medium", "score": 1465.5, "se": 2.95, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "writing", "band": "medium", "score": 73.77, "se": 3.69, "evidence": 1.406, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:claude-opus-4-8-max-effort", "score": 79.66, "se": 3.29, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:claude-opus-4-8-max-effort", "score": 67.87, "se": 6.6, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-opus-4-8", "score": 1478.51, "se": 2.73, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "writing", "band": "medium", "score": 79.1, "se": 3.35, "evidence": 1.0905, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:grok-4.7-xhigh", "score": 80.14, "se": 3.26, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:grok-4.7-xhigh", "score": 78.05, "se": 5.85, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:grok-4.7-xhigh", "score": 1441.89, "se": 6.93, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "writing", "band": "medium", "score": 73.03, "se": 3.74, "evidence": 0.9538, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gpt-5.4-xhigh", "score": 82.63, "se": 3.09, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gpt-5.4-xhigh", "score": 63.43, "se": 6.81, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.4", "score": 1460.12, "se": 2.75, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "writing", "band": "medium", "score": 67.75, "se": 3.88, "evidence": 0.587, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 76.1, "se": 3.48, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 59.4, "se": 6.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-sonnet-4-6", "score": 1474.33, "se": 2.65, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "writing", "band": "medium", "score": 68.11, "se": 3.87, "evidence": 0.3099, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:claude-sonnet-5-xhigh-effort", "score": 74.97, "se": 3.54, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:claude-sonnet-5-xhigh-effort", "score": 61.25, "se": 6.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-sonnet-5-high", "score": 1466.87, "se": 2.97, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "writing", "band": "medium", "score": 67.18, "se": 3.9, "evidence": 0.0157, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:deepseek-v4-pro", "score": 78.13, "se": 3.38, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:deepseek-v4-pro", "score": 56.22, "se": 7.02, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-pro", "score": 1452.39, "se": 2.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "writing", "band": "medium", "score": 67.18, "se": 3.9, "evidence": 0.0157, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:deepseek-v4-pro", "score": 78.13, "se": 3.38, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:deepseek-v4-pro", "score": 56.22, "se": 7.02, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-pro", "score": 1452.39, "se": 2.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "writing", "band": "medium", "score": 68.41, "se": 3.84, "evidence": -0.2048, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gemini-3.5-flash-lite-high", "score": 71.82, "se": 3.67, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gemini-3.5-flash-lite-high", "score": 65, "se": 6.75, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-lite", "score": 1442.6, "se": 3.26, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "writing", "band": "medium", "score": 64.92, "se": 3.94, "evidence": -0.6631, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gpt-5.4-mini-xhigh", "score": 70.95, "se": 3.71, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gpt-5.4-mini-xhigh", "score": 58.88, "se": 6.96, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.4-mini-high", "score": 1433.91, "se": 2.78, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "writing", "band": "medium", "score": 66.25, "se": 3.9, "evidence": -0.7168, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:deepseek-v4-flash", "score": 70.12, "se": 3.74, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:deepseek-v4-flash", "score": 62.38, "se": 6.85, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-flash", "score": 1427.94, "se": 2.97, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "writing", "band": "medium", "score": 66.25, "se": 3.9, "evidence": -0.7168, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:deepseek-v4-flash", "score": 70.12, "se": 3.74, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:deepseek-v4-flash", "score": 62.38, "se": 6.85, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-flash", "score": 1427.94, "se": 2.97, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "writing", "band": "medium", "rating": 1421.21, "evidence": -0.818, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:mistral-medium-3.5", "score": 1421.21, "se": 5.11, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "writing", "band": "medium", "rating": 1414.99, "evidence": -1.0738, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-haiku-4-5-20251001", "score": 1414.99, "se": 1.92, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "writing", "band": "medium", "rating": 1403.29, "evidence": -1.422, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-4.1-2025-04-14", "score": 1403.29, "se": 2.94, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "writing", "band": "medium", "rating": 1400.45, "evidence": -1.4746, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:mistral-large-3", "score": 1400.45, "se": 2.28, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "writing", "band": "medium", "score": 64.58, "se": 3.88, "evidence": -1.7294, "sources": [ { "benchmark": "livebench:category/Language", "result": "livebench:category/Language:gpt-5.4-nano-xhigh", "score": 62.51, "se": 3.95, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/paraphrase", "result": "livebench:task/paraphrase:gpt-5.4-nano-xhigh", "score": 66.65, "se": 6.67, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.4-nano-high", "score": 1386.82, "se": 2.82, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "writing", "band": "medium", "rating": 1373.26, "evidence": -2.0607, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5-mini-high", "score": 1373.26, "se": 3.87, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "writing", "band": "medium", "rating": 1372.9, "evidence": -2.0714, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-4.1-mini-2025-04-14", "score": 1372.9, "se": 3.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "writing", "band": "medium", "rating": 1337.69, "evidence": -2.3977, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:mistral-small-2506", "score": 1337.69, "se": 4.55, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "writing", "band": "medium", "rating": 1323.37, "evidence": -2.5678, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-oss-120b", "score": 1323.37, "se": 3.61, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "writing", "band": "medium", "rating": 1293.1, "evidence": -2.8918, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-4o-mini-2024-07-18", "score": 1293.1, "se": 2.48, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "writing", "band": "medium", "rating": 1292.61, "evidence": -2.9013, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:llama-3.3-70b-instruct", "score": 1292.61, "se": 2.58, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "creative", "band": "medium", "score": 82.75, "se": 5.34, "evidence": 2.6239, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gemini-3.8-flash-high", "score": 82.75, "se": 5.34, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gemini-3.8-flash-high", "score": 1484.4, "se": 4.71, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "creative", "band": "medium", "score": 77.25, "se": 5.93, "evidence": 2.247, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gemini-3.1-pro-preview-high", "score": 77.25, "se": 5.93, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gemini-3.1-pro-preview", "score": 1480.34, "se": 2.8, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "creative", "band": "medium", "score": 78.1, "se": 5.85, "evidence": 2.2124, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:claude-fable-5-1-max-effort", "score": 78.1, "se": 5.85, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-fable-5.1-max", "score": 1481.64, "se": 6.56, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "creative", "band": "medium", "score": 80.6, "se": 5.59, "evidence": 2.1442, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:claude-opus-4-8-max-effort", "score": 80.6, "se": 5.59, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-opus-4-8", "score": 1463.47, "se": 3.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "creative", "band": "medium", "score": 70.92, "se": 6.42, "evidence": 1.9827, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:claude-opus-5-5-max-effort", "score": 70.92, "se": 6.42, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-opus-5.5-high", "score": 1514.98, "se": 10.78, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "creative", "band": "medium", "score": 75.2, "se": 6.11, "evidence": 1.7929, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gemini-3.5-flash-high", "score": 75.2, "se": 6.11, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gemini-3.5-flash-medium", "score": 1469.1, "se": 3.79, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "creative", "band": "medium", "score": 73.98, "se": 6.2, "evidence": 1.1725, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gpt-6-astra-max", "score": 73.98, "se": 6.2, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-6-astra-max", "score": 1448.27, "se": 7.45, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "creative", "band": "medium", "score": 72.55, "se": 6.31, "evidence": 1.1307, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gpt-5.5-xhigh", "score": 72.55, "se": 6.31, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-5.5", "score": 1448.03, "se": 3.36, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "creative", "band": "medium", "score": 79.02, "se": 5.76, "evidence": 1.0924, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gpt-5.4-xhigh", "score": 79.02, "se": 5.76, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-5.4", "score": 1434.78, "se": 3.48, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "creative", "band": "medium", "score": 61.32, "se": 6.89, "evidence": 0.971, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:claude-opus-5-max-effort", "score": 61.32, "se": 6.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-opus-5-high", "score": 1472.35, "se": 3.52, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "creative", "band": "medium", "score": 68.9, "se": 6.55, "evidence": 0.9549, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 68.9, "se": 6.55, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-sonnet-4-6", "score": 1449.38, "se": 3.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "creative", "band": "medium", "score": 74.92, "se": 6.13, "evidence": 0.9108, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:grok-4.7-xhigh", "score": 74.92, "se": 6.13, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:grok-4.7-xhigh", "score": 1432.9, "se": 8.83, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "creative", "band": "medium", "score": 67.75, "se": 6.61, "evidence": 0.7389, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:deepseek-v4-pro", "score": 67.75, "se": 6.61, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:deepseek-v4-pro", "score": 1446.13, "se": 3.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "creative", "band": "medium", "score": 67.75, "se": 6.61, "evidence": 0.7389, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:deepseek-v4-pro", "score": 67.75, "se": 6.61, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:deepseek-v4-pro", "score": 1446.13, "se": 3.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "creative", "band": "medium", "score": 67.47, "se": 6.63, "evidence": 0.4014, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gemini-3.5-flash-lite-high", "score": 67.47, "se": 6.63, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gemini-3.5-flash-lite", "score": 1435.4, "se": 4.12, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "creative", "band": "medium", "score": 62.32, "se": 6.85, "evidence": 0.1035, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:claude-sonnet-5-xhigh-effort", "score": 62.32, "se": 6.85, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-sonnet-5-high", "score": 1435.75, "se": 3.74, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "creative", "band": "medium", "score": 67.27, "se": 6.64, "evidence": -0.224, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:deepseek-v4-flash", "score": 67.27, "se": 6.64, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:deepseek-v4-flash", "score": 1407.77, "se": 3.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "creative", "band": "medium", "score": 67.27, "se": 6.64, "evidence": -0.224, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:deepseek-v4-flash", "score": 67.27, "se": 6.64, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:deepseek-v4-flash", "score": 1407.77, "se": 3.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "creative", "band": "medium", "score": 66.63, "se": 6.67, "evidence": -0.3367, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gpt-5.4-mini-xhigh", "score": 66.63, "se": 6.67, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-5.4-mini-high", "score": 1401.56, "se": 3.61, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "creative", "band": "medium", "rating": 1402.38, "evidence": -0.7775, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-4.1-2025-04-14", "score": 1402.38, "se": 3.93, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "creative", "band": "medium", "rating": 1400.31, "evidence": -0.8424, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:mistral-medium-3.5", "score": 1400.31, "se": 7.03, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "creative", "band": "medium", "rating": 1387.4, "evidence": -1.3012, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:claude-haiku-4-5-20251001", "score": 1387.4, "se": 2.46, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "creative", "band": "medium", "score": 67.65, "se": 6.62, "evidence": -1.3274, "sources": [ { "benchmark": "livebench:task/story_generation", "result": "livebench:task/story_generation:gpt-5.4-nano-xhigh", "score": 67.65, "se": 6.62, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-5.4-nano-high", "score": 1336.07, "se": 3.71, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "creative", "band": "medium", "rating": 1373.37, "evidence": -1.5517, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:mistral-large-3", "score": 1373.37, "se": 2.94, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "creative", "band": "medium", "rating": 1349.05, "evidence": -1.7853, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-4.1-mini-2025-04-14", "score": 1349.05, "se": 4.5, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "creative", "band": "medium", "rating": 1323.22, "evidence": -2.2327, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:mistral-small-2506", "score": 1323.22, "se": 6.63, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "creative", "band": "medium", "rating": 1323.71, "evidence": -2.236, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-5-mini-high", "score": 1323.71, "se": 5.34, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "creative", "band": "medium", "rating": 1293.93, "evidence": -2.6455, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-4o-mini-2024-07-18", "score": 1293.93, "se": 3.58, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "creative", "band": "medium", "rating": 1285.34, "evidence": -2.8098, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:llama-3.3-70b-instruct", "score": 1285.34, "se": 3.61, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "creative", "band": "medium", "rating": 1277.82, "evidence": -2.9241, "sources": [ { "benchmark": "lmarena:text_style_control/creative_writing", "result": "lmarena:text_style_control/creative_writing:gpt-oss-120b", "score": 1277.82, "se": 5.08, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "summarization", "band": "medium", "score": 88.23, "se": 4.56, "evidence": 2.045, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gemini-3.1-pro-preview-high", "score": 88.23, "se": 4.56, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "summarization", "band": "medium", "score": 87.5, "se": 4.68, "evidence": 1.9395, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gemini-3.8-flash-high", "score": 87.5, "se": 4.68, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "summarization", "band": "medium", "score": 82.58, "se": 5.36, "evidence": 1.2313, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gemini-3.5-flash-high", "score": 82.58, "se": 5.36, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "summarization", "band": "medium", "score": 79.55, "se": 5.7, "evidence": 0.8212, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gpt-6-astra-max", "score": 79.55, "se": 5.7, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "summarization", "band": "medium", "score": 78.17, "se": 5.84, "evidence": 0.6422, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:grok-4.7-xhigh", "score": 78.17, "se": 5.84, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "summarization", "band": "medium", "score": 77.67, "se": 5.89, "evidence": 0.5783, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:claude-opus-4-8-max-effort", "score": 77.67, "se": 5.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "summarization", "band": "medium", "score": 72.32, "se": 6.33, "evidence": -0.0754, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gpt-5.5-xhigh", "score": 72.32, "se": 6.33, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "summarization", "band": "medium", "score": 72.17, "se": 6.34, "evidence": -0.0931, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:claude-fable-5-1-max-effort", "score": 72.17, "se": 6.34, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "summarization", "band": "medium", "score": 71.77, "se": 6.37, "evidence": -0.1401, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gpt-5.4-nano-xhigh", "score": 71.77, "se": 6.37, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "summarization", "band": "medium", "score": 71.22, "se": 6.4, "evidence": -0.2046, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:claude-sonnet-5-xhigh-effort", "score": 71.22, "se": 6.4, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "summarization", "band": "medium", "score": 69.5, "se": 6.51, "evidence": -0.4037, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:deepseek-v4-pro", "score": 69.5, "se": 6.51, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "summarization", "band": "medium", "score": 69.5, "se": 6.51, "evidence": -0.4037, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:deepseek-v4-pro", "score": 69.5, "se": 6.51, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "summarization", "band": "medium", "score": 69.1, "se": 6.53, "evidence": -0.4498, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:claude-opus-5-5-max-effort", "score": 69.1, "se": 6.53, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "summarization", "band": "medium", "score": 68.85, "se": 6.55, "evidence": -0.4781, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:deepseek-v4-flash", "score": 68.85, "se": 6.55, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "summarization", "band": "medium", "score": 68.85, "se": 6.55, "evidence": -0.4781, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:deepseek-v4-flash", "score": 68.85, "se": 6.55, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "summarization", "band": "medium", "score": 68.47, "se": 6.57, "evidence": -0.5215, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gemini-3.5-flash-lite-high", "score": 68.47, "se": 6.57, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "summarization", "band": "medium", "score": 68.42, "se": 6.57, "evidence": -0.5274, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gpt-5.4-xhigh", "score": 68.42, "se": 6.57, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "summarization", "band": "medium", "score": 66.43, "se": 6.68, "evidence": -0.7521, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:claude-opus-5-max-effort", "score": 66.43, "se": 6.68, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "summarization", "band": "medium", "score": 61.65, "se": 6.88, "evidence": -1.2655, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 61.65, "se": 6.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "summarization", "band": "medium", "score": 59.63, "se": 6.94, "evidence": -1.4642, "sources": [ { "benchmark": "livebench:task/summarize", "result": "livebench:task/summarize:gpt-5.4-mini-xhigh", "score": 59.63, "se": 6.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "extraction", "band": "medium", "score": 91.49, "se": 1.61, "evidence": 2.4112, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gpt-6-astra-max", "score": 82.97, "se": 3.07, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gpt-6-astra-max", "score": 100, "se": 0.98, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-6-astra-max", "score": 1473.1, "se": 5.89, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "extraction", "band": "medium", "score": 90.79, "se": 1.66, "evidence": 2.3911, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gpt-5.5-xhigh", "score": 81.58, "se": 3.17, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gpt-5.5-xhigh", "score": 100, "se": 0.98, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.5", "score": 1475.13, "se": 2.69, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "extraction", "band": "medium", "score": 89.18, "se": 1.89, "evidence": 2.3542, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:claude-opus-5-5-max-effort", "score": 80.31, "se": 3.25, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:claude-opus-5-5-max-effort", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-opus-5.5-high", "score": 1514.42, "se": 8.3, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "extraction", "band": "medium", "score": 88.29, "se": 1.94, "evidence": 1.8696, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gemini-3.1-pro-preview-high", "score": 78.54, "se": 3.35, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gemini-3.1-pro-preview-high", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.1-pro-preview", "score": 1479.34, "se": 2.22, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "extraction", "band": "medium", "score": 87.2, "se": 2.31, "evidence": 1.8341, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:claude-fable-5-1-max-effort", "score": 80.28, "se": 3.25, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:claude-fable-5-1-max-effort", "score": 94.12, "se": 3.29, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-fable-5.1-max", "score": 1497.68, "se": 5.51, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "extraction", "band": "medium", "score": 89.66, "se": 1.73, "evidence": 1.7897, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gpt-5.4-xhigh", "score": 79.31, "se": 3.31, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gpt-5.4-xhigh", "score": 100, "se": 0.98, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.4", "score": 1460.12, "se": 2.75, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "extraction", "band": "medium", "score": 87.02, "se": 2.17, "evidence": 1.4711, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 77.95, "se": 3.39, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 96.08, "se": 2.72, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-sonnet-4-6", "score": 1474.33, "se": 2.65, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "extraction", "band": "medium", "score": 84.34, "se": 2.42, "evidence": 1.353, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:claude-opus-5-max-effort", "score": 74.55, "se": 3.56, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:claude-opus-5-max-effort", "score": 94.12, "se": 3.29, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-opus-5-high", "score": 1496.99, "se": 2.95, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "extraction", "band": "medium", "score": 88.44, "se": 1.79, "evidence": 1.2424, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:grok-4.7-xhigh", "score": 76.88, "se": 3.44, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:grok-4.7-xhigh", "score": 100, "se": 0.98, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:grok-4.7-xhigh", "score": 1441.89, "se": 6.93, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "extraction", "band": "medium", "score": 87.27, "se": 1.85, "evidence": 1.2032, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:deepseek-v4-pro", "score": 74.54, "se": 3.56, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:deepseek-v4-pro", "score": 100, "se": 0.98, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-pro", "score": 1452.39, "se": 2.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "extraction", "band": "medium", "score": 87.27, "se": 1.85, "evidence": 1.2032, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:deepseek-v4-pro", "score": 74.54, "se": 3.56, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:deepseek-v4-pro", "score": 100, "se": 0.98, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-pro", "score": 1452.39, "se": 2.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "extraction", "band": "medium", "score": 84.89, "se": 2.08, "evidence": 0.9564, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:claude-sonnet-5-xhigh-effort", "score": 71.74, "se": 3.68, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:claude-sonnet-5-xhigh-effort", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-sonnet-5-high", "score": 1466.87, "se": 2.97, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "extraction", "band": "medium", "score": 85.4, "se": 1.92, "evidence": 0.6648, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gpt-5.4-mini-xhigh", "score": 70.79, "se": 3.71, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gpt-5.4-mini-xhigh", "score": 100, "se": 0.98, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.4-mini-high", "score": 1433.91, "se": 2.78, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "extraction", "band": "medium", "score": 80.08, "se": 2.54, "evidence": 0.4927, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:claude-opus-4-8-max-effort", "score": 66.03, "se": 3.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:claude-opus-4-8-max-effort", "score": 94.12, "se": 3.29, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-opus-4-8", "score": 1478.51, "se": 2.73, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "extraction", "band": "medium", "score": 80.47, "se": 2.38, "evidence": 0.1455, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gemini-3.5-flash-high", "score": 64.86, "se": 3.9, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gemini-3.5-flash-high", "score": 96.08, "se": 2.72, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-medium", "score": 1465.5, "se": 2.95, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "extraction", "band": "medium", "score": 75.05, "se": 2.45, "evidence": 0.1335, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gemini-3.8-flash-high", "score": 54.01, "se": 4.07, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gemini-3.8-flash-high", "score": 96.08, "se": 2.72, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.8-flash-high", "score": 1487.18, "se": 3.75, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "extraction", "band": "medium", "score": 83.03, "se": 2.14, "evidence": -0.1281, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:deepseek-v4-flash", "score": 68.02, "se": 3.81, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:deepseek-v4-flash", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-flash", "score": 1427.94, "se": 2.97, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "extraction", "band": "medium", "score": 83.03, "se": 2.14, "evidence": -0.1281, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:deepseek-v4-flash", "score": 68.02, "se": 3.81, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:deepseek-v4-flash", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:deepseek-v4-flash", "score": 1427.94, "se": 2.97, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "extraction", "band": "medium", "score": 75.65, "se": 2.25, "evidence": -0.5594, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gemini-3.5-flash-lite-high", "score": 53.25, "se": 4.07, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gemini-3.5-flash-lite-high", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gemini-3.5-flash-lite", "score": 1442.6, "se": 3.26, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "extraction", "band": "medium", "rating": 1421.21, "evidence": -0.818, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:mistral-medium-3.5", "score": 1421.21, "se": 5.11, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "extraction", "band": "medium", "score": 82.84, "se": 2.14, "evidence": -1.021, "sources": [ { "benchmark": "livebench:category/Data Analysis", "result": "livebench:category/Data Analysis:gpt-5.4-nano-xhigh", "score": 67.64, "se": 3.82, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:task/tablereformat", "result": "livebench:task/tablereformat:gpt-5.4-nano-xhigh", "score": 98.04, "se": 1.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5.4-nano-high", "score": 1386.82, "se": 2.82, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "extraction", "band": "medium", "rating": 1414.99, "evidence": -1.0738, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:claude-haiku-4-5-20251001", "score": 1414.99, "se": 1.92, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "extraction", "band": "medium", "rating": 1403.29, "evidence": -1.422, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-4.1-2025-04-14", "score": 1403.29, "se": 2.94, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "extraction", "band": "medium", "rating": 1400.45, "evidence": -1.4746, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:mistral-large-3", "score": 1400.45, "se": 2.28, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "extraction", "band": "medium", "rating": 1373.26, "evidence": -2.0607, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-5-mini-high", "score": 1373.26, "se": 3.87, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "extraction", "band": "medium", "rating": 1372.9, "evidence": -2.0714, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-4.1-mini-2025-04-14", "score": 1372.9, "se": 3.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "extraction", "band": "medium", "rating": 1337.69, "evidence": -2.3977, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:mistral-small-2506", "score": 1337.69, "se": 4.55, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "extraction", "band": "medium", "rating": 1323.37, "evidence": -2.5678, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-oss-120b", "score": 1323.37, "se": 3.61, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "extraction", "band": "medium", "rating": 1293.1, "evidence": -2.8918, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:gpt-4o-mini-2024-07-18", "score": 1293.1, "se": 2.48, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "extraction", "band": "medium", "rating": 1292.61, "evidence": -2.9013, "sources": [ { "benchmark": "lmarena:text_style_control/instruction_following", "result": "lmarena:text_style_control/instruction_following:llama-3.3-70b-instruct", "score": 1292.61, "se": 2.58, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "classification", "band": "medium", "score": 81.41, "se": 2.75, "evidence": 2.4908, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gemini-3.8-flash-high", "score": 81.41, "se": 2.75, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "classification", "band": "medium", "score": 79.1, "se": 2.88, "evidence": 2.1835, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gemini-3.1-pro-preview-high", "score": 79.1, "se": 2.88, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "classification", "band": "medium", "score": 75.6, "se": 3.04, "evidence": 1.5559, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gemini-3.5-flash-high", "score": 75.6, "se": 3.04, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "classification", "band": "medium", "score": 75.58, "se": 3.04, "evidence": 1.5514, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gpt-6-astra-max", "score": 75.58, "se": 3.04, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "classification", "band": "medium", "score": 75.28, "se": 3.05, "evidence": 1.4818, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:grok-4.7-xhigh", "score": 75.28, "se": 3.05, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "classification", "band": "medium", "score": 72.99, "se": 3.14, "evidence": 0.9467, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:claude-fable-5-1-max-effort", "score": 72.99, "se": 3.14, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "classification", "band": "medium", "score": 72.03, "se": 3.17, "evidence": 0.7198, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:claude-opus-4-8-max-effort", "score": 72.03, "se": 3.17, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "classification", "band": "medium", "score": 70.73, "se": 3.22, "evidence": 0.4155, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gpt-5.5-xhigh", "score": 70.73, "se": 3.22, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "classification", "band": "medium", "score": 70.22, "se": 3.23, "evidence": 0.2976, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gpt-5.4-xhigh", "score": 70.22, "se": 3.23, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "classification", "band": "medium", "score": 67.24, "se": 3.32, "evidence": -0.3662, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gemini-3.5-flash-lite-high", "score": 67.24, "se": 3.32, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "classification", "band": "medium", "score": 67.2, "se": 3.32, "evidence": -0.3748, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gpt-5.4-nano-xhigh", "score": 67.2, "se": 3.32, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "classification", "band": "medium", "score": 65.74, "se": 3.36, "evidence": -0.6836, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:claude-opus-5-5-max-effort", "score": 65.74, "se": 3.36, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "classification", "band": "medium", "score": 63.86, "se": 3.4, "evidence": -1.0575, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:claude-sonnet-5-xhigh-effort", "score": 63.86, "se": 3.4, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "classification", "band": "medium", "score": 63.77, "se": 3.4, "evidence": -1.0756, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:claude-opus-5-max-effort", "score": 63.77, "se": 3.4, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "classification", "band": "medium", "score": 63.22, "se": 3.41, "evidence": -1.1843, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 63.22, "se": 3.41, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "classification", "band": "medium", "score": 63.14, "se": 3.41, "evidence": -1.2003, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:deepseek-v4-flash", "score": 63.14, "se": 3.41, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "classification", "band": "medium", "score": 63.14, "se": 3.41, "evidence": -1.2003, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:deepseek-v4-flash", "score": 63.14, "se": 3.41, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "classification", "band": "medium", "score": 62.35, "se": 3.43, "evidence": -1.3549, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:deepseek-v4-pro", "score": 62.35, "se": 3.43, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "classification", "band": "medium", "score": 62.35, "se": 3.43, "evidence": -1.3549, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:deepseek-v4-pro", "score": 62.35, "se": 3.43, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "classification", "band": "medium", "score": 59.8, "se": 3.47, "evidence": -1.7906, "sources": [ { "benchmark": "livebench:category/IF", "result": "livebench:category/IF:gpt-5.4-mini-xhigh", "score": 59.8, "se": 3.47, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "conversation", "band": "medium", "score": 73.5, "se": 1.4, "evidence": 3.3414, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gemini-3.1-pro-preview_high", "score": 73.5, "se": 1.3963164754810065, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gemini-3.1-pro-preview", "score": 1496.48, "se": 2.74, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "conversation", "band": "medium", "score": 75.6, "se": 1.36, "evidence": 3.0952, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-6-astra_max", "score": 75.6, "se": 1.3588548437881387, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-6-astra-max", "score": 1488.7, "se": 8.88, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "conversation", "band": "medium", "score": 69.7, "se": 1.45, "evidence": 3.0565, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gemini-3.8-flash_high", "score": 69.7, "se": 1.4539683710535192, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gemini-3.8-flash-high", "score": 1501.17, "se": 5.23, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "conversation", "band": "medium", "score": 72.2, "se": 1.42, "evidence": 2.9415, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-opus-5-5_max", "score": 72.2, "se": 1.4174516461485274, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-opus-5.5-high", "score": 1496.84, "se": 13.39, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "conversation", "band": "medium", "score": 70.8, "se": 1.44, "evidence": 2.6739, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-fable-5-1_max", "score": 70.8, "se": 1.4385511563477433, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-fable-5.1-max", "score": 1486.89, "se": 8.05, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "conversation", "band": "medium", "score": 66.2, "se": 1.5, "evidence": 2.1147, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gemini-3.5-flash_high", "score": 66.2, "se": 1.4965960710224384, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gemini-3.5-flash-high", "score": 1480.81, "se": 3.79, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "conversation", "band": "medium", "score": 63, "se": 1.53, "evidence": 2.0384, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-5.5_xhigh", "score": 63, "se": 1.5275252316519465, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-5.5", "score": 1484.17, "se": 3.37, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "conversation", "band": "medium", "score": 53, "se": 1.58, "evidence": 2.0162, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-opus-4-8_max", "score": 53, "se": 1.5790799515836722, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-opus-4-8", "score": 1494.54, "se": 3.43, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "conversation", "band": "medium", "score": 59.9, "se": 1.55, "evidence": 1.995, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-opus-5_max", "score": 59.9, "se": 1.550610974549837, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-opus-5-max", "score": 1485.96, "se": 4.92, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "conversation", "band": "medium", "score": 52.91, "se": 1.58, "evidence": 1.1135, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:deepseek-v4-pro-0813_max", "score": 52.91, "se": 1.5808394665706598, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:deepseek-v4-pro", "score": 1474.99, "se": 3.58, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "conversation", "band": "medium", "score": 52.91, "se": 1.58, "evidence": 1.1135, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:deepseek-v4-pro-0813_max", "score": 52.91, "se": 1.5808394665706598, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:deepseek-v4-pro", "score": 1474.99, "se": 3.58, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "conversation", "band": "medium", "score": 45.1, "se": 1.57, "evidence": 1.0502, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-5.4-2026-03-05_xhigh", "score": 45.1, "se": 1.5743152379585443, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-5.4", "score": 1480.21, "se": 3.35, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "conversation", "band": "medium", "score": 32.8, "se": 1.49, "evidence": 0.5803, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-sonnet-4-6_max", "score": 32.8, "se": 1.4853842487270312, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-sonnet-4-6", "score": 1481.56, "se": 3.28, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "conversation", "band": "medium", "score": 33.7, "se": 1.5, "evidence": 0.2186, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-sonnet-5_max", "score": 33.7, "se": 1.495508791865346, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-sonnet-5-high", "score": 1473.54, "se": 3.87, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "conversation", "band": "medium", "rating": 1464.55, "evidence": 0.1278, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gemini-3.5-flash-lite", "score": 1464.55, "se": 4.36, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "conversation", "band": "medium", "score": 33.63, "se": 1.5, "evidence": -0.4802, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:deepseek-v4-flash-0731_max", "score": 33.63, "se": 1.495531772489948, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:deepseek-v4-flash", "score": 1453.21, "se": 3.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "conversation", "band": "medium", "score": 33.63, "se": 1.5, "evidence": -0.4802, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:deepseek-v4-flash-0731_max", "score": 33.63, "se": 1.495531772489948, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:deepseek-v4-flash", "score": 1453.21, "se": 3.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "conversation", "band": "medium", "score": 29.4, "se": 1.44, "evidence": -0.492, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-5.4-mini-2026-03-17_high", "score": 29.4, "se": 1.4414290540008265, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-5.4-mini-high", "score": 1465.49, "se": 3.41, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "conversation", "band": "medium", "rating": 1432.88, "evidence": -0.9942, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:mistral-medium-3.5", "score": 1432.88, "se": 7.15, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "conversation", "band": "medium", "score": 31.1, "se": 1.46, "evidence": -1.1605, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-4.1-2025-04-14", "score": 31.1, "se": 1.4645596385722692, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-4.1-2025-04-14", "score": 1430.17, "se": 3.51, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "conversation", "band": "medium", "rating": 1424.72, "evidence": -1.2058, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:grok-4.7-xhigh", "score": 1424.72, "se": 10.56, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "conversation", "band": "medium", "rating": 1419.64, "evidence": -1.3967, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:mistral-large-3", "score": 1419.64, "se": 2.91, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "conversation", "band": "medium", "score": 13.2, "se": 1.07, "evidence": -2.0879, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:claude-haiku-4-5-20251001", "score": 13.2, "se": 1.0709373963527966, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:claude-haiku-4-5-20251001", "score": 1424.96, "se": 2.37, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "conversation", "band": "medium", "rating": 1364.93, "evidence": -2.3115, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:mistral-small-2506", "score": 1364.93, "se": 5.49, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "conversation", "band": "medium", "score": 11.7, "se": 1.02, "evidence": -2.4702, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-5.4-nano-2026-03-17_high", "score": 11.7, "se": 1.0169287802713345, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-5.4-nano-high", "score": 1413.3, "se": 3.46, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "conversation", "band": "medium", "score": 21.6, "se": 1.3, "evidence": -2.4777, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-5-mini-2025-08-07_high", "score": 21.6, "se": 1.3019735539307762, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-5-mini-high", "score": 1372.45, "se": 4.79, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "conversation", "band": "medium", "rating": 1326.76, "evidence": -2.6343, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-oss-120b", "score": 1326.76, "se": 4.47, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "conversation", "band": "medium", "score": 12.7, "se": 1.05, "evidence": -2.674, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-4.1-mini-2025-04-14", "score": 12.7, "se": 1.0534798620855645, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-4.1-mini-2025-04-14", "score": 1393.42, "se": 3.9, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "conversation", "band": "medium", "rating": 1316.53, "evidence": -2.8168, "sources": [ { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:llama-3.3-70b-instruct", "score": 1316.53, "se": 3.41, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "conversation", "band": "medium", "score": 8.3, "se": 0.87, "evidence": -3.7948, "sources": [ { "benchmark": "epoch:simpleqa_verified", "result": "epoch:simpleqa_verified:gpt-4o-mini-2024-07-18", "score": 8.3, "se": 0.8728527206074758, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/multi_turn", "result": "lmarena:text_style_control/multi_turn:gpt-4o-mini-2024-07-18", "score": 1311.05, "se": 3.44, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "analysis", "band": "medium", "score": 80.6, "se": 2.8, "evidence": 2.5747, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:claude-opus-5_unknown", "score": 80.6, "se": 2.8, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "analysis", "band": "medium", "score": 79.6, "se": 2.85, "evidence": 2.4913, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:gemini-3.1-pro-preview", "score": 79.6, "se": 2.85, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "analysis", "band": "medium", "score": 76.7, "se": 2.99, "evidence": 2.2267, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:gemini-3.5-flash_unknown", "score": 76.7, "se": 2.99, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "analysis", "band": "medium", "score": 69, "se": 3.27, "evidence": 1.1754, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:gpt-5.5_unknown", "score": 69, "se": 3.27, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "analysis", "band": "medium", "score": 64.8, "se": 3.38, "evidence": 0.6559, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:claude-opus-4-8_unknown", "score": 64.8, "se": 3.38, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "analysis", "band": "medium", "score": 61.1, "se": 3.45, "evidence": 0.3062, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:deepseek-v4-flash-0731_unknown", "score": 61.1, "se": 3.45, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "analysis", "band": "medium", "score": 61.1, "se": 3.45, "evidence": 0.3062, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:deepseek-v4-flash-0731_unknown", "score": 61.1, "se": 3.45, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "analysis", "band": "medium", "score": 60.6, "se": 3.46, "evidence": 0.2635, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:claude-sonnet-5_unknown", "score": 60.6, "se": 3.46, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "analysis", "band": "medium", "score": 27, "se": 3.14, "evidence": -1.4277, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:gpt-4.1-2025-04-14", "score": 27, "se": 3.14, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "analysis", "band": "medium", "score": 22.5, "se": 2.95, "evidence": -1.7759, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:mistral-large-2407", "score": 22.5, "se": 2.95, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "analysis", "band": "medium", "score": 22.1, "se": 2.93, "evidence": -1.808, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:gpt-oss-120b", "score": 22.1, "se": 2.93, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "analysis", "band": "medium", "score": 19.9, "se": 2.82, "evidence": -2.0236, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:Llama-3.3-70B-Instruct", "score": 19.9, "se": 2.82, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "analysis", "band": "medium", "score": 10.7, "se": 2.19, "evidence": -2.9647, "sources": [ { "benchmark": "epoch:simplebench_external", "result": "epoch:simplebench_external:gpt-4o-mini-2024-07-18", "score": 10.7, "se": 2.19, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "analysis", "band": "hard", "score": 91.37, "se": 1.36, "evidence": 3.5239, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:claude-opus-5-5_max", "score": 90.59, "se": 1.9434397190192136, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:claude-opus-5-5-max-effort", "score": 92.15, "se": 1.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-opus-5.5-high", "score": 1533.24, "se": 6.17, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "analysis", "band": "hard", "score": 94.21, "se": 1.15, "evidence": 3.4644, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-6-astra_max", "score": 95.77, "se": 1.3696812097020312, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gpt-6-astra-max", "score": 92.65, "se": 1.84, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-6-astra-max", "score": 1501.07, "se": 4.36, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "analysis", "band": "hard", "score": 92.34, "se": 1.3, "evidence": 3.3752, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gemini-3.8-flash_high", "score": 95.39, "se": 1.401136318210652, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gemini-3.8-flash-high", "score": 89.29, "se": 2.18, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gemini-3.8-flash-high", "score": 1515.06, "se": 3.14, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "analysis", "band": "hard", "score": 92.22, "se": 1.35, "evidence": 3.3463, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:claude-opus-5", "score": 92.93, "se": 1.8263105420199484, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:claude-opus-5-max-effort", "score": 91.21, "se": 1.99, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-opus-5-high", "score": 1515.88, "se": 2.37, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "analysis", "band": "hard", "score": 91.69, "se": 1.94, "evidence": 2.9054, "sources": [ { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:claude-fable-5-1-max-effort", "score": 91.69, "se": 1.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-fable-5.1-max", "score": 1521.51, "se": 4.03, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "analysis", "band": "hard", "score": 89.46, "se": 1.5, "evidence": 2.6119, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gemini-3.1-pro-preview", "score": 94.1, "se": 1.7000000000000002, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gemini-3.1-pro-preview-high", "score": 84, "se": 2.58, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gemini-3.1-pro-preview", "score": 1507.28, "se": 1.87, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "analysis", "band": "hard", "score": 86.93, "se": 1.73, "evidence": 2.3282, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:claude-opus-4-8_none", "score": 85.35, "se": 2.5190921114603886, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:claude-opus-4-8-max-effort", "score": 89.19, "se": 2.18, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-opus-4-8", "score": 1505.25, "se": 2.33, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "analysis", "band": "hard", "score": 85.78, "se": 1.73, "evidence": 1.6176, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gemini-3.5-flash_low", "score": 88.89, "se": 2.239078763821682, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gemini-3.5-flash-high", "score": 82, "se": 2.7, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gemini-3.5-flash-high", "score": 1494.76, "se": 2.35, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "analysis", "band": "hard", "score": 82.37, "se": 1.96, "evidence": 1.4793, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-5.5_none", "score": 77.27, "se": 2.9857515673386437, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gpt-5.5-xhigh", "score": 89.65, "se": 2.14, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-5.5", "score": 1499.27, "se": 2.23, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "analysis", "band": "hard", "score": 84.49, "se": 1.8, "evidence": 1.4387, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:claude-sonnet-5_max", "score": 80.3, "se": 2.8335609732463336, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:claude-sonnet-5-xhigh-effort", "score": 88.69, "se": 2.23, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-sonnet-5-high", "score": 1490.7, "se": 2.5, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "analysis", "band": "hard", "score": 81.78, "se": 1.93, "evidence": 1.3207, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:claude-sonnet-4-6_max", "score": 78.79, "se": 2.9126522834586774, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 84.77, "se": 2.53, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-sonnet-4-6", "score": 1503.4, "se": 2.19, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "analysis", "band": "hard", "score": 80.26, "se": 2.05, "evidence": 0.8132, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-5.4-2026-03-05_none", "score": 74.75, "se": 3.095405547036587, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gpt-5.4-xhigh", "score": 88.12, "se": 2.28, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-5.4", "score": 1486.84, "se": 2.28, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "qwen3.5-9b", "task": "analysis", "band": "hard", "score": 78.98, "se": 2.37, "evidence": 0.3085, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:qwen3.5-9B", "score": 78.98, "se": 2.365568030958479, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "analysis", "band": "hard", "score": 79, "se": 2.02, "evidence": 0.265, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:deepseek-v4-flash-0731_max", "score": 91.04, "se": 1.7655466909172535, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:deepseek-v4-flash", "score": 70.58, "se": 3.21, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:deepseek-v4-flash", "score": 1458.77, "se": 2.43, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "analysis", "band": "hard", "score": 79, "se": 2.02, "evidence": 0.265, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:deepseek-v4-flash-0731_max", "score": 91.04, "se": 1.7655466909172535, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:deepseek-v4-flash", "score": 70.58, "se": 3.21, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:deepseek-v4-flash", "score": 1458.77, "se": 2.43, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "analysis", "band": "hard", "score": 77.96, "se": 2.06, "evidence": 0.0675, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:deepseek-v4-pro_none", "score": 73.23, "se": 3.1544498882702823, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:deepseek-v4-pro", "score": 82.69, "se": 2.66, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:deepseek-v4-pro", "score": 1480.17, "se": 2.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "analysis", "band": "hard", "score": 77.96, "se": 2.06, "evidence": 0.0675, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:deepseek-v4-pro_none", "score": 73.23, "se": 3.1544498882702823, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:deepseek-v4-pro", "score": 82.69, "se": 2.66, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:deepseek-v4-pro", "score": 1480.17, "se": 2.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "analysis", "band": "hard", "score": 82.65, "se": 2.66, "evidence": 0.0017, "sources": [ { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:grok-4.7-xhigh", "score": 82.65, "se": 2.66, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:grok-4.7-xhigh", "score": 1461.97, "se": 5.24, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "analysis", "band": "hard", "score": 68.73, "se": 2.28, "evidence": -0.8031, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gemini-3.5-flash-lite_low", "score": 75.76, "se": 3.053289223393202, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gemini-3.5-flash-lite-high", "score": 60.19, "se": 3.44, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gemini-3.5-flash-lite", "score": 1473.61, "se": 2.69, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "analysis", "band": "hard", "score": 71.72, "se": 3.21, "evidence": -1.3835, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-5-mini-2025-08-07_minimal", "score": 71.72, "se": 3.2087795587867514, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-5-mini-high", "score": 1403.99, "se": 3.11, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "analysis", "band": "hard", "score": 75.76, "se": 2.74, "evidence": -1.5761, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:openai/gpt-oss-120b_high", "score": 75.76, "se": 2.7437163443883765, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-oss-120b", "score": 1361.94, "se": 2.88, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "analysis", "band": "hard", "score": 66.92, "se": 2.82, "evidence": -1.6638, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-4.1-2025-04-14", "score": 66.92, "se": 2.821440093855908, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-4.1-2025-04-14", "score": 1432.35, "se": 2.51, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "analysis", "band": "hard", "score": 67.1, "se": 2.4, "evidence": -1.7314, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-5.4-mini-2026-03-17_none", "score": 64.14, "se": 3.416903640391528, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gpt-5.4-mini-xhigh", "score": 71.32, "se": 3.18, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-5.4-mini-high", "score": 1468.69, "se": 2.29, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "analysis", "band": "hard", "score": 59.53, "se": 2.82, "evidence": -1.9953, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:mistral-medium-2505", "score": 59.53, "se": 2.824697229654645, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:mistral-medium-3.5", "score": 1445.62, "se": 3.92, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "analysis", "band": "hard", "score": 60.48, "se": 2.82, "evidence": -2.005, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:claude-haiku-4-5-20251001", "score": 60.48, "se": 2.815453502302083, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:claude-haiku-4-5-20251001", "score": 1440.7, "se": 1.56, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "analysis", "band": "hard", "score": 65.85, "se": 2.71, "evidence": -2.0579, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-4.1-mini-2025-04-14", "score": 65.85, "se": 2.7051500965295916, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-4.1-mini-2025-04-14", "score": 1402.33, "se": 2.91, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "analysis", "band": "hard", "score": 66.08, "se": 2.37, "evidence": -2.3969, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-5.4-nano-2026-03-17_none", "score": 55.56, "se": 3.5402943770953677, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Reasoning", "result": "livebench:category/Reasoning:gpt-5.4-nano-xhigh", "score": 81.1, "se": 2.75, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-5.4-nano-high", "score": 1421.19, "se": 2.32, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "analysis", "band": "hard", "score": 51.33, "se": 2.69, "evidence": -2.7322, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:mistral-large-2411", "score": 51.33, "se": 2.6887818295509085, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:mistral-large-3", "score": 1431.66, "se": 1.84, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "analysis", "band": "hard", "score": 47.47, "se": 2.76, "evidence": -3.3263, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:mistral-small-2503", "score": 47.47, "se": 2.761649514001224, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:mistral-small-2506", "score": 1374.01, "se": 3.53, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "analysis", "band": "hard", "score": 47.44, "se": 2.85, "evidence": -3.5597, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:Llama-3.3-70B-Instruct", "score": 47.44, "se": 2.8523027562061025, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:llama-3.3-70b-instruct", "score": 1320.67, "se": 2.55, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "analysis", "band": "hard", "score": 37.72, "se": 2.41, "evidence": -3.9688, "sources": [ { "benchmark": "epoch:gpqa_diamond", "result": "epoch:gpqa_diamond:gpt-4o-mini-2024-07-18", "score": 37.72, "se": 2.4068925424620136, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/hard_prompts", "result": "lmarena:text_style_control/hard_prompts:gpt-4o-mini-2024-07-18", "score": 1310.92, "se": 2.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "analysis", "band": "frontier", "score": 92.92, "se": 1.52, "evidence": 4.2112, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:claude-opus-5_high", "score": 88.33, "se": 2.93, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:claude-opus-5_high", "score": 97.5, "se": 0.78, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-opus-5-high", "score": 1542.36, "se": 4.36, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "analysis", "band": "frontier", "score": 94.59, "se": 1.32, "evidence": 4.0254, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:claude-opus-5-5_max", "score": 91.67, "se": 2.52, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:claude-opus-5-5_max", "score": 97.5, "se": 0.78, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-opus-5.5-high", "score": 1538.97, "se": 14.22, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "analysis", "band": "frontier", "score": 73.18, "se": 1.55, "evidence": 3.2917, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:claude-fable-5-1_low", "score": 78.33, "se": 3.76, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:claude-fable-5-1_low", "score": 90, "se": 1.5, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:claude-fable-5-1_xhigh", "score": 46.5, "se": 2, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-fable-5.1-max", "score": 1519.26, "se": 9.89, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "analysis", "band": "frontier", "score": 73.84, "se": 1.46, "evidence": 3.2854, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gemini-3.1-pro-preview", "score": 77.08, "se": 3.84, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gemini-3.1-pro-preview", "score": 98, "se": 0.7, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:gemini-3.1-pro-preview", "score": 46.44, "se": 1.96, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gemini-3.1-pro-preview", "score": 1508.55, "se": 3.27, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "analysis", "band": "frontier", "score": 66.79, "se": 1.73, "evidence": 2.9193, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-6-astra_none", "score": 59.58, "se": 4.48, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-6-astra_none", "score": 86, "se": 1.73, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:gpt-6-astra_unknown", "score": 54.8, "se": 1.94, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-6-astra-max", "score": 1513.04, "se": 10.46, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "analysis", "band": "frontier", "score": 75.11, "se": 2.36, "evidence": 2.8185, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:claude-opus-4-8_low", "score": 62.22, "se": 4.43, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:claude-opus-4-8_low", "score": 88, "se": 1.62, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-opus-4-8", "score": 1516.58, "se": 3.98, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "analysis", "band": "frontier", "score": 44.52, "se": 1.96, "evidence": 2.3553, "sources": [ { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:gemini-3.8-flash_unknown", "score": 44.52, "se": 1.96, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gemini-3.8-flash-high", "score": 1524.79, "se": 5.8, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "analysis", "band": "frontier", "score": 73.46, "se": 2.39, "evidence": 2.3277, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:claude-sonnet-4-6_high", "score": 60.42, "se": 4.46, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:claude-sonnet-4-6_high", "score": 86.5, "se": 1.71, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-sonnet-4-6", "score": 1506.29, "se": 4.04, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "analysis", "band": "frontier", "score": 54.75, "se": 2.4, "evidence": 2.0828, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-5.5_low", "score": 33.33, "se": 4.3, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-5.5_low", "score": 76.17, "se": 2.13, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-5.5", "score": 1511.44, "se": 3.98, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "analysis", "band": "frontier", "rating": 1511.75, "evidence": 1.8388, "sources": [ { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-sonnet-5-high", "score": 1511.75, "se": 4.54, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "analysis", "band": "frontier", "score": 45.04, "se": 1.77, "evidence": 1.1894, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-5.4-2026-03-05_low", "score": 29.17, "se": 4.15, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-5.4-2026-03-05_low", "score": 68.17, "se": 2.33, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:gpt-5.4-2026-03-05_xhigh", "score": 36.24, "se": 1.8800000000000001, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-5.4", "score": 1492.04, "se": 4.21, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "analysis", "band": "frontier", "score": 28.86, "se": 1.8, "evidence": 0.9036, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gemini-3.5-flash_minimal", "score": 8.89, "se": 2.6, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gemini-3.5-flash_minimal", "score": 48.83, "se": 2.5, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gemini-3.5-flash-high", "score": 1492.29, "se": 4.42, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "analysis", "band": "frontier", "rating": 1458.63, "evidence": -0.4622, "sources": [ { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:grok-4.7-xhigh", "score": 1458.63, "se": 12.2, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "analysis", "band": "frontier", "score": 9.26, "se": 1.09, "evidence": -0.7295, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gemini-3.5-flash-lite_low", "score": 1.53, "se": 1.12, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gemini-3.5-flash-lite_low", "score": 17, "se": 1.88, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gemini-3.5-flash-lite", "score": 1469.22, "se": 5.03, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "analysis", "band": "frontier", "score": 6.92, "se": 0.94, "evidence": -0.8202, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:deepseek-v4-pro-0813_none", "score": 0.83, "se": 0.83, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:deepseek-v4-pro-0813_none", "score": 13, "se": 1.68, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:deepseek-v4-pro", "score": 1479.66, "se": 4.3, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "analysis", "band": "frontier", "score": 6.92, "se": 0.94, "evidence": -0.8202, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:deepseek-v4-pro-0813_none", "score": 0.83, "se": 0.83, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:deepseek-v4-pro-0813_none", "score": 13, "se": 1.68, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:deepseek-v4-pro", "score": 1479.66, "se": 4.3, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "analysis", "band": "frontier", "score": 7.06, "se": 0.97, "evidence": -0.8215, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-5.4-mini-2026-03-17_low", "score": 1.11, "se": 0.96, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-5.4-mini-2026-03-17_low", "score": 13, "se": 1.68, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-5.4-mini-high", "score": 1480.17, "se": 4.22, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "analysis", "band": "frontier", "score": 6.96, "se": 1.04, "evidence": -1.2837, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:deepseek-v4-flash-0731_none", "score": 2.08, "se": 1.3, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:deepseek-v4-flash-0731_none", "score": 11.83, "se": 1.62, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:deepseek-v4-flash", "score": 1457.05, "se": 4.48, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "analysis", "band": "frontier", "score": 6.96, "se": 1.04, "evidence": -1.2837, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:deepseek-v4-flash-0731_none", "score": 2.08, "se": 1.3, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:deepseek-v4-flash-0731_none", "score": 11.83, "se": 1.62, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:deepseek-v4-flash", "score": 1457.05, "se": 4.48, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "analysis", "band": "frontier", "score": 7.79, "se": 1.01, "evidence": -1.38, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:claude-haiku-4-5-20251001", "score": 1.25, "se": 1.01, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:claude-haiku-4-5-20251001", "score": 14.33, "se": 1.75, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:claude-haiku-4-5-20251001", "score": 1451.44, "se": 3.09, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "analysis", "band": "frontier", "score": 9.93, "se": 1.12, "evidence": -1.3853, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-5.4-nano-2026-03-17_low", "score": 1.53, "se": 1.12, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-5.4-nano-2026-03-17_low", "score": 18.33, "se": 1.93, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-5.4-nano-high", "score": 1437.48, "se": 4.3, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "analysis", "band": "frontier", "rating": 1427.35, "evidence": -1.423, "sources": [ { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:mistral-large-3", "score": 1427.35, "se": 3.79, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "analysis", "band": "frontier", "score": 4.52, "se": 0.81, "evidence": -1.569, "sources": [ { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:mistral-medium-2505", "score": 4.52, "se": 0.8100000000000002, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:mistral-medium-3.5", "score": 1434.51, "se": 8.79, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "analysis", "band": "frontier", "score": 15.75, "se": 0.94, "evidence": -1.7641, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-5-mini-2025-08-07_low", "score": 0.83, "se": 0.83, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-5-mini-2025-08-07_low", "score": 26.33, "se": 2.2, "unit": "percent", "weight": 0.425, "settings": "close" }, { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:gpt-5-mini-2025-08-07_unknown", "score": 19.44, "se": 1.5500000000000003, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-5-mini-high", "score": 1400.98, "se": 9.01, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "analysis", "band": "frontier", "rating": 1356.91, "evidence": -2.4067, "sources": [ { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-oss-120b", "score": 1356.91, "se": 8.18, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "analysis", "band": "frontier", "rating": 1334.62, "evidence": -2.542, "sources": [ { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:mistral-small-2506", "score": 1334.62, "se": 10.08, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "analysis", "band": "frontier", "rating": 1297.53, "evidence": -2.8421, "sources": [ { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:llama-3.3-70b-instruct", "score": 1297.53, "se": 5.73, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "analysis", "band": "frontier", "score": 3.77, "se": 0.52, "evidence": -3.1039, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-4.1-2025-04-14", "score": 0.42, "se": 0.59, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-4.1-2025-04-14", "score": 5.5, "se": 1.14, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:hle_external", "result": "epoch:hle_external:gpt-4.1-2025-04-14", "score": 5.4, "se": 0.89, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-4.1-2025-04-14", "score": 1405.36, "se": 6, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "analysis", "band": "frontier", "score": 1.75, "se": 0.51, "evidence": -3.2756, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-4.1-mini-2025-04-14", "score": 0, "se": 0.42, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:arc_agi_external", "result": "epoch:arc_agi_external:gpt-4.1-mini-2025-04-14", "score": 3.5, "se": 0.92, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-4.1-mini-2025-04-14", "score": 1383.41, "se": 6.66, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "analysis", "band": "frontier", "score": 0, "se": 0.42, "evidence": -3.3363, "sources": [ { "benchmark": "epoch:arc_agi_2_external", "result": "epoch:arc_agi_2_external:gpt-4o-mini-2024-07-18", "score": 0, "se": 0.42, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/expert", "result": "lmarena:text_style_control/expert:gpt-4o-mini-2024-07-18", "score": 1284.94, "se": 5.55, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "math", "band": "medium", "rating": 1522.49, "evidence": 2.7023, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-opus-5-high", "score": 1522.49, "se": 6.18, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "math", "band": "medium", "rating": 1516.35, "evidence": 2.5376, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gemini-3.8-flash-high", "score": 1516.35, "se": 8.87, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "math", "band": "medium", "rating": 1517.46, "evidence": 2.4924, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-fable-5.1-max", "score": 1517.46, "se": 12.98, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "math", "band": "medium", "rating": 1510.21, "evidence": 2.1916, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-opus-5.5-high", "score": 1510.21, "se": 19.84, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "math", "band": "medium", "rating": 1499.39, "evidence": 2.1503, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-5.5", "score": 1499.39, "se": 5.39, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "math", "band": "medium", "rating": 1496.34, "evidence": 2.0288, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gemini-3.5-flash-high", "score": 1496.34, "se": 6.41, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "math", "band": "medium", "rating": 1490, "evidence": 1.7876, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-6-astra-max", "score": 1490, "se": 14.15, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "math", "band": "medium", "rating": 1487.87, "evidence": 1.7496, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gemini-3.1-pro-preview", "score": 1487.87, "se": 4.17, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "math", "band": "medium", "rating": 1476.25, "evidence": 1.3243, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-opus-4-8", "score": 1476.25, "se": 5.64, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "math", "band": "medium", "rating": 1473.61, "evidence": 1.2434, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-sonnet-5-high", "score": 1473.61, "se": 6.76, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "math", "band": "medium", "rating": 1463.24, "evidence": 0.8705, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-sonnet-4-6", "score": 1463.24, "se": 5.14, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "math", "band": "medium", "rating": 1459.91, "evidence": 0.7315, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-5.4", "score": 1459.91, "se": 5.27, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "math", "band": "medium", "rating": 1444.85, "evidence": 0.096, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:grok-4.7-xhigh", "score": 1444.85, "se": 17.37, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "math", "band": "medium", "rating": 1444.76, "evidence": 0.079, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:deepseek-v4-pro", "score": 1444.76, "se": 5.68, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "math", "band": "medium", "rating": 1444.76, "evidence": 0.079, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:deepseek-v4-pro", "score": 1444.76, "se": 5.68, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "math", "band": "medium", "rating": 1441.85, "evidence": -0.0499, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gemini-3.5-flash-lite", "score": 1441.85, "se": 7.6, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "math", "band": "medium", "rating": 1438.82, "evidence": -0.1686, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-5.4-mini-high", "score": 1438.82, "se": 5.48, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "math", "band": "medium", "rating": 1428.35, "evidence": -0.5263, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:mistral-medium-3.5", "score": 1428.35, "se": 12.31, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "math", "band": "medium", "rating": 1424.75, "evidence": -0.6794, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:deepseek-v4-flash", "score": 1424.75, "se": 6.02, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "math", "band": "medium", "rating": 1424.75, "evidence": -0.6794, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:deepseek-v4-flash", "score": 1424.75, "se": 6.02, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "math", "band": "medium", "rating": 1424.73, "evidence": -0.6906, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-5.4-nano-high", "score": 1424.73, "se": 5.53, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "math", "band": "medium", "rating": 1405.04, "evidence": -1.3611, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-5-mini-high", "score": 1405.04, "se": 7.94, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "math", "band": "medium", "rating": 1401.5, "evidence": -1.524, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:mistral-large-3", "score": 1401.5, "se": 4.64, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "math", "band": "medium", "rating": 1400.23, "evidence": -1.5534, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:claude-haiku-4-5-20251001", "score": 1400.23, "se": 3.65, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "math", "band": "medium", "rating": 1381.12, "evidence": -1.9778, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-oss-120b", "score": 1381.12, "se": 7, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "math", "band": "medium", "rating": 1374.56, "evidence": -2.1042, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-4.1-2025-04-14", "score": 1374.56, "se": 5.25, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "math", "band": "medium", "rating": 1353.43, "evidence": -2.4232, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-4.1-mini-2025-04-14", "score": 1353.43, "se": 5.69, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "math", "band": "medium", "rating": 1337.98, "evidence": -2.533, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:mistral-small-2506", "score": 1337.98, "se": 8.96, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "math", "band": "medium", "rating": 1295.83, "evidence": -2.7931, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:llama-3.3-70b-instruct", "score": 1295.83, "se": 3.98, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "math", "band": "medium", "rating": 1275.35, "evidence": -3, "sources": [ { "benchmark": "lmarena:text_style_control/math", "result": "lmarena:text_style_control/math:gpt-4o-mini-2024-07-18", "score": 1275.35, "se": 3.6, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "math", "band": "hard", "score": 97.08, "se": 0.87, "evidence": 4.6644, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:claude-opus-5-5-max-effort", "score": 97.08, "se": 0.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "math", "band": "hard", "score": 97.01, "se": 0.88, "evidence": 4.6233, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:claude-fable-5-1-max-effort", "score": 97.01, "se": 0.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "math", "band": "hard", "score": 96.81, "se": 0.9, "evidence": 4.5205, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gpt-6-astra-max", "score": 96.81, "se": 0.9, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "math", "band": "hard", "score": 95.86, "se": 1.02, "evidence": 3.9727, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gpt-5.5-xhigh", "score": 95.86, "se": 1.02, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "math", "band": "hard", "score": 95.73, "se": 1.04, "evidence": 3.8878, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:claude-opus-5-max-effort", "score": 95.73, "se": 1.04, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "math", "band": "hard", "score": 95.68, "se": 1.04, "evidence": 3.8662, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:grok-4.7-xhigh", "score": 95.68, "se": 1.04, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "math", "band": "hard", "score": 94.32, "se": 1.19, "evidence": 3.1144, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:claude-opus-4-8-max-effort", "score": 94.32, "se": 1.19, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "math", "band": "hard", "score": 94.15, "se": 1.21, "evidence": 3.0219, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gpt-5.4-xhigh", "score": 94.15, "se": 1.21, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "math", "band": "hard", "score": 97.85, "se": 0.34, "evidence": 2.4559, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:gpt-5-mini-2025-08-07_high", "score": 97.85, "se": 0.3369422414145465, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "math", "band": "hard", "score": 92.94, "se": 1.32, "evidence": 2.42, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:claude-sonnet-5-xhigh-effort", "score": 92.94, "se": 1.32, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "math", "band": "hard", "score": 91.56, "se": 1.43, "evidence": 1.8129, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gemini-3.8-flash-high", "score": 91.56, "se": 1.43, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "math", "band": "hard", "score": 91.04, "se": 1.47, "evidence": 1.6159, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gemini-3.1-pro-preview-high", "score": 91.04, "se": 1.47, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "math", "band": "hard", "score": 90.98, "se": 1.47, "evidence": 1.5964, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gpt-5.4-nano-xhigh", "score": 90.98, "se": 1.47, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "math", "band": "hard", "score": 90.68, "se": 1.49, "evidence": 1.4874, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:deepseek-v4-pro", "score": 90.68, "se": 1.49, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "math", "band": "hard", "score": 90.68, "se": 1.49, "evidence": 1.4874, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:deepseek-v4-pro", "score": 90.68, "se": 1.49, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "math", "band": "hard", "score": 88.24, "se": 1.65, "evidence": 0.7551, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gemini-3.5-flash-high", "score": 88.24, "se": 1.65, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "math", "band": "hard", "score": 86.99, "se": 1.73, "evidence": 0.4351, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 86.99, "se": 1.73, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "math", "band": "hard", "score": 87.29, "se": 0.73, "evidence": -0.4171, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:gpt-4.1-mini-2025-04-14", "score": 87.29, "se": 0.7298484758774334, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "math", "band": "hard", "score": 86.91, "se": 0.77, "evidence": -0.509, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:claude-haiku-4-5-20251001", "score": 86.91, "se": 0.7747551687214772, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "math", "band": "hard", "score": 79.65, "se": 2.07, "evidence": -0.9827, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:deepseek-v4-flash", "score": 79.65, "se": 2.07, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "math", "band": "hard", "score": 79.65, "se": 2.07, "evidence": -0.9827, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:deepseek-v4-flash", "score": 79.65, "se": 2.07, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "math", "band": "hard", "score": 78.46, "se": 2.11, "evidence": -1.1412, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gpt-5.4-mini-xhigh", "score": 78.46, "se": 2.11, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "math", "band": "hard", "score": 73.74, "se": 2.26, "evidence": -1.6941, "sources": [ { "benchmark": "livebench:category/Mathematics", "result": "livebench:category/Mathematics:gemini-3.5-flash-lite-high", "score": 73.74, "se": 2.26, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "math", "band": "hard", "score": 83.01, "se": 0.85, "evidence": -1.7079, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:gpt-4.1-2025-04-14", "score": 83.01, "se": 0.8490146044318322, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "math", "band": "hard", "score": 81.63, "se": 0.9, "evidence": -2.0461, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:mistral-medium-2505", "score": 81.63, "se": 0.8995651651009511, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "math", "band": "hard", "score": 52.63, "se": 1.13, "evidence": -7.9632, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:gpt-4o-mini-2024-07-18", "score": 52.63, "se": 1.1275275696871734, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "math", "band": "hard", "score": 50.28, "se": 1.15, "evidence": -8.5033, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:mistral-large-2411", "score": 50.28, "se": 1.1469451623480957, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "math", "band": "hard", "score": 46.77, "se": 1.2, "evidence": -9.2312, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:mistral-small-2503", "score": 46.77, "se": 1.1995700377965737, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "math", "band": "hard", "score": 41.6, "se": 1.18, "evidence": -10.5588, "sources": [ { "benchmark": "epoch:math_level_5", "result": "epoch:math_level_5:Llama-3.3-70B-Instruct", "score": 41.6, "se": 1.1816670264886684, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "math", "band": "frontier", "score": 96.84, "se": 2.6, "evidence": 3.2765, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-6-astra_max", "score": 93.68, "se": 1.4434038295343787, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-6-astra_max", "score": 100, "unit": "percent", "weight": 0.7, "settings": "mismatch", "estimated_se": true } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "math", "band": "frontier", "score": 95.62, "se": 2.64, "evidence": 3.1546, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:claude-opus-5-5_max", "score": 91.23, "se": 1.6786209858442096, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-opus-5-5_max", "score": 100, "unit": "percent", "weight": 0.7, "settings": "mismatch", "estimated_se": true } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "math", "band": "frontier", "score": 95.09, "se": 2.65, "evidence": 3.1024, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:claude-fable-5-1_max", "score": 90.18, "se": 1.7662067789031757, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-fable-5-1_max", "score": 100, "unit": "percent", "weight": 0.7, "settings": "mismatch", "estimated_se": true } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "math", "band": "frontier", "score": 92.77, "se": 1.56, "evidence": 2.7932, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:claude-opus-5_max", "score": 85.61, "se": 2.082489457534394, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-opus-5", "score": 97.78, "se": 2.2222222222222223, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "math", "band": "frontier", "score": 72.3, "se": 2.53, "evidence": 2.2264, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:claude-opus-4-8_max", "score": 47.24, "se": 2.9366965952163047, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:claude-opus-4-8_max", "score": 80, "se": 2.3735633163877066, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-opus-4-8_none", "score": 84.44, "se": 5.463890236888291, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "math", "band": "frontier", "score": 83.66, "se": 1.45, "evidence": 2.1984, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gemini-3.8-flash_high", "score": 68.42, "se": 2.7582557383453006, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gemini-3.8-flash_high", "score": 98.89, "se": 0.8724007409153889, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "math", "band": "frontier", "score": 64.05, "se": 1.7, "evidence": 1.7647, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gemini-3.1-pro-preview", "score": 36.9, "se": 2.8000000000000003, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gemini-3.1-pro-preview", "score": 59.65, "se": 2.911181956525014, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gemini-3.1-pro-preview", "score": 95.6, "se": 3.1, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "math", "band": "frontier", "score": 65.25, "se": 2.19, "evidence": 1.5565, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gemini-3.5-flash_high", "score": 38.97, "se": 2.868659061400048, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gemini-3.5-flash_high", "score": 62.81, "se": 2.867975375107462, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash_low", "score": 88.89, "se": 4.737793696791343, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "math", "band": "frontier", "score": 88.89, "se": 4.46, "evidence": 1.4673, "sources": [ { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:openai/gpt-oss-120b_high", "score": 88.89, "se": 4.463343655613273, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "math", "band": "frontier", "score": 75.99, "se": 1.8, "evidence": 1.2914, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-flash-0731_max", "score": 57.54, "se": 2.9329899789943816, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:deepseek-v4-flash-0731_max", "score": 94.44, "se": 2.0854366486931895, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "math", "band": "frontier", "score": 75.99, "se": 1.8, "evidence": 1.2914, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-flash-0731_max", "score": 57.54, "se": 2.9329899789943816, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:deepseek-v4-flash-0731_max", "score": 94.44, "se": 2.0854366486931895, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "math", "band": "frontier", "score": 60.88, "se": 3.29, "evidence": 1.0671, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gpt-5.4-2026-03-05_xhigh", "score": 47.6, "se": 2.9000000000000004, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-2026-03-05_xhigh", "score": 78.6, "se": 2.4338000553262393, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-5.4-2026-03-05_none", "score": 57.78, "se": 7.446027270295804, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "math", "band": "frontier", "score": 72.8, "se": 3.33, "evidence": 0.9613, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:claude-sonnet-5_max", "score": 65.61, "se": 2.818576398907258, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-sonnet-5_max", "score": 80, "se": 6.030226891555273, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "math", "band": "frontier", "score": 69.1, "se": 4.46, "evidence": 0.7754, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-5.5_xhigh", "score": 85.26, "se": 2.1034091168852487, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-5.5_none", "score": 57.78, "se": 7.446027270295804, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "math", "band": "frontier", "score": 51.76, "se": 3.69, "evidence": 0.487, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:claude-sonnet-4-6_16K", "score": 32.4, "se": 2.812163604063986, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-sonnet-4-6_max", "score": 71.11, "se": 6.832943242540508, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "math", "band": "frontier", "score": 54.04, "se": 4.58, "evidence": -0.5047, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-pro-0813_max", "score": 64.56, "se": 2.8383475205435627, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:deepseek-v4-pro_none", "score": 46.67, "se": 7.521014330903548, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "math", "band": "frontier", "score": 54.04, "se": 4.58, "evidence": -0.5047, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:deepseek-v4-pro-0813_max", "score": 64.56, "se": 2.8383475205435627, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:deepseek-v4-pro_none", "score": 46.67, "se": 7.521014330903548, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "math", "band": "frontier", "score": 44.63, "se": 4.22, "evidence": -0.6698, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gemini-3.5-flash-lite_high", "score": 25.96, "se": 2.601675082238553, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gemini-3.5-flash-lite_low", "score": 60, "se": 7.385489458759962, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "math", "band": "frontier", "score": 34.09, "se": 2.88, "evidence": -0.9588, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gpt-5-mini-2025-08-07_high", "score": 27.24, "se": 2.6188332965202883, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-5-mini-2025-08-07_low", "score": 18.25, "se": 2.2917941277334046, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-5-mini-2025-08-07_minimal", "score": 55.56, "se": 7.4911095829249135, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "qwen3.5-9b", "task": "math", "band": "frontier", "score": 42.78, "se": 6.53, "evidence": -1.0437, "sources": [ { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:qwen3.5-9B", "score": 42.78, "se": 6.533819469304575, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "math", "band": "frontier", "score": 19.99, "se": 3.17, "evidence": -1.6603, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:claude-haiku-4-5-20251001", "score": 4.14, "se": 1.1715642254112493, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:claude-haiku-4-5-20251001", "score": 35.83, "se": 6.2386260142321825, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "math", "band": "frontier", "score": 25.68, "se": 2.9, "evidence": -1.6835, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gpt-5.4-nano-2026-03-17_high", "score": 25.86, "se": 2.58, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-nano-2026-03-17_none", "score": 4.56, "se": 1.2380896644748405, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-5.4-nano-2026-03-17_none", "score": 46.67, "se": 7.521014330903548, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "math", "band": "frontier", "score": 23.58, "se": 2.66, "evidence": -1.9583, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gpt-5.4-mini-2026-03-17_high", "score": 28.28, "se": 2.0650000000000004, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-5.4-mini-2026-03-17_none", "score": 17.19, "se": 2.2389789548206736, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-5.4-mini-2026-03-17_none", "score": 26.67, "se": 6.666666666666668, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "math", "band": "frontier", "score": 18.62, "se": 2.15, "evidence": -1.9984, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gpt-4.1-mini-2025-04-14", "score": 4.48, "se": 1.217207560947464, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-4.1-mini-2025-04-14", "score": 6.67, "se": 1.480176785956286, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-4.1-mini-2025-04-14", "score": 44.72, "se": 6.148017573965347, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "math", "band": "frontier", "score": 16.29, "se": 2.99, "evidence": -2.1592, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:mistral-medium-2505", "score": 0.35, "se": 0.3460207612456747, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:mistral-medium-2505", "score": 32.22, "se": 5.965611798482264, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "math", "band": "frontier", "score": 16.6, "se": 2.21, "evidence": -2.2063, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:gpt-4.1-2025-04-14", "score": 5.52, "se": 1.3430381628597847, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-4.1-2025-04-14", "score": 5.96, "se": 1.4053607746337793, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-4.1-2025-04-14", "score": 38.33, "se": 6.324056203789373, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "math", "band": "frontier", "score": 5.28, "se": 2.59, "evidence": -2.6614, "sources": [ { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:mistral-small-2501", "score": 5.28, "se": 2.589584594604903, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "math", "band": "frontier", "score": 5.14, "se": 2.36, "evidence": -2.6648, "sources": [ { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:Llama-3.3-70B-Instruct", "score": 5.14, "se": 2.3611111111111107, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "math", "band": "frontier", "score": 1.14, "se": 0.5, "evidence": -3.2655, "sources": [ { "benchmark": "epoch:frontiermath", "result": "epoch:frontiermath:mistral-large-2411", "score": 0.34, "se": 0.3448275862068965, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:mistral-large-2402", "score": 1.94, "se": 0.9489457497216071, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "math", "band": "frontier", "score": 3.82, "se": 1.39, "evidence": -3.474, "sources": [ { "benchmark": "epoch:frontiermath_tiers_1_3_v2", "result": "epoch:frontiermath_tiers_1_3_v2:gpt-4o-mini-2024-07-18", "score": 0.7, "se": 0.49534089619636035, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:otis_mock_aime_2024_2025", "result": "epoch:otis_mock_aime_2024_2025:gpt-4o-mini-2024-07-18", "score": 6.94, "se": 2.73963714244758, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "code_write", "band": "hard", "score": 79.46, "se": 3.42, "evidence": 4.3355, "sources": [ { "benchmark": "epoch:mirrorcode", "result": "epoch:mirrorcode:claude-opus-5-5_max", "score": 77.4, "se": 8.309999999999999, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-opus-5-5-max-effort", "score": 71.72, "se": 5.31, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:claude-opus-5-5-max-effort", "score": 89.25, "se": 2.86, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-opus-5.5-high", "score": 1538.02, "se": 9.74, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-opus-5.5-max", "score": 1817.84, "se": 8.42, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "code_write", "band": "hard", "score": 75.25, "se": 3.73, "evidence": 3.7782, "sources": [ { "benchmark": "epoch:mirrorcode", "result": "epoch:mirrorcode:claude-fable-5-1_high", "score": 73.3, "se": 9.17, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-fable-5-1-max-effort", "score": 66.06, "se": 5.58, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:claude-fable-5-1-max-effort", "score": 86.38, "se": 3.17, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-fable-5.1-max", "score": 1532.47, "se": 6.54, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-fable-5.1-max", "score": 1750.67, "se": 4.94, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "code_write", "band": "hard", "score": 73.33, "se": 3.33, "evidence": 3.1548, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-opus-5-max-effort", "score": 65.2, "se": 5.61, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:claude-opus-5-max-effort", "score": 81.45, "se": 3.59, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-opus-5-high", "score": 1533.7, "se": 3.23, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-opus-5-max", "score": 1693.68, "se": 3.3, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "code_write", "band": "hard", "score": 61.46, "se": 4.57, "evidence": 2.905, "sources": [ { "benchmark": "epoch:mirrorcode", "result": "epoch:mirrorcode:gpt-6-astra_high", "score": 46.7, "se": 11.86, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-6-astra-max", "score": 57.32, "se": 5.83, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gpt-6-astra-max", "score": 80.36, "se": 3.67, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-6-astra-max", "score": 1539.04, "se": 6.85, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gpt-6-astra-max", "score": 1789.11, "se": 5.33, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "code_write", "band": "hard", "score": 70.04, "se": 3.42, "evidence": 2.0575, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-sonnet-5-xhigh-effort", "score": 59.39, "se": 5.79, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:claude-sonnet-5-xhigh-effort", "score": 80.68, "se": 3.65, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-sonnet-5-high", "score": 1518.92, "se": 3.26, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-sonnet-5-high", "score": 1539.52, "se": 3.17, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "code_write", "band": "hard", "score": 66.17, "se": 3.44, "evidence": 2.0552, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-opus-4-8-max-effort", "score": 50.51, "se": 5.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:claude-opus-4-8-max-effort", "score": 81.83, "se": 3.56, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-opus-4-8", "score": 1530.24, "se": 2.96, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-opus-4-8", "score": 1534.11, "se": 2.88, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "code_write", "band": "hard", "score": 63.36, "se": 3.59, "evidence": 1.5472, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.8-flash-high", "score": 54.24, "se": 5.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gemini-3.8-flash-high", "score": 72.49, "se": 4.13, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gemini-3.8-flash-high", "score": 1530.8, "se": 4.12, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gemini-3.8-flash-high", "score": 1582.66, "se": 4.04, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "code_write", "band": "hard", "score": 48.71, "se": 3.05, "evidence": 1.421, "sources": [ { "benchmark": "epoch:mirrorcode", "result": "epoch:mirrorcode:gpt-5.5_high", "score": 10, "se": 6.042179781166438, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.5-xhigh", "score": 53.99, "se": 5.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gpt-5.5-xhigh", "score": 82.15, "se": 3.54, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-5.5", "score": 1510.53, "se": 2.88, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "code_write", "band": "hard", "score": 60.95, "se": 3.47, "evidence": 1.3337, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 42.63, "se": 5.83, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 79.27, "se": 3.75, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-sonnet-4-6", "score": 1528.69, "se": 2.87, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-sonnet-4-6", "score": 1521.58, "se": 2.62, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "code_write", "band": "hard", "score": 65.58, "se": 3.52, "evidence": 1.1411, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:grok-4.7-xhigh", "score": 53.99, "se": 5.87, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:grok-4.7-xhigh", "score": 77.16, "se": 3.88, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:grok-4.7-xhigh", "score": 1492.11, "se": 8.6, "unit": "rating", "weight": 0.51, "settings": "close" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:grok-4.7-xhigh", "score": 1636.27, "se": 6.13, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "code_write", "band": "hard", "score": 63.59, "se": 3.51, "evidence": 0.8132, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.5-flash-high", "score": 48.99, "se": 5.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gemini-3.5-flash-high", "score": 78.18, "se": 3.82, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gemini-3.5-flash-high", "score": 1506.91, "se": 3.13, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gemini-3.5-flash-high", "score": 1499.09, "se": 3.69, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "code_write", "band": "hard", "score": 48.98, "se": 3.48, "evidence": 0.5974, "sources": [ { "benchmark": "epoch:mirrorcode", "result": "epoch:mirrorcode:gpt-5.4-2026-03-05_high", "score": 15.56, "se": 7.7094504410037406, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.4-xhigh", "score": 53.84, "se": 5.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gpt-5.4-xhigh", "score": 77.54, "se": 3.86, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-5.4", "score": 1511.2, "se": 2.96, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gpt-5.4", "score": 1394.87, "se": 5.47, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "code_write", "band": "hard", "score": 43.16, "se": 2.8, "evidence": 0.5591, "sources": [ { "benchmark": "epoch:mirrorcode", "result": "epoch:mirrorcode:gemini-3.1-pro-preview_high", "score": 8.89, "se": 4.561939004190449, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.1-pro-preview-high", "score": 44.14, "se": 5.85, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gemini-3.1-pro-preview-high", "score": 76.45, "se": 3.92, "unit": "percent", "weight": 0.85, "settings": "close" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gemini-3.1-pro-preview", "score": 1521.27, "se": 2.38, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gemini-3.1-pro-preview", "score": 1446.24, "se": 2.57, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "code_write", "band": "hard", "score": 60.66, "se": 3.53, "evidence": 0.085, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.5-flash-lite-high", "score": 45.25, "se": 5.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gemini-3.5-flash-lite-high", "score": 76.07, "se": 3.94, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gemini-3.5-flash-lite", "score": 1500.99, "se": 3.55, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gemini-3.5-flash-lite", "score": 1439.71, "se": 20.69, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "code_write", "band": "hard", "score": 56.31, "se": 3.6, "evidence": -0.4157, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-pro", "score": 42.63, "se": 5.83, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:deepseek-v4-pro", "score": 69.99, "se": 4.24, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:deepseek-v4-pro", "score": 1500.38, "se": 3.02, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:deepseek-v4-pro", "score": 1446.22, "se": 3.22, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "code_write", "band": "hard", "score": 56.31, "se": 3.6, "evidence": -0.4157, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-pro", "score": 42.63, "se": 5.83, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:deepseek-v4-pro", "score": 69.99, "se": 4.24, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:deepseek-v4-pro", "score": 1500.38, "se": 3.02, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:deepseek-v4-pro", "score": 1446.22, "se": 3.22, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "code_write", "band": "hard", "score": 53.43, "se": 3.57, "evidence": -0.5864, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-flash", "score": 37.63, "se": 5.71, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:deepseek-v4-flash", "score": 69.23, "se": 4.27, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:deepseek-v4-flash", "score": 1483.9, "se": 3.18, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:deepseek-v4.1-flash-max", "score": 1620.01, "se": 5.33, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "code_write", "band": "hard", "score": 53.43, "se": 3.57, "evidence": -0.5864, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-flash", "score": 37.63, "se": 5.71, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:deepseek-v4-flash", "score": 69.23, "se": 4.27, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:deepseek-v4-flash", "score": 1483.9, "se": 3.18, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:deepseek-v4.1-flash-max", "score": 1620.01, "se": 5.33, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "code_write", "band": "hard", "score": 56.65, "se": 3.58, "evidence": -0.6957, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.4-mini-xhigh", "score": 41.67, "se": 5.81, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gpt-5.4-mini-xhigh", "score": 71.62, "se": 4.17, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-5.4-mini-high", "score": 1494.66, "se": 3.01, "unit": "rating", "weight": 0.42, "settings": "mismatch" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:gpt-5.4-mini-high", "score": 1396.89, "se": 3.32, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "code_write", "band": "hard", "score": 58.81, "se": 3.61, "evidence": -1.1148, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.4-nano-xhigh", "score": 46.77, "se": 5.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Coding", "result": "livebench:category/Coding:gpt-5.4-nano-xhigh", "score": 70.84, "se": 4.2, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-5.4-nano-high", "score": 1460.1, "se": 3.03, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "code_write", "band": "hard", "rating": 1480.83, "evidence": -1.5103, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:claude-haiku-4-5-20251001", "score": 1480.83, "se": 2.09, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:claude-haiku-4-5-20251001", "score": 1329.36, "se": 2.49, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "code_write", "band": "hard", "rating": 1456.1, "evidence": -1.6855, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-4.1-2025-04-14", "score": 1456.1, "se": 3.39, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "code_write", "band": "hard", "rating": 1478.63, "evidence": -1.7226, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:mistral-medium-3.5", "score": 1478.63, "se": 5.54, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:mistral-medium-3.5", "score": 1263.11, "se": 7.53, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "code_write", "band": "hard", "rating": 1432.98, "evidence": -2.0583, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-4.1-mini-2025-04-14", "score": 1432.98, "se": 3.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "code_write", "band": "hard", "rating": 1431.19, "evidence": -2.085, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-5-mini-high", "score": 1431.19, "se": 4.32, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "code_write", "band": "hard", "rating": 1469.27, "evidence": -2.1545, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:mistral-large-3", "score": 1469.27, "se": 2.48, "unit": "rating", "weight": 0.6, "settings": "exact" }, { "benchmark": "lmarena:webdev/overall", "result": "lmarena:webdev/overall:mistral-large-3", "score": 1229.84, "se": 12.99, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "code_write", "band": "hard", "rating": 1412.14, "evidence": -2.3739, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:mistral-small-2506", "score": 1412.14, "se": 5.12, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "code_write", "band": "hard", "rating": 1389.67, "evidence": -2.5862, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-oss-120b", "score": 1389.67, "se": 3.94, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "code_write", "band": "hard", "rating": 1347.98, "evidence": -2.8802, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:gpt-4o-mini-2024-07-18", "score": 1347.98, "se": 3.35, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "llama-3.3-70b-instruct-turbo", "task": "code_write", "band": "hard", "rating": 1345.74, "evidence": -2.9129, "sources": [ { "benchmark": "lmarena:text_style_control/coding", "result": "lmarena:text_style_control/coding:llama-3.3-70b-instruct", "score": 1345.74, "se": 3.35, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "code_fix", "band": "hard", "score": 71.72, "se": 5.31, "evidence": 2.9518, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-opus-5-5-max-effort", "score": 71.72, "se": 5.31, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "code_fix", "band": "hard", "score": 66.06, "se": 5.58, "evidence": 2.3957, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-fable-5-1-max-effort", "score": 66.06, "se": 5.58, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "code_fix", "band": "hard", "score": 65.2, "se": 5.61, "evidence": 2.2966, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-opus-5-max-effort", "score": 65.2, "se": 5.61, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "code_fix", "band": "hard", "score": 69.68, "se": 2.34, "evidence": 1.9421, "sources": [ { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:gpt-5.4-2026-03-05_high", "score": 76.86, "se": 1.9189407003691976, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-5.4-2026-03-05_unknown", "score": 81.8, "se": 2, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.4-xhigh", "score": 53.84, "se": 5.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "code_fix", "band": "hard", "score": 59.39, "se": 5.79, "evidence": 1.6167, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-sonnet-5-xhigh-effort", "score": 59.39, "se": 5.79, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "code_fix", "band": "hard", "score": 64.16, "se": 3.09, "evidence": 1.5525, "sources": [ { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:gemini-3.5-flash_high", "score": 79.34, "se": 1.8422420250424887, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.5-flash-high", "score": 48.99, "se": 5.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "code_fix", "band": "hard", "score": 57.32, "se": 5.83, "evidence": 1.3729, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-6-astra-max", "score": 57.32, "se": 5.83, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "code_fix", "band": "hard", "score": 59.02, "se": 3.58, "evidence": 1.3669, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-5.5_unknown", "score": 66.07, "se": 2.4915582762, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.5-xhigh", "score": 53.99, "se": 5.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "code_fix", "band": "hard", "score": 54.24, "se": 5.87, "evidence": 1.0165, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.8-flash-high", "score": 54.24, "se": 5.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "code_fix", "band": "hard", "score": 53.99, "se": 5.87, "evidence": 0.9881, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:grok-4.7-xhigh", "score": 53.99, "se": 5.87, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "code_fix", "band": "hard", "score": 57.05, "se": 3.52, "evidence": 0.6462, "sources": [ { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:deepseek-v4-pro_max", "score": 77.64, "se": 1.897828052158487, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-pro", "score": 42.63, "se": 5.83, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "code_fix", "band": "hard", "score": 57.05, "se": 3.52, "evidence": 0.6462, "sources": [ { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:deepseek-v4-pro_max", "score": 77.64, "se": 1.897828052158487, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-pro", "score": 42.63, "se": 5.83, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "code_fix", "band": "hard", "score": 50.51, "se": 5.89, "evidence": 0.5911, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-opus-4-8-max-effort", "score": 50.51, "se": 5.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "code_fix", "band": "hard", "score": 59.89, "se": 2.15, "evidence": 0.5335, "sources": [ { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:claude-sonnet-4-6", "score": 75.21, "se": 1.9648182701390322, "unit": "percent", "weight": 1, "settings": "exact" }, { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:claude-sonnet-4-6", "score": 53.4, "se": 2.8, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 42.63, "se": 5.83, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "code_fix", "band": "hard", "score": 49.78, "se": 4, "evidence": 0.2285, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gemini-3.1-pro-preview", "score": 59.36, "se": 4.2009785316, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.1-pro-preview-high", "score": 44.14, "se": 5.85, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "code_fix", "band": "hard", "score": 46.77, "se": 5.88, "evidence": 0.1732, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.4-nano-xhigh", "score": 46.77, "se": 5.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "code_fix", "band": "hard", "score": 49.83, "se": 1.88, "evidence": 0.0331, "sources": [ { "benchmark": "epoch:aider_polyglot_external", "result": "epoch:aider_polyglot_external:gpt-4.1-2025-04-14", "score": 52.4, "se": 3.33, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:gpt-4.1-2025-04-14", "score": 48.54, "se": 2.2835863746433964, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "code_fix", "band": "hard", "score": 45.25, "se": 5.87, "evidence": 0.0083, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gemini-3.5-flash-lite-high", "score": 45.25, "se": 5.87, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "code_fix", "band": "hard", "score": 47, "se": 1.67, "evidence": -0.0076, "sources": [ { "benchmark": "epoch:swe_bench_verified", "result": "epoch:swe_bench_verified:gpt-5-mini-2025-08-07_medium", "score": 64.67, "se": 2.1749601139671304, "unit": "percent", "weight": 0.7, "settings": "mismatch" }, { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-5-mini-2025-08-07_unknown", "score": 22.25, "se": 2.6242804005, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "code_fix", "band": "hard", "score": 41.67, "se": 5.81, "evidence": -0.3802, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:gpt-5.4-mini-xhigh", "score": 41.67, "se": 5.81, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "code_fix", "band": "hard", "score": 37.63, "se": 5.71, "evidence": -0.7906, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-flash", "score": 37.63, "se": 5.71, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "code_fix", "band": "hard", "score": 37.63, "se": 5.71, "evidence": -0.7906, "sources": [ { "benchmark": "livebench:category/Agentic Coding", "result": "livebench:category/Agentic Coding:deepseek-v4-flash", "score": 37.63, "se": 5.71, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "code_fix", "band": "hard", "score": 32.4, "se": 3.12, "evidence": -1.4148, "sources": [ { "benchmark": "epoch:aider_polyglot_external", "result": "epoch:aider_polyglot_external:gpt-4.1-mini-2025-04-14", "score": 32.4, "se": 3.12, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "code_fix", "band": "hard", "score": 25.56, "se": 1.91, "evidence": -1.8361, "sources": [ { "benchmark": "epoch:aider_polyglot_external", "result": "epoch:aider_polyglot_external:openai/gpt-oss-120b_high", "score": 41.8, "se": 3.29, "unit": "percent", "weight": 0.35, "settings": "mismatch" }, { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-oss-120b", "score": 14.2, "se": 2.3, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "code_fix", "band": "hard", "score": 13.9, "se": 2.7, "evidence": -2.7578, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:claude-haiku-4-5-20251001", "score": 13.9, "se": 2.7, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "qwen3.5-9b", "task": "code_fix", "band": "hard", "score": 9.21, "se": 2.35, "evidence": -3.3725, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:qwen3.5-9B", "score": 9.21, "se": 2.3513565589, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "codestral-2508", "task": "code_fix", "band": "hard", "score": 11.1, "se": 2.09, "evidence": -3.8252, "sources": [ { "benchmark": "epoch:aider_polyglot_external", "result": "epoch:aider_polyglot_external:codestral-2501", "score": 11.1, "se": 2.09, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "code_fix", "band": "hard", "score": 3.6, "se": 1.24, "evidence": -5.1843, "sources": [ { "benchmark": "epoch:aider_polyglot_external", "result": "epoch:aider_polyglot_external:gpt-4o-mini-2024-07-18", "score": 3.6, "se": 1.24, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "query", "band": "medium", "score": 58.37, "se": 6.84, "evidence": 0.9752, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gpt-6-astra-max", "score": 58.37, "se": 6.84, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "query", "band": "medium", "score": 56.15, "se": 6.88, "evidence": 0.7326, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:claude-fable-5-1-max-effort", "score": 56.15, "se": 6.88, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "query", "band": "medium", "score": 55.9, "se": 6.89, "evidence": 0.7051, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gpt-5.5-xhigh", "score": 55.9, "se": 6.89, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "query", "band": "medium", "score": 52.35, "se": 6.93, "evidence": 0.321, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gemini-3.1-pro-preview-high", "score": 52.35, "se": 6.93, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "query", "band": "medium", "score": 52.33, "se": 6.93, "evidence": 0.3189, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:claude-opus-5-5-max-effort", "score": 52.33, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "query", "band": "medium", "score": 51.96, "se": 6.93, "evidence": 0.2791, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:claude-opus-5-max-effort", "score": 51.96, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "query", "band": "medium", "score": 51.75, "se": 6.93, "evidence": 0.2565, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gpt-5.4-xhigh", "score": 51.75, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "query", "band": "medium", "score": 51.08, "se": 6.93, "evidence": 0.1844, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:claude-opus-4-8-max-effort", "score": 51.08, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "query", "band": "medium", "score": 50.59, "se": 6.93, "evidence": 0.1317, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gpt-5.4-nano-xhigh", "score": 50.59, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "query", "band": "medium", "score": 50.5, "se": 6.93, "evidence": 0.122, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gemini-3.5-flash-high", "score": 50.5, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "query", "band": "medium", "score": 49.39, "se": 6.93, "evidence": 0.0025, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gpt-5.4-mini-xhigh", "score": 49.39, "se": 6.93, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro", "task": "query", "band": "medium", "score": 48.19, "se": 6.93, "evidence": -0.1266, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:deepseek-v4-pro", "score": 48.19, "se": 6.93, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-v4-pro@together", "task": "query", "band": "medium", "score": 48.19, "se": 6.93, "evidence": -0.1266, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:deepseek-v4-pro", "score": 48.19, "se": 6.93, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash", "task": "query", "band": "medium", "score": 46.12, "se": 6.91, "evidence": -0.3498, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:deepseek-v4-flash", "score": 46.12, "se": 6.91, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "deepseek-flash@together", "task": "query", "band": "medium", "score": 46.12, "se": 6.91, "evidence": -0.3498, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:deepseek-v4-flash", "score": 46.12, "se": 6.91, "unit": "percent", "weight": 1, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "query", "band": "medium", "score": 45.33, "se": 6.9, "evidence": -0.4353, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gemini-3.8-flash-high", "score": 45.33, "se": 6.9, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "query", "band": "medium", "score": 45.08, "se": 6.9, "evidence": -0.4622, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:claude-sonnet-4-6-thinking-auto-medium-effort", "score": 45.08, "se": 6.9, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "grok-4.7", "task": "query", "band": "medium", "score": 44.92, "se": 6.9, "evidence": -0.4795, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:grok-4.7-xhigh", "score": 44.92, "se": 6.9, "unit": "percent", "weight": 0.85, "settings": "close" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "query", "band": "medium", "score": 42.44, "se": 6.85, "evidence": -0.7496, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:claude-sonnet-5-xhigh-effort", "score": 42.44, "se": 6.85, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "query", "band": "medium", "score": 40.62, "se": 6.81, "evidence": -0.9494, "sources": [ { "benchmark": "livebench:task/tablejoin", "result": "livebench:task/tablejoin:gemini-3.5-flash-lite-high", "score": 40.62, "se": 6.81, "unit": "percent", "weight": 0.7, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "tool_use", "band": "hard", "score": 81.8, "se": 2, "evidence": 3, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-5.4-2026-03-05_unknown", "score": 81.8, "se": 2, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "tool_use", "band": "hard", "score": 66.07, "se": 2.49, "evidence": 1.9105, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-5.5_unknown", "score": 66.07, "se": 2.4915582762, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:agent/overall", "result": "lmarena:agent/overall:GPT 5.5", "score": 0.01, "se": 0, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "tool_use", "band": "hard", "score": 59.36, "se": 4.2, "evidence": 1.2581, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gemini-3.1-pro-preview", "score": 59.36, "se": 4.2009785316, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "lmarena:agent/overall", "result": "lmarena:agent/overall:Gemini 3.1 Pro Preview", "score": -0.07, "se": 0, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "tool_use", "band": "hard", "score": 62.75, "se": 1.82, "evidence": 0.6885, "sources": [ { "benchmark": "epoch:os_world_external", "result": "epoch:os_world_external:claude-sonnet-4-6", "score": 72.1, "se": 2.33, "unit": "percent", "weight": 0.5, "settings": "exact" }, { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:claude-sonnet-4-6", "score": 53.4, "se": 2.8, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "tool_use", "band": "hard", "rating": -0.12, "evidence": 0, "sources": [ { "benchmark": "lmarena:agent/overall", "result": "lmarena:agent/overall:Mistral Medium 3.5", "score": -0.12, "se": 0.01, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "tool_use", "band": "hard", "score": 22.25, "se": 2.62, "evidence": -0.6393, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-5-mini-2025-08-07_unknown", "score": 22.25, "se": 2.6242804005, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-oss-120b", "task": "tool_use", "band": "hard", "score": 14.2, "se": 2.3, "evidence": -1.815, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:gpt-oss-120b", "score": 14.2, "se": 2.3, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-haiku-4-5", "task": "tool_use", "band": "hard", "score": 13.9, "se": 2.7, "evidence": -1.856, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:claude-haiku-4-5-20251001", "score": 13.9, "se": 2.7, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "qwen3.5-9b", "task": "tool_use", "band": "hard", "score": 9.21, "se": 2.35, "evidence": -2.5467, "sources": [ { "benchmark": "epoch:terminalbench_external", "result": "epoch:terminalbench_external:qwen3.5-9B", "score": 9.21, "se": 2.3513565589, "unit": "percent", "weight": 0.5, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5", "task": "vision", "band": "medium", "rating": 1320.86, "evidence": 2.729, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:claude-opus-5-high", "score": 1320.86, "se": 3.55, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-fable-5-1", "task": "vision", "band": "medium", "rating": 1318.93, "evidence": 2.6437, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:claude-fable-5.1-max", "score": 1318.93, "se": 5.21, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.8-flash", "task": "vision", "band": "medium", "rating": 1312.02, "evidence": 2.3731, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gemini-3.8-flash-high", "score": 1312.02, "se": 6.2, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash", "task": "vision", "band": "medium", "rating": 1310.49, "evidence": 2.3606, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gemini-3.5-flash-high", "score": 1310.49, "se": 3.65, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.5", "task": "vision", "band": "medium", "rating": 1297.26, "evidence": 1.5571, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-5.5", "score": 1297.26, "se": 3.03, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gemini-3.1-pro-preview", "task": "vision", "band": "medium", "rating": 1296.21, "evidence": 1.4895, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gemini-3.1-pro-preview", "score": 1296.21, "se": 2.68, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4", "task": "vision", "band": "medium", "rating": 1291.91, "evidence": 1.1757, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-5.4", "score": 1291.91, "se": 3.32, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-4-8", "task": "vision", "band": "medium", "rating": 1287.93, "evidence": 0.8991, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:claude-opus-4-8", "score": 1287.93, "se": 3.35, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-sonnet-4-6", "task": "vision", "band": "medium", "rating": 1282.75, "evidence": 0.5404, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:claude-sonnet-4-6", "score": 1282.75, "se": 3.15, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-6-astra", "task": "vision", "band": "medium", "rating": 1280.05, "evidence": 0.4636, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-6-astra-max", "score": 1280.05, "se": 6.02, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "claude-sonnet-5", "task": "vision", "band": "medium", "rating": 1275.4, "evidence": 0.1449, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:claude-sonnet-5-high", "score": 1275.4, "se": 3.59, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gemini-3.5-flash-lite", "task": "vision", "band": "medium", "rating": 1266.09, "evidence": -0.1767, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gemini-3.5-flash-lite", "score": 1266.09, "se": 4.34, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5.4-mini", "task": "vision", "band": "medium", "rating": 1244.85, "evidence": -0.6, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-5.4-mini-high", "score": 1244.85, "se": 3.18, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "mistral-medium-3-5", "task": "vision", "band": "medium", "rating": 1222.37, "evidence": -1.0911, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:mistral-medium-3.5", "score": 1222.37, "se": 4.85, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-large-2512", "task": "vision", "band": "medium", "rating": 1221.46, "evidence": -1.1038, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:mistral-large-3", "score": 1221.46, "se": 4.19, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4.1", "task": "vision", "band": "medium", "rating": 1210.31, "evidence": -1.4754, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-4.1-2025-04-14", "score": 1210.31, "se": 3.4, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-5-mini", "task": "vision", "band": "medium", "rating": 1201.96, "evidence": -1.8223, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-5-mini-high", "score": 1201.96, "se": 3.98, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-5.4-nano", "task": "vision", "band": "medium", "rating": 1196.06, "evidence": -2.0098, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-5.4-nano-high", "score": 1196.06, "se": 3.24, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": false }, { "model": "gpt-4.1-mini", "task": "vision", "band": "medium", "rating": 1181.29, "evidence": -2.3977, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-4.1-mini-2025-04-14", "score": 1181.29, "se": 3.81, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "mistral-small-2603", "task": "vision", "band": "medium", "rating": 1141.55, "evidence": -2.7, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:mistral-small-2506", "score": 1141.55, "se": 4.63, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "gpt-4o-mini", "task": "vision", "band": "medium", "rating": 1065.48, "evidence": -3, "sources": [ { "benchmark": "lmarena:vision/overall", "result": "lmarena:vision/overall:gpt-4o-mini-2024-07-18", "score": 1065.48, "se": 4.23, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": false }, { "model": "claude-opus-5-5", "task": "document_qa", "band": "medium", "rating": 1527.81, "evidence": 2.9319, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-opus-5.5-high", "score": 1527.81, "se": 7.62, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-fable-5-1", "task": "document_qa", "band": "medium", "rating": 1511.39, "evidence": 2.6357, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-fable-5.1-max", "score": 1511.39, "se": 4.81, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gemini-3.8-flash", "task": "document_qa", "band": "medium", "rating": 1507.93, "evidence": 2.5444, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gemini-3.8-flash-high", "score": 1507.93, "se": 3.53, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-opus-5", "task": "document_qa", "band": "medium", "rating": 1504.8, "evidence": 2.4232, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-opus-5-high", "score": 1504.8, "se": 2.78, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-opus-4-8", "task": "document_qa", "band": "medium", "rating": 1499.75, "evidence": 2.1502, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-opus-4-8", "score": 1499.75, "se": 2.63, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gemini-3.1-pro-preview", "task": "document_qa", "band": "medium", "rating": 1498.89, "evidence": 2.0962, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gemini-3.1-pro-preview", "score": 1498.89, "se": 2.18, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "claude-sonnet-4-6", "task": "document_qa", "band": "medium", "rating": 1492.39, "evidence": 1.7065, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-sonnet-4-6", "score": 1492.39, "se": 2.56, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-5.5", "task": "document_qa", "band": "medium", "rating": 1486.64, "evidence": 1.3706, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-5.5", "score": 1486.64, "se": 2.55, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-6-astra", "task": "document_qa", "band": "medium", "rating": 1485.58, "evidence": 1.2869, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-6-astra-max", "score": 1485.58, "se": 5.29, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-sonnet-5", "task": "document_qa", "band": "medium", "rating": 1481.83, "evidence": 1.0956, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-sonnet-5-high", "score": 1481.83, "se": 2.83, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gemini-3.5-flash", "task": "document_qa", "band": "medium", "rating": 1481.48, "evidence": 1.0726, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gemini-3.5-flash-high", "score": 1481.48, "se": 2.74, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gpt-5.4", "task": "document_qa", "band": "medium", "rating": 1477.36, "evidence": 0.8229, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-5.4", "score": 1477.36, "se": 2.66, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "deepseek-v4-pro", "task": "document_qa", "band": "medium", "rating": 1471.65, "evidence": 0.4986, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:deepseek-v4-pro", "score": 1471.65, "se": 2.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "deepseek-v4-pro@together", "task": "document_qa", "band": "medium", "rating": 1471.65, "evidence": 0.4986, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:deepseek-v4-pro", "score": 1471.65, "se": 2.72, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gemini-3.5-flash-lite", "task": "document_qa", "band": "medium", "rating": 1461.62, "evidence": 0.0922, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gemini-3.5-flash-lite", "score": 1461.62, "se": 3.08, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "grok-4.7", "task": "document_qa", "band": "medium", "rating": 1450.54, "evidence": -0.3571, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:grok-4.7-xhigh", "score": 1450.54, "se": 6.42, "unit": "rating", "weight": 0.51, "settings": "close" } ], "zabio_tested": true }, { "model": "deepseek-flash", "task": "document_qa", "band": "medium", "rating": 1447.41, "evidence": -0.4232, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:deepseek-v4-flash", "score": 1447.41, "se": 2.84, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "deepseek-flash@together", "task": "document_qa", "band": "medium", "rating": 1447.41, "evidence": -0.4232, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:deepseek-v4-flash", "score": 1447.41, "se": 2.84, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-5.4-mini", "task": "document_qa", "band": "medium", "rating": 1446.73, "evidence": -0.4447, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-5.4-mini-high", "score": 1446.73, "se": 2.69, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-haiku-4-5", "task": "document_qa", "band": "medium", "rating": 1436.32, "evidence": -0.9762, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:claude-haiku-4-5-20251001", "score": 1436.32, "se": 1.93, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "mistral-medium-3-5", "task": "document_qa", "band": "medium", "rating": 1429.62, "evidence": -1.1737, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:mistral-medium-3.5", "score": 1429.62, "se": 4.65, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-4.1", "task": "document_qa", "band": "medium", "rating": 1426.9, "evidence": -1.2569, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-4.1-2025-04-14", "score": 1426.9, "se": 3.27, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "mistral-large-2512", "task": "document_qa", "band": "medium", "rating": 1416.95, "evidence": -1.516, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:mistral-large-3", "score": 1416.95, "se": 2.24, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-5.4-nano", "task": "document_qa", "band": "medium", "rating": 1402.6, "evidence": -1.7689, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-5.4-nano-high", "score": 1402.6, "se": 2.74, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gpt-4.1-mini", "task": "document_qa", "band": "medium", "rating": 1389.88, "evidence": -1.9552, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-4.1-mini-2025-04-14", "score": 1389.88, "se": 3.83, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-5-mini", "task": "document_qa", "band": "medium", "rating": 1370.25, "evidence": -2.2161, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-5-mini-high", "score": 1370.25, "se": 4.19, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "mistral-small-2603", "task": "document_qa", "band": "medium", "rating": 1358.76, "evidence": -2.3356, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:mistral-small-2506", "score": 1358.76, "se": 5.14, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-4o-mini", "task": "document_qa", "band": "medium", "rating": 1326.18, "evidence": -2.6932, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-4o-mini-2024-07-18", "score": 1326.18, "se": 3.63, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-oss-120b", "task": "document_qa", "band": "medium", "rating": 1324.42, "evidence": -2.7316, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:gpt-oss-120b", "score": 1324.42, "se": 3.96, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "llama-3.3-70b-instruct-turbo", "task": "document_qa", "band": "medium", "rating": 1313.38, "evidence": -2.9546, "sources": [ { "benchmark": "lmarena:text_style_control/longer_query", "result": "lmarena:text_style_control/longer_query:llama-3.3-70b-instruct", "score": 1313.38, "se": 3.41, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "claude-opus-5", "task": "document_qa", "band": "hard", "rating": 1516.27, "evidence": 2.7432, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:claude-opus-5-high", "score": 1516.27, "se": 3.58, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-fable-5-1", "task": "document_qa", "band": "hard", "rating": 1512.58, "evidence": 2.6568, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:claude-fable-5.1-max", "score": 1512.58, "se": 7.75, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gpt-5-mini", "task": "document_qa", "band": "hard", "score": 69.4, "se": 5, "evidence": 2.4445, "sources": [ { "benchmark": "epoch:fictionlivebench_external", "result": "epoch:fictionlivebench_external:gpt-5-mini-2025-08-07_medium", "score": 69.4, "unit": "percent", "weight": 0.35, "settings": "mismatch", "estimated_se": true } ], "zabio_tested": true }, { "model": "gpt-4.1", "task": "document_qa", "band": "hard", "score": 63.9, "se": 5, "evidence": 1.995, "sources": [ { "benchmark": "epoch:fictionlivebench_external", "result": "epoch:fictionlivebench_external:gpt-4.1-2025-04-14", "score": 63.9, "unit": "percent", "weight": 0.5, "settings": "exact", "estimated_se": true } ], "zabio_tested": true }, { "model": "gpt-5.5", "task": "document_qa", "band": "hard", "rating": 1485.72, "evidence": 1.5087, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:gpt-5.5", "score": 1485.72, "se": 3.18, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "claude-sonnet-4-6", "task": "document_qa", "band": "hard", "rating": 1482.24, "evidence": 1.2688, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:claude-sonnet-4-6", "score": 1482.24, "se": 3, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-5.4", "task": "document_qa", "band": "hard", "rating": 1471, "evidence": -0.1193, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:gpt-5.4", "score": 1471, "se": 3.08, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gpt-6-astra", "task": "document_qa", "band": "hard", "rating": 1467.52, "evidence": -0.3496, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:gpt-6-astra-max", "score": 1467.52, "se": 7.28, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "claude-sonnet-5", "task": "document_qa", "band": "hard", "rating": 1466.14, "evidence": -0.6239, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:claude-sonnet-5-high", "score": 1466.14, "se": 3.9, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gpt-4.1-mini", "task": "document_qa", "band": "hard", "score": 44.4, "se": 5, "evidence": -0.7535, "sources": [ { "benchmark": "epoch:fictionlivebench_external", "result": "epoch:fictionlivebench_external:gpt-4.1-mini-2025-04-14", "score": 44.4, "unit": "percent", "weight": 0.5, "settings": "exact", "estimated_se": true } ], "zabio_tested": true }, { "model": "claude-opus-4-8", "task": "document_qa", "band": "hard", "rating": 1464.07, "evidence": -0.8144, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:claude-opus-4-8", "score": 1464.07, "se": 3.55, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "gemini-3.5-flash", "task": "document_qa", "band": "hard", "rating": 1463.27, "evidence": -0.8703, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:gemini-3.5-flash-medium", "score": 1463.27, "se": 4.01, "unit": "rating", "weight": 0.42, "settings": "mismatch" } ], "zabio_tested": true }, { "model": "gpt-oss-120b", "task": "document_qa", "band": "hard", "score": 36.1, "se": 5, "evidence": -1.6945, "sources": [ { "benchmark": "epoch:fictionlivebench_external", "result": "epoch:fictionlivebench_external:chutes/gpt-oss-120b", "score": 36.1, "unit": "percent", "weight": 0.5, "settings": "exact", "estimated_se": true } ], "zabio_tested": true }, { "model": "llama-3.3-70b-instruct-turbo", "task": "document_qa", "band": "hard", "score": 33.3, "se": 5, "evidence": -1.9914, "sources": [ { "benchmark": "epoch:fictionlivebench_external", "result": "epoch:fictionlivebench_external:Llama-3.3-70B-Instruct", "score": 33.3, "unit": "percent", "weight": 0.5, "settings": "exact", "estimated_se": true } ], "zabio_tested": true }, { "model": "gemini-3.1-pro-preview", "task": "document_qa", "band": "hard", "rating": 1443.92, "evidence": -2.4, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:gemini-3.1-pro-preview", "score": 1443.92, "se": 2.67, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true }, { "model": "claude-haiku-4-5", "task": "document_qa", "band": "hard", "rating": 1419.97, "evidence": -3, "sources": [ { "benchmark": "lmarena:document/overall", "result": "lmarena:document/overall:claude-haiku-4-5-20251001", "score": 1419.97, "se": 2.99, "unit": "rating", "weight": 0.6, "settings": "exact" } ], "zabio_tested": true } ], "curves": [ { "task": "writing", "band": "medium", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.36, "rating": 1337.69, "evidence": -2.3977, "off_curve": false, "rank": 27 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.36, "rating": 1323.37, "evidence": -2.5678, "off_curve": true, "rank": 28 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.36, "rating": 1293.1, "evidence": -2.8918, "off_curve": true, "rank": 29 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.66, "score": 64.58, "se": 3.88, "evidence": -1.7294, "off_curve": false, "rank": 24 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.72, "score": 66.25, "se": 3.9, "evidence": -0.7168, "off_curve": false, "rank": 18 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.72, "score": 66.25, "se": 3.9, "evidence": -0.7168, "off_curve": true, "rank": 19 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 0.96, "rating": 1372.9, "evidence": -2.0714, "off_curve": true, "rank": 26 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 1, "rating": 1400.45, "evidence": -1.4746, "off_curve": true, "rank": 23 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 1, "rating": 1373.26, "evidence": -2.0607, "off_curve": true, "rank": 25 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.24, "score": 68.41, "se": 3.84, "evidence": -0.2048, "off_curve": false, "rank": 16 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 1.248, "rating": 1292.61, "evidence": -2.9013, "off_curve": true, "rank": 30 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 2.1, "score": 83.9, "se": 3.13, "evidence": 2.8788, "off_curve": false, "rank": 1 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 2.4, "score": 64.92, "se": 3.94, "evidence": -0.6631, "off_curve": true, "rank": 17 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 2.64, "score": 67.18, "se": 3.9, "evidence": 0.0157, "off_curve": true, "rank": 14 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 2.64, "score": 67.18, "se": 3.9, "evidence": 0.0157, "off_curve": true, "rank": 15 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 2.8, "rating": 1414.99, "evidence": -1.0738, "off_curve": true, "rank": 21 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 4.2, "rating": 1421.21, "evidence": -0.818, "off_curve": true, "rank": 20 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 4.4, "score": 79.1, "se": 3.35, "evidence": 1.0905, "off_curve": true, "rank": 10 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 4.8, "score": 80.05, "se": 3.38, "evidence": 1.8648, "off_curve": true, "rank": 8 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 4.8, "rating": 1403.29, "evidence": -1.422, "off_curve": true, "rank": 22 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 5.6, "score": 68.11, "se": 3.87, "evidence": 0.3099, "off_curve": true, "rank": 13 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 6.4, "score": 82.62, "se": 3.18, "evidence": 2.5373, "off_curve": true, "rank": 4 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 8, "score": 73.03, "se": 3.74, "evidence": 0.9538, "off_curve": true, "rank": 11 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 8.4, "score": 67.75, "se": 3.88, "evidence": 0.587, "off_curve": true, "rank": 12 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 11.2, "score": 75.27, "se": 3.67, "evidence": 2.177, "off_curve": true, "rank": 7 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 14, "score": 77.21, "se": 3.6, "evidence": 2.3657, "off_curve": true, "rank": 5 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 14, "score": 73.77, "se": 3.69, "evidence": 1.406, "off_curve": true, "rank": 9 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 16, "score": 79.55, "se": 3.46, "evidence": 2.1771, "off_curve": true, "rank": 6 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 28, "score": 80.51, "se": 3.43, "evidence": 2.716, "off_curve": false, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 28, "score": 83.84, "se": 3.17, "evidence": 2.6149, "off_curve": true, "rank": 3 } ], "picks": { "best": "gemini-3.8-flash", "balanced": "gemini-3.8-flash", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "creative", "band": "medium", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.36, "rating": 1323.22, "evidence": -2.2327, "off_curve": false, "rank": 26 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.36, "rating": 1293.93, "evidence": -2.6455, "off_curve": true, "rank": 28 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.36, "rating": 1277.82, "evidence": -2.9241, "off_curve": true, "rank": 30 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.66, "score": 67.65, "se": 6.62, "evidence": -1.3274, "off_curve": false, "rank": 23 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.72, "score": 67.27, "se": 6.64, "evidence": -0.224, "off_curve": false, "rank": 17 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.72, "score": 67.27, "se": 6.64, "evidence": -0.224, "off_curve": true, "rank": 18 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 0.96, "rating": 1349.05, "evidence": -1.7853, "off_curve": true, "rank": 25 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 1, "rating": 1373.37, "evidence": -1.5517, "off_curve": true, "rank": 24 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 1, "rating": 1323.71, "evidence": -2.236, "off_curve": true, "rank": 27 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.24, "score": 67.47, "se": 6.63, "evidence": 0.4014, "off_curve": false, "rank": 15 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 1.248, "rating": 1285.34, "evidence": -2.8098, "off_curve": true, "rank": 29 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 2.1, "score": 82.75, "se": 5.34, "evidence": 2.6239, "off_curve": false, "rank": 1 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 2.4, "score": 66.63, "se": 6.67, "evidence": -0.3367, "off_curve": true, "rank": 19 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 2.64, "score": 67.75, "se": 6.61, "evidence": 0.7389, "off_curve": true, "rank": 13 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 2.64, "score": 67.75, "se": 6.61, "evidence": 0.7389, "off_curve": true, "rank": 14 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 2.8, "rating": 1387.4, "evidence": -1.3012, "off_curve": true, "rank": 22 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 4.2, "rating": 1400.31, "evidence": -0.8424, "off_curve": true, "rank": 21 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 4.4, "score": 74.92, "se": 6.13, "evidence": 0.9108, "off_curve": true, "rank": 12 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 4.8, "score": 75.2, "se": 6.11, "evidence": 1.7929, "off_curve": true, "rank": 6 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 4.8, "rating": 1402.38, "evidence": -0.7775, "off_curve": true, "rank": 20 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 5.6, "score": 62.32, "se": 6.85, "evidence": 0.1035, "off_curve": true, "rank": 16 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 6.4, "score": 77.25, "se": 5.93, "evidence": 2.247, "off_curve": true, "rank": 2 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 8, "score": 79.02, "se": 5.76, "evidence": 1.0924, "off_curve": true, "rank": 9 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 8.4, "score": 68.9, "se": 6.55, "evidence": 0.9549, "off_curve": true, "rank": 11 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 11.2, "score": 70.92, "se": 6.42, "evidence": 1.9827, "off_curve": false, "rank": 5 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 14, "score": 80.6, "se": 5.59, "evidence": 2.1442, "off_curve": true, "rank": 4 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 14, "score": 61.32, "se": 6.89, "evidence": 0.971, "off_curve": true, "rank": 10 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 16, "score": 72.55, "se": 6.31, "evidence": 1.1307, "off_curve": true, "rank": 8 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 28, "score": 78.1, "se": 5.85, "evidence": 2.2124, "off_curve": true, "rank": 3 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 28, "score": 73.98, "se": 6.2, "evidence": 1.1725, "off_curve": true, "rank": 7 } ], "picks": { "best": "gemini-3.8-flash", "balanced": "gemini-3.8-flash", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "summarization", "band": "medium", "points": [ { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.73, "score": 71.77, "se": 6.37, "evidence": -0.1401, "off_curve": false, "rank": 9 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.96, "score": 68.85, "se": 6.55, "evidence": -0.4781, "off_curve": true, "rank": 14 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.96, "score": 68.85, "se": 6.55, "evidence": -0.4781, "off_curve": true, "rank": 15 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.22, "score": 68.47, "se": 6.57, "evidence": -0.5215, "off_curve": true, "rank": 16 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 2.55, "score": 87.5, "se": 4.68, "evidence": 1.9395, "off_curve": false, "rank": 2 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 2.7, "score": 59.63, "se": 6.94, "evidence": -1.4642, "off_curve": true, "rank": 20 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 3.96, "score": 69.5, "se": 6.51, "evidence": -0.4037, "off_curve": true, "rank": 11 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 3.96, "score": 69.5, "se": 6.51, "evidence": -0.4037, "off_curve": true, "rank": 12 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 5.4, "score": 82.58, "se": 5.36, "evidence": 1.2313, "off_curve": true, "rank": 3 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 6.6, "score": 78.17, "se": 5.84, "evidence": 0.6422, "off_curve": true, "rank": 5 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 6.8, "score": 71.22, "se": 6.4, "evidence": -0.2046, "off_curve": true, "rank": 10 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 7.2, "score": 88.23, "se": 4.56, "evidence": 2.045, "off_curve": false, "rank": 1 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 9, "score": 68.42, "se": 6.57, "evidence": -0.5274, "off_curve": true, "rank": 17 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 10.2, "score": 61.65, "se": 6.88, "evidence": -1.2655, "off_curve": true, "rank": 19 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 13.6, "score": 69.1, "se": 6.53, "evidence": -0.4498, "off_curve": true, "rank": 13 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 17, "score": 77.67, "se": 5.89, "evidence": 0.5783, "off_curve": true, "rank": 6 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 17, "score": 66.43, "se": 6.68, "evidence": -0.7521, "off_curve": true, "rank": 18 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 18, "score": 72.32, "se": 6.33, "evidence": -0.0754, "off_curve": true, "rank": 7 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 34, "score": 79.55, "se": 5.7, "evidence": 0.8212, "off_curve": true, "rank": 4 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 34, "score": 72.17, "se": 6.34, "evidence": -0.0931, "off_curve": true, "rank": 8 } ], "picks": { "best": "gemini-3.8-flash", "balanced": "gemini-3.8-flash", "cheap": "gpt-5.4-nano", "rule": "relative, findings 17 B8" } }, { "task": "extraction", "band": "medium", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.24, "rating": 1337.69, "evidence": -2.3977, "off_curve": false, "rank": 27 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.24, "rating": 1323.37, "evidence": -2.5678, "off_curve": true, "rank": 28 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.24, "rating": 1293.1, "evidence": -2.8918, "off_curve": true, "rank": 29 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.41, "score": 82.84, "se": 2.14, "evidence": -1.021, "off_curve": false, "rank": 21 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.48, "score": 83.03, "se": 2.14, "evidence": -0.1281, "off_curve": false, "rank": 17 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.48, "score": 83.03, "se": 2.14, "evidence": -0.1281, "off_curve": true, "rank": 18 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 0.6, "rating": 1373.26, "evidence": -2.0607, "off_curve": true, "rank": 25 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 0.64, "rating": 1372.9, "evidence": -2.0714, "off_curve": true, "rank": 26 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 0.7, "rating": 1400.45, "evidence": -1.4746, "off_curve": true, "rank": 24 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 0.74, "score": 75.65, "se": 2.25, "evidence": -0.5594, "off_curve": true, "rank": 19 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 1.04, "rating": 1292.61, "evidence": -2.9013, "off_curve": true, "rank": 30 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 1.35, "score": 75.05, "se": 2.45, "evidence": 0.1335, "off_curve": true, "rank": 16 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 1.5, "score": 85.4, "se": 1.92, "evidence": 0.6648, "off_curve": false, "rank": 13 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 1.8, "rating": 1414.99, "evidence": -1.0738, "off_curve": true, "rank": 22 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 1.848, "score": 87.27, "se": 1.85, "evidence": 1.2032, "off_curve": false, "rank": 10 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 1.848, "score": 87.27, "se": 1.85, "evidence": 1.2032, "off_curve": true, "rank": 11 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 2.7, "rating": 1421.21, "evidence": -0.818, "off_curve": true, "rank": 20 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 3, "score": 80.47, "se": 2.38, "evidence": 0.1455, "off_curve": true, "rank": 15 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 3.08, "score": 88.44, "se": 1.79, "evidence": 1.2424, "off_curve": true, "rank": 9 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 3.2, "rating": 1403.29, "evidence": -1.422, "off_curve": true, "rank": 23 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 3.6, "score": 84.89, "se": 2.08, "evidence": 0.9564, "off_curve": true, "rank": 12 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 4, "score": 88.29, "se": 1.94, "evidence": 1.8696, "off_curve": false, "rank": 4 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 5, "score": 89.66, "se": 1.73, "evidence": 1.7897, "off_curve": true, "rank": 6 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 5.4, "score": 87.02, "se": 2.17, "evidence": 1.4711, "off_curve": true, "rank": 7 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 7.2, "score": 89.18, "se": 1.89, "evidence": 2.3542, "off_curve": false, "rank": 3 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 9, "score": 84.34, "se": 2.42, "evidence": 1.353, "off_curve": true, "rank": 8 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 9, "score": 80.08, "se": 2.54, "evidence": 0.4927, "off_curve": true, "rank": 14 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 10, "score": 90.79, "se": 1.66, "evidence": 2.3911, "off_curve": true, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 18, "score": 91.49, "se": 1.61, "evidence": 2.4112, "off_curve": true, "rank": 1 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 18, "score": 87.2, "se": 2.31, "evidence": 1.8341, "off_curve": true, "rank": 5 } ], "picks": { "best": "claude-opus-5-5", "balanced": "deepseek-v4-pro", "cheap": "gpt-5.4-mini", "rule": "relative, findings 17 B8" } }, { "task": "classification", "band": "medium", "points": [ { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.41, "score": 67.2, "se": 3.32, "evidence": -0.3748, "off_curve": false, "rank": 11 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.48, "score": 63.14, "se": 3.41, "evidence": -1.2003, "off_curve": true, "rank": 16 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.48, "score": 63.14, "se": 3.41, "evidence": -1.2003, "off_curve": true, "rank": 17 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 0.74, "score": 67.24, "se": 3.32, "evidence": -0.3662, "off_curve": false, "rank": 10 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 1.35, "score": 81.41, "se": 2.75, "evidence": 2.4908, "off_curve": false, "rank": 1 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 1.5, "score": 59.8, "se": 3.47, "evidence": -1.7906, "off_curve": true, "rank": 20 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 1.848, "score": 62.35, "se": 3.43, "evidence": -1.3549, "off_curve": true, "rank": 18 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 1.848, "score": 62.35, "se": 3.43, "evidence": -1.3549, "off_curve": true, "rank": 19 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 3, "score": 75.6, "se": 3.04, "evidence": 1.5559, "off_curve": true, "rank": 3 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 3.08, "score": 75.28, "se": 3.05, "evidence": 1.4818, "off_curve": true, "rank": 5 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 3.6, "score": 63.86, "se": 3.4, "evidence": -1.0575, "off_curve": true, "rank": 13 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 4, "score": 79.1, "se": 2.88, "evidence": 2.1835, "off_curve": true, "rank": 2 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 5, "score": 70.22, "se": 3.23, "evidence": 0.2976, "off_curve": true, "rank": 9 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 5.4, "score": 63.22, "se": 3.41, "evidence": -1.1843, "off_curve": true, "rank": 15 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 7.2, "score": 65.74, "se": 3.36, "evidence": -0.6836, "off_curve": true, "rank": 12 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 9, "score": 72.03, "se": 3.17, "evidence": 0.7198, "off_curve": true, "rank": 7 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 9, "score": 63.77, "se": 3.4, "evidence": -1.0756, "off_curve": true, "rank": 14 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 10, "score": 70.73, "se": 3.22, "evidence": 0.4155, "off_curve": true, "rank": 8 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 18, "score": 75.58, "se": 3.04, "evidence": 1.5514, "off_curve": true, "rank": 4 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 18, "score": 72.99, "se": 3.14, "evidence": 0.9467, "off_curve": true, "rank": 6 } ], "picks": { "best": "gemini-3.8-flash", "balanced": "gemini-3.8-flash", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "conversation", "band": "medium", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.48, "rating": 1364.93, "evidence": -2.3115, "off_curve": false, "rank": 24 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.48, "rating": 1326.76, "evidence": -2.6343, "off_curve": true, "rank": 27 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.48, "score": 8.3, "se": 0.87, "evidence": -3.7948, "off_curve": true, "rank": 30 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.82, "score": 11.7, "se": 1.02, "evidence": -2.4702, "off_curve": false, "rank": 25 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.96, "score": 33.63, "se": 1.5, "evidence": -0.4802, "off_curve": false, "rank": 16 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.96, "score": 33.63, "se": 1.5, "evidence": -0.4802, "off_curve": true, "rank": 17 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 1.2, "score": 21.6, "se": 1.3, "evidence": -2.4777, "off_curve": true, "rank": 26 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 1.28, "score": 12.7, "se": 1.05, "evidence": -2.674, "off_curve": true, "rank": 28 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 1.4, "rating": 1419.64, "evidence": -1.3967, "off_curve": true, "rank": 22 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.48, "rating": 1464.55, "evidence": 0.1278, "off_curve": false, "rank": 15 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 2.08, "rating": 1316.53, "evidence": -2.8168, "off_curve": true, "rank": 29 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 2.7, "score": 69.7, "se": 1.45, "evidence": 3.0565, "off_curve": false, "rank": 3 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 3, "score": 29.4, "se": 1.44, "evidence": -0.492, "off_curve": true, "rank": 18 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 3.6, "score": 13.2, "se": 1.07, "evidence": -2.0879, "off_curve": true, "rank": 23 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 3.696, "score": 52.91, "se": 1.58, "evidence": 1.1135, "off_curve": true, "rank": 10 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 3.696, "score": 52.91, "se": 1.58, "evidence": 1.1135, "off_curve": true, "rank": 11 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 5.4, "rating": 1432.88, "evidence": -0.9942, "off_curve": true, "rank": 19 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 6, "score": 66.2, "se": 1.5, "evidence": 2.1147, "off_curve": true, "rank": 6 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 6.16, "rating": 1424.72, "evidence": -1.2058, "off_curve": true, "rank": 21 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 6.4, "score": 31.1, "se": 1.46, "evidence": -1.1605, "off_curve": true, "rank": 20 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 7.2, "score": 33.7, "se": 1.5, "evidence": 0.2186, "off_curve": true, "rank": 14 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 8, "score": 73.5, "se": 1.4, "evidence": 3.3414, "off_curve": false, "rank": 1 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 10, "score": 45.1, "se": 1.57, "evidence": 1.0502, "off_curve": true, "rank": 12 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 10.8, "score": 32.8, "se": 1.49, "evidence": 0.5803, "off_curve": true, "rank": 13 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 14.4, "score": 72.2, "se": 1.42, "evidence": 2.9415, "off_curve": true, "rank": 4 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 18, "score": 53, "se": 1.58, "evidence": 2.0162, "off_curve": true, "rank": 8 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 18, "score": 59.9, "se": 1.55, "evidence": 1.995, "off_curve": true, "rank": 9 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 20, "score": 63, "se": 1.53, "evidence": 2.0384, "off_curve": true, "rank": 7 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 36, "score": 75.6, "se": 1.36, "evidence": 3.0952, "off_curve": false, "rank": 2 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 36, "score": 70.8, "se": 1.44, "evidence": 2.6739, "off_curve": true, "rank": 5 } ], "picks": { "best": "gemini-3.1-pro-preview", "balanced": "gemini-3.1-pro-preview", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "analysis", "band": "medium", "points": [ { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.36, "score": 22.1, "se": 2.93, "evidence": -1.808, "off_curve": false, "rank": 11 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.36, "score": 10.7, "se": 2.19, "evidence": -2.9647, "off_curve": true, "rank": 13 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.72, "score": 61.1, "se": 3.45, "evidence": 0.3062, "off_curve": false, "rank": 6 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.72, "score": 61.1, "se": 3.45, "evidence": 0.3062, "off_curve": true, "rank": 7 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 1, "score": 22.5, "se": 2.95, "evidence": -1.7759, "off_curve": true, "rank": 10 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 1.248, "score": 19.9, "se": 2.82, "evidence": -2.0236, "off_curve": true, "rank": 12 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 4.8, "score": 76.7, "se": 2.99, "evidence": 2.2267, "off_curve": false, "rank": 3 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 4.8, "score": 27, "se": 3.14, "evidence": -1.4277, "off_curve": true, "rank": 9 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 5.6, "score": 60.6, "se": 3.46, "evidence": 0.2635, "off_curve": true, "rank": 8 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 6.4, "score": 79.6, "se": 2.85, "evidence": 2.4913, "off_curve": false, "rank": 2 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 14, "score": 80.6, "se": 2.8, "evidence": 2.5747, "off_curve": false, "rank": 1 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 14, "score": 64.8, "se": 3.38, "evidence": 0.6559, "off_curve": true, "rank": 5 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 16, "score": 69, "se": 3.27, "evidence": 1.1754, "off_curve": true, "rank": 4 } ], "picks": { "best": "gemini-3.5-flash", "balanced": "gemini-3.5-flash", "cheap": "gemini-3.5-flash", "rule": "relative, findings 17 B8" } }, { "task": "analysis", "band": "hard", "points": [ { "model": "qwen3.5-9b", "price_per_1k_tasks_usd": 0.565, "score": 78.98, "se": 2.37, "evidence": 0.3085, "off_curve": false, "rank": 13 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.84, "score": 75.76, "se": 2.74, "evidence": -1.5761, "off_curve": true, "rank": 21 }, { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.84, "score": 47.47, "se": 2.76, "evidence": -3.3263, "off_curve": true, "rank": 29 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.84, "score": 37.72, "se": 2.41, "evidence": -3.9688, "off_curve": true, "rank": 31 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 1.525, "score": 66.08, "se": 2.37, "evidence": -2.3969, "off_curve": true, "rank": 27 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 1.68, "score": 79, "se": 2.02, "evidence": 0.265, "off_curve": false, "rank": 14 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 1.68, "score": 79, "se": 2.02, "evidence": 0.265, "off_curve": true, "rank": 15 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 2.24, "score": 65.85, "se": 2.71, "evidence": -2.0579, "off_curve": true, "rank": 26 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 2.3, "score": 71.72, "se": 3.21, "evidence": -1.3835, "off_curve": true, "rank": 20 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 2.35, "score": 51.33, "se": 2.69, "evidence": -2.7322, "off_curve": true, "rank": 28 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 2.85, "score": 68.73, "se": 2.28, "evidence": -0.8031, "off_curve": true, "rank": 19 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 3.016, "score": 47.44, "se": 2.85, "evidence": -3.5597, "off_curve": true, "rank": 30 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 4.875, "score": 92.34, "se": 1.3, "evidence": 3.3752, "off_curve": false, "rank": 3 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 5.55, "score": 67.1, "se": 2.4, "evidence": -1.7314, "off_curve": true, "rank": 23 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 6.204, "score": 77.96, "se": 2.06, "evidence": 0.0675, "off_curve": true, "rank": 16 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 6.204, "score": 77.96, "se": 2.06, "evidence": 0.0675, "off_curve": true, "rank": 17 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 6.5, "score": 60.48, "se": 2.82, "evidence": -2.005, "off_curve": true, "rank": 25 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 9.75, "score": 59.53, "se": 2.82, "evidence": -1.9953, "off_curve": true, "rank": 24 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 10.34, "score": 82.65, "se": 2.66, "evidence": 0.0017, "off_curve": true, "rank": 18 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 11.1, "score": 85.78, "se": 1.73, "evidence": 1.6176, "off_curve": true, "rank": 8 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 11.2, "score": 66.92, "se": 2.82, "evidence": -1.6638, "off_curve": true, "rank": 22 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 13, "score": 84.49, "se": 1.8, "evidence": 1.4387, "off_curve": true, "rank": 10 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 14.8, "score": 89.46, "se": 1.5, "evidence": 2.6119, "off_curve": true, "rank": 6 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 18.5, "score": 80.26, "se": 2.05, "evidence": 0.8132, "off_curve": true, "rank": 12 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 19.5, "score": 81.78, "se": 1.93, "evidence": 1.3207, "off_curve": true, "rank": 11 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 26, "score": 91.37, "se": 1.36, "evidence": 3.5239, "off_curve": false, "rank": 1 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 32.5, "score": 92.22, "se": 1.35, "evidence": 3.3463, "off_curve": true, "rank": 4 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 32.5, "score": 86.93, "se": 1.73, "evidence": 2.3282, "off_curve": true, "rank": 7 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 37, "score": 82.37, "se": 1.96, "evidence": 1.4793, "off_curve": true, "rank": 9 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 65, "score": 94.21, "se": 1.15, "evidence": 3.4644, "off_curve": true, "rank": 2 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 65, "score": 91.69, "se": 1.94, "evidence": 2.9054, "off_curve": true, "rank": 5 } ], "picks": { "best": "claude-opus-5-5", "balanced": "claude-opus-5-5", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "analysis", "band": "frontier", "points": [ { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 1.35, "rating": 1356.91, "evidence": -2.4067, "off_curve": false, "rank": 25 }, { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 1.35, "rating": 1334.62, "evidence": -2.542, "off_curve": true, "rank": 26 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 1.35, "score": 0, "se": 0.42, "evidence": -3.3363, "off_curve": true, "rank": 30 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 2.475, "score": 9.93, "se": 1.12, "evidence": -1.3853, "off_curve": false, "rank": 21 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 2.7, "score": 6.96, "se": 1.04, "evidence": -1.2837, "off_curve": false, "rank": 18 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 2.7, "score": 6.96, "se": 1.04, "evidence": -1.2837, "off_curve": true, "rank": 19 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 3.6, "score": 1.75, "se": 0.51, "evidence": -3.2756, "off_curve": true, "rank": 29 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 3.75, "rating": 1427.35, "evidence": -1.423, "off_curve": true, "rank": 22 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 3.75, "score": 15.75, "se": 0.94, "evidence": -1.7641, "off_curve": true, "rank": 24 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 4.65, "score": 9.26, "se": 1.09, "evidence": -0.7295, "off_curve": false, "rank": 14 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 4.68, "rating": 1297.53, "evidence": -2.8421, "off_curve": true, "rank": 27 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 7.875, "score": 44.52, "se": 1.96, "evidence": 2.3553, "off_curve": false, "rank": 7 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 9, "score": 7.06, "se": 0.97, "evidence": -0.8215, "off_curve": true, "rank": 17 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 9.9, "score": 6.92, "se": 0.94, "evidence": -0.8202, "off_curve": true, "rank": 15 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 9.9, "score": 6.92, "se": 0.94, "evidence": -0.8202, "off_curve": true, "rank": 16 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 10.5, "score": 7.79, "se": 1.01, "evidence": -1.38, "off_curve": true, "rank": 20 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 15.75, "score": 4.52, "se": 0.81, "evidence": -1.569, "off_curve": true, "rank": 23 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 16.5, "rating": 1458.63, "evidence": -0.4622, "off_curve": true, "rank": 13 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 18, "score": 28.86, "se": 1.8, "evidence": 0.9036, "off_curve": true, "rank": 12 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 18, "score": 3.77, "se": 0.52, "evidence": -3.1039, "off_curve": true, "rank": 28 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 21, "rating": 1511.75, "evidence": 1.8388, "off_curve": true, "rank": 10 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 24, "score": 73.84, "se": 1.46, "evidence": 3.2854, "off_curve": true, "rank": 4 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 30, "score": 45.04, "se": 1.77, "evidence": 1.1894, "off_curve": true, "rank": 11 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 31.5, "score": 73.46, "se": 2.39, "evidence": 2.3277, "off_curve": true, "rank": 8 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 42, "score": 94.59, "se": 1.32, "evidence": 4.0254, "off_curve": false, "rank": 2 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 52.5, "score": 92.92, "se": 1.52, "evidence": 4.2112, "off_curve": true, "rank": 1 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 52.5, "score": 75.11, "se": 2.36, "evidence": 2.8185, "off_curve": true, "rank": 6 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 60, "score": 54.75, "se": 2.4, "evidence": 2.0828, "off_curve": true, "rank": 9 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 105, "score": 73.18, "se": 1.55, "evidence": 3.2917, "off_curve": true, "rank": 3 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 105, "score": 66.79, "se": 1.73, "evidence": 2.9193, "off_curve": true, "rank": 5 } ], "picks": { "best": "claude-opus-5-5", "balanced": "claude-opus-5-5", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "math", "band": "medium", "points": [ { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.36, "rating": 1381.12, "evidence": -1.9778, "off_curve": false, "rank": 25 }, { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.36, "rating": 1337.98, "evidence": -2.533, "off_curve": true, "rank": 28 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.36, "rating": 1275.35, "evidence": -3, "off_curve": true, "rank": 30 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.66, "rating": 1424.73, "evidence": -0.6906, "off_curve": false, "rank": 21 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.72, "rating": 1424.75, "evidence": -0.6794, "off_curve": false, "rank": 19 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.72, "rating": 1424.75, "evidence": -0.6794, "off_curve": true, "rank": 20 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 0.96, "rating": 1353.43, "evidence": -2.4232, "off_curve": true, "rank": 27 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 1, "rating": 1405.04, "evidence": -1.3611, "off_curve": true, "rank": 22 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 1, "rating": 1401.5, "evidence": -1.524, "off_curve": true, "rank": 23 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.24, "rating": 1441.85, "evidence": -0.0499, "off_curve": false, "rank": 16 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 1.248, "rating": 1295.83, "evidence": -2.7931, "off_curve": true, "rank": 29 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 2.1, "rating": 1516.35, "evidence": 2.5376, "off_curve": false, "rank": 2 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 2.4, "rating": 1438.82, "evidence": -0.1686, "off_curve": true, "rank": 17 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 2.64, "rating": 1444.76, "evidence": 0.079, "off_curve": true, "rank": 14 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 2.64, "rating": 1444.76, "evidence": 0.079, "off_curve": true, "rank": 15 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 2.8, "rating": 1400.23, "evidence": -1.5534, "off_curve": true, "rank": 24 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 4.2, "rating": 1428.35, "evidence": -0.5263, "off_curve": true, "rank": 18 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 4.4, "rating": 1444.85, "evidence": 0.096, "off_curve": true, "rank": 13 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 4.8, "rating": 1496.34, "evidence": 2.0288, "off_curve": true, "rank": 6 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 4.8, "rating": 1374.56, "evidence": -2.1042, "off_curve": true, "rank": 26 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 5.6, "rating": 1473.61, "evidence": 1.2434, "off_curve": true, "rank": 10 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 6.4, "rating": 1487.87, "evidence": 1.7496, "off_curve": true, "rank": 8 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 8, "rating": 1459.91, "evidence": 0.7315, "off_curve": true, "rank": 12 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 8.4, "rating": 1463.24, "evidence": 0.8705, "off_curve": true, "rank": 11 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 11.2, "rating": 1510.21, "evidence": 2.1916, "off_curve": true, "rank": 4 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 14, "rating": 1522.49, "evidence": 2.7023, "off_curve": false, "rank": 1 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 14, "rating": 1476.25, "evidence": 1.3243, "off_curve": true, "rank": 9 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 16, "rating": 1499.39, "evidence": 2.1503, "off_curve": true, "rank": 5 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 28, "rating": 1517.46, "evidence": 2.4924, "off_curve": true, "rank": 3 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 28, "rating": 1490, "evidence": 1.7876, "off_curve": true, "rank": 7 } ], "picks": { "best": "gemini-3.8-flash", "balanced": "gemini-3.8-flash", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "math", "band": "hard", "points": [ { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.84, "score": 52.63, "se": 1.13, "evidence": -7.9632, "off_curve": false, "rank": 26 }, { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.84, "score": 46.77, "se": 1.2, "evidence": -9.2312, "off_curve": true, "rank": 28 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 1.525, "score": 90.98, "se": 1.47, "evidence": 1.5964, "off_curve": false, "rank": 13 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 1.68, "score": 79.65, "se": 2.07, "evidence": -0.9827, "off_curve": true, "rank": 20 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 1.68, "score": 79.65, "se": 2.07, "evidence": -0.9827, "off_curve": true, "rank": 21 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 2.24, "score": 87.29, "se": 0.73, "evidence": -0.4171, "off_curve": true, "rank": 18 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 2.3, "score": 97.85, "se": 0.34, "evidence": 2.4559, "off_curve": false, "rank": 9 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 2.35, "score": 50.28, "se": 1.15, "evidence": -8.5033, "off_curve": true, "rank": 27 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 2.85, "score": 73.74, "se": 2.26, "evidence": -1.6941, "off_curve": true, "rank": 23 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 3.016, "score": 41.6, "se": 1.18, "evidence": -10.5588, "off_curve": true, "rank": 29 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 4.875, "score": 91.56, "se": 1.43, "evidence": 1.8129, "off_curve": true, "rank": 11 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 5.55, "score": 78.46, "se": 2.11, "evidence": -1.1412, "off_curve": true, "rank": 22 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 6.204, "score": 90.68, "se": 1.49, "evidence": 1.4874, "off_curve": true, "rank": 14 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 6.204, "score": 90.68, "se": 1.49, "evidence": 1.4874, "off_curve": true, "rank": 15 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 6.5, "score": 86.91, "se": 0.77, "evidence": -0.509, "off_curve": true, "rank": 19 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 9.75, "score": 81.63, "se": 0.9, "evidence": -2.0461, "off_curve": true, "rank": 25 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 10.34, "score": 95.68, "se": 1.04, "evidence": 3.8662, "off_curve": true, "rank": 6 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 11.1, "score": 88.24, "se": 1.65, "evidence": 0.7551, "off_curve": true, "rank": 16 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 11.2, "score": 83.01, "se": 0.85, "evidence": -1.7079, "off_curve": true, "rank": 24 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 13, "score": 92.94, "se": 1.32, "evidence": 2.42, "off_curve": true, "rank": 10 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 14.8, "score": 91.04, "se": 1.47, "evidence": 1.6159, "off_curve": true, "rank": 12 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 18.5, "score": 94.15, "se": 1.21, "evidence": 3.0219, "off_curve": true, "rank": 8 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 19.5, "score": 86.99, "se": 1.73, "evidence": 0.4351, "off_curve": true, "rank": 17 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 26, "score": 97.08, "se": 0.87, "evidence": 4.6644, "off_curve": true, "rank": 1 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 32.5, "score": 95.73, "se": 1.04, "evidence": 3.8878, "off_curve": true, "rank": 5 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 32.5, "score": 94.32, "se": 1.19, "evidence": 3.1144, "off_curve": true, "rank": 7 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 37, "score": 95.86, "se": 1.02, "evidence": 3.9727, "off_curve": true, "rank": 4 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 65, "score": 97.01, "se": 0.88, "evidence": 4.6233, "off_curve": true, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 65, "score": 96.81, "se": 0.9, "evidence": 4.5205, "off_curve": true, "rank": 3 } ], "picks": { "best": "gpt-5-mini", "balanced": "gpt-5-mini", "cheap": "gpt-5-mini", "rule": "relative, findings 17 B8" } }, { "task": "math", "band": "frontier", "points": [ { "model": "qwen3.5-9b", "price_per_1k_tasks_usd": 0.885, "score": 42.78, "se": 6.53, "evidence": -1.0437, "off_curve": false, "rank": 20 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 1.35, "score": 88.89, "se": 4.46, "evidence": 1.4673, "off_curve": false, "rank": 9 }, { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 1.35, "score": 5.28, "se": 2.59, "evidence": -2.6614, "off_curve": true, "rank": 27 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 1.35, "score": 3.82, "se": 1.39, "evidence": -3.474, "off_curve": true, "rank": 30 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 2.475, "score": 25.68, "se": 2.9, "evidence": -1.6835, "off_curve": true, "rank": 22 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 2.7, "score": 75.99, "se": 1.8, "evidence": 1.2914, "off_curve": false, "rank": 10 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 2.7, "score": 75.99, "se": 1.8, "evidence": 1.2914, "off_curve": true, "rank": 11 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 3.6, "score": 18.62, "se": 2.15, "evidence": -1.9984, "off_curve": true, "rank": 24 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 3.75, "score": 34.09, "se": 2.88, "evidence": -0.9588, "off_curve": true, "rank": 19 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 3.75, "score": 1.14, "se": 0.5, "evidence": -3.2655, "off_curve": true, "rank": 29 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 4.65, "score": 44.63, "se": 4.22, "evidence": -0.6698, "off_curve": true, "rank": 18 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 4.68, "score": 5.14, "se": 2.36, "evidence": -2.6648, "off_curve": true, "rank": 28 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 7.875, "score": 83.66, "se": 1.45, "evidence": 2.1984, "off_curve": false, "rank": 6 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 9, "score": 23.58, "se": 2.66, "evidence": -1.9583, "off_curve": true, "rank": 23 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 9.9, "score": 54.04, "se": 4.58, "evidence": -0.5047, "off_curve": true, "rank": 16 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 9.9, "score": 54.04, "se": 4.58, "evidence": -0.5047, "off_curve": true, "rank": 17 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 10.5, "score": 19.99, "se": 3.17, "evidence": -1.6603, "off_curve": true, "rank": 21 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 15.75, "score": 16.29, "se": 2.99, "evidence": -2.1592, "off_curve": true, "rank": 25 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 18, "score": 65.25, "se": 2.19, "evidence": 1.5565, "off_curve": true, "rank": 8 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 18, "score": 16.6, "se": 2.21, "evidence": -2.2063, "off_curve": true, "rank": 26 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 21, "score": 72.8, "se": 3.33, "evidence": 0.9613, "off_curve": true, "rank": 13 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 24, "score": 64.05, "se": 1.7, "evidence": 1.7647, "off_curve": true, "rank": 7 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 30, "score": 60.88, "se": 3.29, "evidence": 1.0671, "off_curve": true, "rank": 12 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 31.5, "score": 51.76, "se": 3.69, "evidence": 0.487, "off_curve": true, "rank": 15 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 42, "score": 95.62, "se": 2.64, "evidence": 3.1546, "off_curve": false, "rank": 2 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 52.5, "score": 92.77, "se": 1.56, "evidence": 2.7932, "off_curve": true, "rank": 4 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 52.5, "score": 72.3, "se": 2.53, "evidence": 2.2264, "off_curve": true, "rank": 5 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 60, "score": 69.1, "se": 4.46, "evidence": 0.7754, "off_curve": true, "rank": 14 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 105, "score": 96.84, "se": 2.6, "evidence": 3.2765, "off_curve": false, "rank": 1 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 105, "score": 95.09, "se": 2.65, "evidence": 3.1024, "off_curve": true, "rank": 3 } ], "picks": { "best": "claude-opus-5-5", "balanced": "deepseek-flash", "cheap": "gpt-oss-120b", "rule": "relative, findings 17 B8" } }, { "task": "code_write", "band": "hard", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 1.11, "rating": 1412.14, "evidence": -2.3739, "off_curve": false, "rank": 27 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 1.11, "rating": 1389.67, "evidence": -2.5862, "off_curve": true, "rank": 28 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 1.11, "rating": 1347.98, "evidence": -2.8802, "off_curve": true, "rank": 29 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 2.0875, "score": 58.81, "se": 3.61, "evidence": -1.1148, "off_curve": false, "rank": 20 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 2.22, "score": 53.43, "se": 3.57, "evidence": -0.5864, "off_curve": false, "rank": 17 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 2.22, "score": 53.43, "se": 3.57, "evidence": -0.5864, "off_curve": true, "rank": 18 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 2.96, "rating": 1432.98, "evidence": -2.0583, "off_curve": true, "rank": 24 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 3.025, "rating": 1469.27, "evidence": -2.1545, "off_curve": true, "rank": 26 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 3.2, "rating": 1431.19, "evidence": -2.085, "off_curve": true, "rank": 25 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 3.484, "rating": 1345.74, "evidence": -2.9129, "off_curve": true, "rank": 30 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 3.975, "score": 60.66, "se": 3.53, "evidence": 0.085, "off_curve": false, "rank": 14 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 6.5625, "score": 63.36, "se": 3.59, "evidence": 1.5472, "off_curve": false, "rank": 7 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 7.575, "score": 56.65, "se": 3.58, "evidence": -0.6957, "off_curve": true, "rank": 19 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 7.986, "score": 56.31, "se": 3.6, "evidence": -0.4157, "off_curve": true, "rank": 15 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 7.986, "score": 56.31, "se": 3.6, "evidence": -0.4157, "off_curve": true, "rank": 16 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 8.75, "rating": 1480.83, "evidence": -1.5103, "off_curve": true, "rank": 21 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 13.125, "rating": 1478.63, "evidence": -1.7226, "off_curve": true, "rank": 23 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 13.31, "score": 65.58, "se": 3.52, "evidence": 1.1411, "off_curve": false, "rank": 10 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 14.8, "rating": 1456.1, "evidence": -1.6855, "off_curve": true, "rank": 22 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 15.15, "score": 63.59, "se": 3.51, "evidence": 0.8132, "off_curve": true, "rank": 11 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 17.5, "score": 70.04, "se": 3.42, "evidence": 2.0575, "off_curve": true, "rank": 5 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 20.2, "score": 43.16, "se": 2.8, "evidence": 0.5591, "off_curve": true, "rank": 13 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 25.25, "score": 48.98, "se": 3.48, "evidence": 0.5974, "off_curve": true, "rank": 12 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 26.25, "score": 60.95, "se": 3.47, "evidence": 1.3337, "off_curve": true, "rank": 9 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 35, "score": 79.46, "se": 3.42, "evidence": 4.3355, "off_curve": false, "rank": 1 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 43.75, "score": 73.33, "se": 3.33, "evidence": 3.1548, "off_curve": true, "rank": 3 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 43.75, "score": 66.17, "se": 3.44, "evidence": 2.0552, "off_curve": true, "rank": 6 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 50.5, "score": 48.71, "se": 3.05, "evidence": 1.421, "off_curve": true, "rank": 8 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 87.5, "score": 75.25, "se": 3.73, "evidence": 3.7782, "off_curve": true, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 87.5, "score": 61.46, "se": 4.57, "evidence": 2.905, "off_curve": true, "rank": 4 } ], "picks": { "best": "claude-opus-5-5", "balanced": "claude-opus-5-5", "cheap": "claude-opus-5-5", "rule": "relative, findings 17 B8" } }, { "task": "code_fix", "band": "hard", "points": [ { "model": "qwen3.5-9b", "price_per_1k_tasks_usd": 0.6775, "score": 9.21, "se": 2.35, "evidence": -3.3725, "off_curve": false, "rank": 26 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 1.11, "score": 25.56, "se": 1.91, "evidence": -1.8361, "off_curve": false, "rank": 24 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 1.11, "score": 3.6, "se": 1.24, "evidence": -5.1843, "off_curve": true, "rank": 28 }, { "model": "codestral-2508", "price_per_1k_tasks_usd": 1.815, "score": 11.1, "se": 2.09, "evidence": -3.8252, "off_curve": true, "rank": 27 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 2.0875, "score": 46.77, "se": 5.88, "evidence": 0.1732, "off_curve": false, "rank": 16 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 2.22, "score": 37.63, "se": 5.71, "evidence": -0.7906, "off_curve": true, "rank": 21 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 2.22, "score": 37.63, "se": 5.71, "evidence": -0.7906, "off_curve": true, "rank": 22 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 2.96, "score": 32.4, "se": 3.12, "evidence": -1.4148, "off_curve": true, "rank": 23 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 3.2, "score": 47, "se": 1.67, "evidence": -0.0076, "off_curve": false, "rank": 19 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 3.975, "score": 45.25, "se": 5.87, "evidence": 0.0083, "off_curve": true, "rank": 18 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 6.5625, "score": 54.24, "se": 5.87, "evidence": 1.0165, "off_curve": false, "rank": 9 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 7.575, "score": 41.67, "se": 5.81, "evidence": -0.3802, "off_curve": true, "rank": 20 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 7.986, "score": 57.05, "se": 3.52, "evidence": 0.6462, "off_curve": true, "rank": 11 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 7.986, "score": 57.05, "se": 3.52, "evidence": 0.6462, "off_curve": true, "rank": 12 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 8.75, "score": 13.9, "se": 2.7, "evidence": -2.7578, "off_curve": true, "rank": 25 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 13.31, "score": 53.99, "se": 5.87, "evidence": 0.9881, "off_curve": true, "rank": 10 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 14.8, "score": 49.83, "se": 1.88, "evidence": 0.0331, "off_curve": true, "rank": 17 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 15.15, "score": 64.16, "se": 3.09, "evidence": 1.5525, "off_curve": true, "rank": 6 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 17.5, "score": 59.39, "se": 5.79, "evidence": 1.6167, "off_curve": false, "rank": 5 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 20.2, "score": 49.78, "se": 4, "evidence": 0.2285, "off_curve": true, "rank": 15 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 25.25, "score": 69.68, "se": 2.34, "evidence": 1.9421, "off_curve": true, "rank": 4 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 26.25, "score": 59.89, "se": 2.15, "evidence": 0.5335, "off_curve": true, "rank": 14 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 35, "score": 71.72, "se": 5.31, "evidence": 2.9518, "off_curve": false, "rank": 1 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 43.75, "score": 65.2, "se": 5.61, "evidence": 2.2966, "off_curve": true, "rank": 3 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 43.75, "score": 50.51, "se": 5.89, "evidence": 0.5911, "off_curve": true, "rank": 13 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 50.5, "score": 59.02, "se": 3.58, "evidence": 1.3669, "off_curve": true, "rank": 8 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 87.5, "score": 66.06, "se": 5.58, "evidence": 2.3957, "off_curve": true, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 87.5, "score": 57.32, "se": 5.83, "evidence": 1.3729, "off_curve": true, "rank": 7 } ], "picks": { "best": "claude-opus-5-5", "balanced": "claude-opus-5-5", "cheap": "claude-sonnet-5", "rule": "relative, findings 17 B8" } }, { "task": "query", "band": "medium", "points": [ { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.41, "score": 50.59, "se": 6.93, "evidence": 0.1317, "off_curve": false, "rank": 9 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 0.48, "score": 46.12, "se": 6.91, "evidence": -0.3498, "off_curve": true, "rank": 14 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 0.48, "score": 46.12, "se": 6.91, "evidence": -0.3498, "off_curve": true, "rank": 15 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 0.74, "score": 40.62, "se": 6.81, "evidence": -0.9494, "off_curve": true, "rank": 20 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 1.35, "score": 45.33, "se": 6.9, "evidence": -0.4353, "off_curve": true, "rank": 16 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 1.5, "score": 49.39, "se": 6.93, "evidence": 0.0025, "off_curve": true, "rank": 11 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 1.848, "score": 48.19, "se": 6.93, "evidence": -0.1266, "off_curve": true, "rank": 12 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 1.848, "score": 48.19, "se": 6.93, "evidence": -0.1266, "off_curve": true, "rank": 13 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 3, "score": 50.5, "se": 6.93, "evidence": 0.122, "off_curve": true, "rank": 10 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 3.08, "score": 44.92, "se": 6.9, "evidence": -0.4795, "off_curve": true, "rank": 18 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 3.6, "score": 42.44, "se": 6.85, "evidence": -0.7496, "off_curve": true, "rank": 19 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 4, "score": 52.35, "se": 6.93, "evidence": 0.321, "off_curve": false, "rank": 4 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 5, "score": 51.75, "se": 6.93, "evidence": 0.2565, "off_curve": true, "rank": 7 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 5.4, "score": 45.08, "se": 6.9, "evidence": -0.4622, "off_curve": true, "rank": 17 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 7.2, "score": 52.33, "se": 6.93, "evidence": 0.3189, "off_curve": true, "rank": 5 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 9, "score": 51.96, "se": 6.93, "evidence": 0.2791, "off_curve": true, "rank": 6 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 9, "score": 51.08, "se": 6.93, "evidence": 0.1844, "off_curve": true, "rank": 8 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 10, "score": 55.9, "se": 6.89, "evidence": 0.7051, "off_curve": false, "rank": 3 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 18, "score": 58.37, "se": 6.84, "evidence": 0.9752, "off_curve": false, "rank": 1 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 18, "score": 56.15, "se": 6.88, "evidence": 0.7326, "off_curve": true, "rank": 2 } ], "picks": { "best": "gpt-5.4-nano", "balanced": "gpt-5.4-nano", "cheap": "gpt-5.4-nano", "rule": "relative, findings 17 B8" } }, { "task": "tool_use", "band": "hard", "points": [ { "model": "qwen3.5-9b", "price_per_1k_tasks_usd": 1.13, "score": 9.21, "se": 2.35, "evidence": -2.5467, "off_curve": false, "rank": 9 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 1.68, "score": 14.2, "se": 2.3, "evidence": -1.815, "off_curve": false, "rank": 7 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 4.6, "score": 22.25, "se": 2.62, "evidence": -0.6393, "off_curve": false, "rank": 6 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 13, "score": 13.9, "se": 2.7, "evidence": -1.856, "off_curve": true, "rank": 8 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 19.5, "rating": -0.12, "evidence": 0, "off_curve": false, "rank": 5 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 29.6, "score": 59.36, "se": 4.2, "evidence": 1.2581, "off_curve": false, "rank": 3 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 37, "score": 81.8, "se": 2, "evidence": 3, "off_curve": false, "rank": 1 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 39, "score": 62.75, "se": 1.82, "evidence": 0.6885, "off_curve": true, "rank": 4 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 74, "score": 66.07, "se": 2.49, "evidence": 1.9105, "off_curve": true, "rank": 2 } ], "picks": { "best": "gpt-5.4", "balanced": "gpt-5.4", "cheap": "gpt-5.4", "rule": "relative, findings 17 B8" } }, { "task": "vision", "band": "medium", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.36, "rating": 1141.55, "evidence": -2.7, "off_curve": false, "rank": 20 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.36, "rating": 1065.48, "evidence": -3, "off_curve": true, "rank": 21 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 0.66, "rating": 1196.06, "evidence": -2.0098, "off_curve": false, "rank": 18 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 0.96, "rating": 1181.29, "evidence": -2.3977, "off_curve": true, "rank": 19 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 1, "rating": 1221.46, "evidence": -1.1038, "off_curve": false, "rank": 15 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 1, "rating": 1201.96, "evidence": -1.8223, "off_curve": true, "rank": 17 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.24, "rating": 1266.09, "evidence": -0.1767, "off_curve": false, "rank": 12 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 2.1, "rating": 1312.02, "evidence": 2.3731, "off_curve": false, "rank": 3 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 2.4, "rating": 1244.85, "evidence": -0.6, "off_curve": true, "rank": 13 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 4.2, "rating": 1222.37, "evidence": -1.0911, "off_curve": true, "rank": 14 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 4.8, "rating": 1310.49, "evidence": 2.3606, "off_curve": true, "rank": 4 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 4.8, "rating": 1210.31, "evidence": -1.4754, "off_curve": true, "rank": 16 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 5.6, "rating": 1275.4, "evidence": 0.1449, "off_curve": true, "rank": 11 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 6.4, "rating": 1296.21, "evidence": 1.4895, "off_curve": true, "rank": 6 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 8, "rating": 1291.91, "evidence": 1.1757, "off_curve": true, "rank": 7 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 8.4, "rating": 1282.75, "evidence": 0.5404, "off_curve": true, "rank": 9 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 14, "rating": 1320.86, "evidence": 2.729, "off_curve": false, "rank": 1 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 14, "rating": 1287.93, "evidence": 0.8991, "off_curve": true, "rank": 8 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 16, "rating": 1297.26, "evidence": 1.5571, "off_curve": true, "rank": 5 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 28, "rating": 1318.93, "evidence": 2.6437, "off_curve": true, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 28, "rating": 1280.05, "evidence": 0.4636, "off_curve": true, "rank": 10 } ], "picks": { "best": "claude-opus-5", "balanced": "claude-opus-5", "cheap": "gemini-3.8-flash", "rule": "relative, findings 17 B8" } }, { "task": "document_qa", "band": "medium", "points": [ { "model": "mistral-small-2603", "price_per_1k_tasks_usd": 0.72, "rating": 1358.76, "evidence": -2.3356, "off_curve": false, "rank": 27 }, { "model": "gpt-4o-mini", "price_per_1k_tasks_usd": 0.72, "rating": 1326.18, "evidence": -2.6932, "off_curve": true, "rank": 28 }, { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 0.72, "rating": 1324.42, "evidence": -2.7316, "off_curve": true, "rank": 29 }, { "model": "gpt-5.4-nano", "price_per_1k_tasks_usd": 1.14, "rating": 1402.6, "evidence": -1.7689, "off_curve": false, "rank": 24 }, { "model": "deepseek-flash", "price_per_1k_tasks_usd": 1.44, "rating": 1447.41, "evidence": -0.4232, "off_curve": false, "rank": 17 }, { "model": "deepseek-flash@together", "price_per_1k_tasks_usd": 1.44, "rating": 1447.41, "evidence": -0.4232, "off_curve": true, "rank": 18 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 1.6, "rating": 1370.25, "evidence": -2.2161, "off_curve": true, "rank": 26 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 1.92, "rating": 1389.88, "evidence": -1.9552, "off_curve": true, "rank": 25 }, { "model": "gemini-3.5-flash-lite", "price_per_1k_tasks_usd": 1.96, "rating": 1461.62, "evidence": 0.0922, "off_curve": false, "rank": 15 }, { "model": "mistral-large-2512", "price_per_1k_tasks_usd": 2.2, "rating": 1416.95, "evidence": -1.516, "off_curve": true, "rank": 23 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 3.744, "rating": 1313.38, "evidence": -2.9546, "off_curve": true, "rank": 30 }, { "model": "gemini-3.8-flash", "price_per_1k_tasks_usd": 3.9, "rating": 1507.93, "evidence": 2.5444, "off_curve": false, "rank": 3 }, { "model": "gpt-5.4-mini", "price_per_1k_tasks_usd": 4.2, "rating": 1446.73, "evidence": -0.4447, "off_curve": true, "rank": 19 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 5.2, "rating": 1436.32, "evidence": -0.9762, "off_curve": true, "rank": 20 }, { "model": "deepseek-v4-pro", "price_per_1k_tasks_usd": 5.808, "rating": 1471.65, "evidence": 0.4986, "off_curve": true, "rank": 13 }, { "model": "deepseek-v4-pro@together", "price_per_1k_tasks_usd": 5.808, "rating": 1471.65, "evidence": 0.4986, "off_curve": true, "rank": 14 }, { "model": "mistral-medium-3-5", "price_per_1k_tasks_usd": 7.8, "rating": 1429.62, "evidence": -1.1737, "off_curve": true, "rank": 21 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 8.4, "rating": 1481.48, "evidence": 1.0726, "off_curve": true, "rank": 11 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 9.6, "rating": 1426.9, "evidence": -1.2569, "off_curve": true, "rank": 22 }, { "model": "grok-4.7", "price_per_1k_tasks_usd": 9.68, "rating": 1450.54, "evidence": -0.3571, "off_curve": true, "rank": 16 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 10.4, "rating": 1481.83, "evidence": 1.0956, "off_curve": true, "rank": 10 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 11.2, "rating": 1498.89, "evidence": 2.0962, "off_curve": true, "rank": 6 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 14, "rating": 1477.36, "evidence": 0.8229, "off_curve": true, "rank": 12 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 15.6, "rating": 1492.39, "evidence": 1.7065, "off_curve": true, "rank": 7 }, { "model": "claude-opus-5-5", "price_per_1k_tasks_usd": 20.8, "rating": 1527.81, "evidence": 2.9319, "off_curve": false, "rank": 1 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 26, "rating": 1504.8, "evidence": 2.4232, "off_curve": true, "rank": 4 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 26, "rating": 1499.75, "evidence": 2.1502, "off_curve": true, "rank": 5 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 28, "rating": 1486.64, "evidence": 1.3706, "off_curve": true, "rank": 8 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 52, "rating": 1511.39, "evidence": 2.6357, "off_curve": true, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 52, "rating": 1485.58, "evidence": 1.2869, "off_curve": true, "rank": 9 } ], "picks": { "best": "claude-opus-5-5", "balanced": "claude-opus-5-5", "cheap": "claude-opus-5-5", "rule": "relative, findings 17 B8" } }, { "task": "document_qa", "band": "hard", "points": [ { "model": "gpt-oss-120b", "price_per_1k_tasks_usd": 1.74, "score": 36.1, "se": 5, "evidence": -1.6945, "off_curve": false, "rank": 13 }, { "model": "gpt-5-mini", "price_per_1k_tasks_usd": 3.8, "score": 69.4, "se": 5, "evidence": 2.4445, "off_curve": false, "rank": 3 }, { "model": "gpt-4.1-mini", "price_per_1k_tasks_usd": 4.64, "score": 44.4, "se": 5, "evidence": -0.7535, "off_curve": false, "rank": 10 }, { "model": "llama-3.3-70b-instruct-turbo", "price_per_1k_tasks_usd": 9.256, "score": 33.3, "se": 5, "evidence": -1.9914, "off_curve": false, "rank": 14 }, { "model": "claude-haiku-4-5", "price_per_1k_tasks_usd": 12.5, "rating": 1419.97, "evidence": -3, "off_curve": false, "rank": 16 }, { "model": "gemini-3.5-flash", "price_per_1k_tasks_usd": 20.1, "rating": 1463.27, "evidence": -0.8703, "off_curve": false, "rank": 12 }, { "model": "gpt-4.1", "price_per_1k_tasks_usd": 23.2, "score": 63.9, "se": 5, "evidence": 1.995, "off_curve": false, "rank": 4 }, { "model": "claude-sonnet-5", "price_per_1k_tasks_usd": 25, "rating": 1466.14, "evidence": -0.6239, "off_curve": false, "rank": 9 }, { "model": "gemini-3.1-pro-preview", "price_per_1k_tasks_usd": 26.8, "rating": 1443.92, "evidence": -2.4, "off_curve": false, "rank": 15 }, { "model": "gpt-5.4", "price_per_1k_tasks_usd": 33.5, "rating": 1471, "evidence": -0.1193, "off_curve": false, "rank": 7 }, { "model": "claude-sonnet-4-6", "price_per_1k_tasks_usd": 37.5, "rating": 1482.24, "evidence": 1.2688, "off_curve": false, "rank": 6 }, { "model": "claude-opus-5", "price_per_1k_tasks_usd": 62.5, "rating": 1516.27, "evidence": 2.7432, "off_curve": false, "rank": 1 }, { "model": "claude-opus-4-8", "price_per_1k_tasks_usd": 62.5, "rating": 1464.07, "evidence": -0.8144, "off_curve": false, "rank": 11 }, { "model": "gpt-5.5", "price_per_1k_tasks_usd": 67, "rating": 1485.72, "evidence": 1.5087, "off_curve": false, "rank": 5 }, { "model": "claude-fable-5-1", "price_per_1k_tasks_usd": 125, "rating": 1512.58, "evidence": 2.6568, "off_curve": false, "rank": 2 }, { "model": "gpt-6-astra", "price_per_1k_tasks_usd": 125, "rating": 1467.52, "evidence": -0.3496, "off_curve": false, "rank": 8 } ], "picks": { "rule": "no priced evidence" } } ], "rankings": [ { "task": "writing", "band": "medium", "entries": [ { "rank": 1, "model": "gemini-3.8-flash", "score": 83.9, "se": 3.13, "evidence": 2.8788, "tied_with_next": true }, { "rank": 2, "model": "claude-fable-5-1", "score": 80.51, "se": 3.43, "evidence": 2.716, "tied_with_next": true }, { "rank": 3, "model": "gpt-6-astra", "score": 83.84, "se": 3.17, "evidence": 2.6149, "tied_with_next": true }, { "rank": 4, "model": "gemini-3.1-pro-preview", "score": 82.62, "se": 3.18, "evidence": 2.5373, "tied_with_next": true }, { "rank": 5, "model": "claude-opus-5", "score": 77.21, "se": 3.6, "evidence": 2.3657, "tied_with_next": true }, { "rank": 6, "model": "gpt-5.5", "score": 79.55, "se": 3.46, "evidence": 2.1771, "tied_with_next": true }, { "rank": 7, "model": "claude-opus-5-5", "score": 75.27, "se": 3.67, "evidence": 2.177, "tied_with_next": true }, { "rank": 8, "model": "gemini-3.5-flash", "score": 80.05, "se": 3.38, "evidence": 1.8648, "tied_with_next": true }, { "rank": 9, "model": "claude-opus-4-8", "score": 73.77, "se": 3.69, "evidence": 1.406, "tied_with_next": true }, { "rank": 10, "model": "grok-4.7", "score": 79.1, "se": 3.35, "evidence": 1.0905, "tied_with_next": true }, { "rank": 11, "model": "gpt-5.4", "score": 73.03, "se": 3.74, "evidence": 0.9538, "tied_with_next": true }, { "rank": 12, "model": "claude-sonnet-4-6", "score": 67.75, "se": 3.88, "evidence": 0.587, "tied_with_next": true }, { "rank": 13, "model": "claude-sonnet-5", "score": 68.11, "se": 3.87, "evidence": 0.3099 }, { "rank": 14, "model": "deepseek-v4-pro", "score": 67.18, "se": 3.9, "evidence": 0.0157, "tied_with_next": true }, { "rank": 15, "model": "deepseek-v4-pro@together", "score": 67.18, "se": 3.9, "evidence": 0.0157 }, { "rank": 16, "model": "gemini-3.5-flash-lite", "score": 68.41, "se": 3.84, "evidence": -0.2048 }, { "rank": 17, "model": "gpt-5.4-mini", "score": 64.92, "se": 3.94, "evidence": -0.6631, "tied_with_next": true }, { "rank": 18, "model": "deepseek-flash", "score": 66.25, "se": 3.9, "evidence": -0.7168, "tied_with_next": true }, { "rank": 19, "model": "deepseek-flash@together", "score": 66.25, "se": 3.9, "evidence": -0.7168 }, { "rank": 20, "model": "mistral-medium-3-5", "rating": 1421.21, "evidence": -0.818 }, { "rank": 21, "model": "claude-haiku-4-5", "rating": 1414.99, "evidence": -1.0738 }, { "rank": 22, "model": "gpt-4.1", "rating": 1403.29, "evidence": -1.422, "tied_with_next": true }, { "rank": 23, "model": "mistral-large-2512", "rating": 1400.45, "evidence": -1.4746 }, { "rank": 24, "model": "gpt-5.4-nano", "score": 64.58, "se": 3.88, "evidence": -1.7294 }, { "rank": 25, "model": "gpt-5-mini", "rating": 1373.26, "evidence": -2.0607, "tied_with_next": true }, { "rank": 26, "model": "gpt-4.1-mini", "rating": 1372.9, "evidence": -2.0714 }, { "rank": 27, "model": "mistral-small-2603", "rating": 1337.69, "evidence": -2.3977 }, { "rank": 28, "model": "gpt-oss-120b", "rating": 1323.37, "evidence": -2.5678 }, { "rank": 29, "model": "gpt-4o-mini", "rating": 1293.1, "evidence": -2.8918, "tied_with_next": true }, { "rank": 30, "model": "llama-3.3-70b-instruct-turbo", "rating": 1292.61, "evidence": -2.9013 } ] }, { "task": "creative", "band": "medium", "entries": [ { "rank": 1, "model": "gemini-3.8-flash", "score": 82.75, "se": 5.34, "evidence": 2.6239, "tied_with_next": true }, { "rank": 2, "model": "gemini-3.1-pro-preview", "score": 77.25, "se": 5.93, "evidence": 2.247, "tied_with_next": true }, { "rank": 3, "model": "claude-fable-5-1", "score": 78.1, "se": 5.85, "evidence": 2.2124 }, { "rank": 4, "model": "claude-opus-4-8", "score": 80.6, "se": 5.59, "evidence": 2.1442, "tied_with_next": true }, { "rank": 5, "model": "claude-opus-5-5", "score": 70.92, "se": 6.42, "evidence": 1.9827 }, { "rank": 6, "model": "gemini-3.5-flash", "score": 75.2, "se": 6.11, "evidence": 1.7929 }, { "rank": 7, "model": "gpt-6-astra", "score": 73.98, "se": 6.2, "evidence": 1.1725, "tied_with_next": true }, { "rank": 8, "model": "gpt-5.5", "score": 72.55, "se": 6.31, "evidence": 1.1307 }, { "rank": 9, "model": "gpt-5.4", "score": 79.02, "se": 5.76, "evidence": 1.0924, "tied_with_next": true }, { "rank": 10, "model": "claude-opus-5", "score": 61.32, "se": 6.89, "evidence": 0.971, "tied_with_next": true }, { "rank": 11, "model": "claude-sonnet-4-6", "score": 68.9, "se": 6.55, "evidence": 0.9549, "tied_with_next": true }, { "rank": 12, "model": "grok-4.7", "score": 74.92, "se": 6.13, "evidence": 0.9108, "tied_with_next": true }, { "rank": 13, "model": "deepseek-v4-pro", "score": 67.75, "se": 6.61, "evidence": 0.7389, "tied_with_next": true }, { "rank": 14, "model": "deepseek-v4-pro@together", "score": 67.75, "se": 6.61, "evidence": 0.7389 }, { "rank": 15, "model": "gemini-3.5-flash-lite", "score": 67.47, "se": 6.63, "evidence": 0.4014, "tied_with_next": true }, { "rank": 16, "model": "claude-sonnet-5", "score": 62.32, "se": 6.85, "evidence": 0.1035 }, { "rank": 17, "model": "deepseek-flash", "score": 67.27, "se": 6.64, "evidence": -0.224, "tied_with_next": true }, { "rank": 18, "model": "deepseek-flash@together", "score": 67.27, "se": 6.64, "evidence": -0.224, "tied_with_next": true }, { "rank": 19, "model": "gpt-5.4-mini", "score": 66.63, "se": 6.67, "evidence": -0.3367, "tied_with_next": true }, { "rank": 20, "model": "gpt-4.1", "rating": 1402.38, "evidence": -0.7775, "tied_with_next": true }, { "rank": 21, "model": "mistral-medium-3-5", "rating": 1400.31, "evidence": -0.8424 }, { "rank": 22, "model": "claude-haiku-4-5", "rating": 1387.4, "evidence": -1.3012 }, { "rank": 23, "model": "gpt-5.4-nano", "score": 67.65, "se": 6.62, "evidence": -1.3274 }, { "rank": 24, "model": "mistral-large-2512", "rating": 1373.37, "evidence": -1.5517 }, { "rank": 25, "model": "gpt-4.1-mini", "rating": 1349.05, "evidence": -1.7853 }, { "rank": 26, "model": "mistral-small-2603", "rating": 1323.22, "evidence": -2.2327, "tied_with_next": true }, { "rank": 27, "model": "gpt-5-mini", "rating": 1323.71, "evidence": -2.236 }, { "rank": 28, "model": "gpt-4o-mini", "rating": 1293.93, "evidence": -2.6455 }, { "rank": 29, "model": "llama-3.3-70b-instruct-turbo", "rating": 1285.34, "evidence": -2.8098 }, { "rank": 30, "model": "gpt-oss-120b", "rating": 1277.82, "evidence": -2.9241 } ] }, { "task": "summarization", "band": "medium", "entries": [ { "rank": 1, "model": "gemini-3.1-pro-preview", "score": 88.23, "se": 4.56, "evidence": 2.045, "tied_with_next": true }, { "rank": 2, "model": "gemini-3.8-flash", "score": 87.5, "se": 4.68, "evidence": 1.9395, "tied_with_next": true }, { "rank": 3, "model": "gemini-3.5-flash", "score": 82.58, "se": 5.36, "evidence": 1.2313, "tied_with_next": true }, { "rank": 4, "model": "gpt-6-astra", "score": 79.55, "se": 5.7, "evidence": 0.8212, "tied_with_next": true }, { "rank": 5, "model": "grok-4.7", "score": 78.17, "se": 5.84, "evidence": 0.6422, "tied_with_next": true }, { "rank": 6, "model": "claude-opus-4-8", "score": 77.67, "se": 5.89, "evidence": 0.5783, "tied_with_next": true }, { "rank": 7, "model": "gpt-5.5", "score": 72.32, "se": 6.33, "evidence": -0.0754, "tied_with_next": true }, { "rank": 8, "model": "claude-fable-5-1", "score": 72.17, "se": 6.34, "evidence": -0.0931, "tied_with_next": true }, { "rank": 9, "model": "gpt-5.4-nano", "score": 71.77, "se": 6.37, "evidence": -0.1401, "tied_with_next": true }, { "rank": 10, "model": "claude-sonnet-5", "score": 71.22, "se": 6.4, "evidence": -0.2046, "tied_with_next": true }, { "rank": 11, "model": "deepseek-v4-pro", "score": 69.5, "se": 6.51, "evidence": -0.4037, "tied_with_next": true }, { "rank": 12, "model": "deepseek-v4-pro@together", "score": 69.5, "se": 6.51, "evidence": -0.4037, "tied_with_next": true }, { "rank": 13, "model": "claude-opus-5-5", "score": 69.1, "se": 6.53, "evidence": -0.4498, "tied_with_next": true }, { "rank": 14, "model": "deepseek-flash", "score": 68.85, "se": 6.55, "evidence": -0.4781, "tied_with_next": true }, { "rank": 15, "model": "deepseek-flash@together", "score": 68.85, "se": 6.55, "evidence": -0.4781, "tied_with_next": true }, { "rank": 16, "model": "gemini-3.5-flash-lite", "score": 68.47, "se": 6.57, "evidence": -0.5215, "tied_with_next": true }, { "rank": 17, "model": "gpt-5.4", "score": 68.42, "se": 6.57, "evidence": -0.5274, "tied_with_next": true }, { "rank": 18, "model": "claude-opus-5", "score": 66.43, "se": 6.68, "evidence": -0.7521, "tied_with_next": true }, { "rank": 19, "model": "claude-sonnet-4-6", "score": 61.65, "se": 6.88, "evidence": -1.2655, "tied_with_next": true }, { "rank": 20, "model": "gpt-5.4-mini", "score": 59.63, "se": 6.94, "evidence": -1.4642 } ] }, { "task": "extraction", "band": "medium", "entries": [ { "rank": 1, "model": "gpt-6-astra", "score": 91.49, "se": 1.61, "evidence": 2.4112, "tied_with_next": true }, { "rank": 2, "model": "gpt-5.5", "score": 90.79, "se": 1.66, "evidence": 2.3911, "tied_with_next": true }, { "rank": 3, "model": "claude-opus-5-5", "score": 89.18, "se": 1.89, "evidence": 2.3542 }, { "rank": 4, "model": "gemini-3.1-pro-preview", "score": 88.29, "se": 1.94, "evidence": 1.8696, "tied_with_next": true }, { "rank": 5, "model": "claude-fable-5-1", "score": 87.2, "se": 2.31, "evidence": 1.8341, "tied_with_next": true }, { "rank": 6, "model": "gpt-5.4", "score": 89.66, "se": 1.73, "evidence": 1.7897, "tied_with_next": true }, { "rank": 7, "model": "claude-sonnet-4-6", "score": 87.02, "se": 2.17, "evidence": 1.4711, "tied_with_next": true }, { "rank": 8, "model": "claude-opus-5", "score": 84.34, "se": 2.42, "evidence": 1.353, "tied_with_next": true }, { "rank": 9, "model": "grok-4.7", "score": 88.44, "se": 1.79, "evidence": 1.2424, "tied_with_next": true }, { "rank": 10, "model": "deepseek-v4-pro", "score": 87.27, "se": 1.85, "evidence": 1.2032, "tied_with_next": true }, { "rank": 11, "model": "deepseek-v4-pro@together", "score": 87.27, "se": 1.85, "evidence": 1.2032, "tied_with_next": true }, { "rank": 12, "model": "claude-sonnet-5", "score": 84.89, "se": 2.08, "evidence": 0.9564, "tied_with_next": true }, { "rank": 13, "model": "gpt-5.4-mini", "score": 85.4, "se": 1.92, "evidence": 0.6648, "tied_with_next": true }, { "rank": 14, "model": "claude-opus-4-8", "score": 80.08, "se": 2.54, "evidence": 0.4927 }, { "rank": 15, "model": "gemini-3.5-flash", "score": 80.47, "se": 2.38, "evidence": 0.1455, "tied_with_next": true }, { "rank": 16, "model": "gemini-3.8-flash", "score": 75.05, "se": 2.45, "evidence": 0.1335, "tied_with_next": true }, { "rank": 17, "model": "deepseek-flash", "score": 83.03, "se": 2.14, "evidence": -0.1281, "tied_with_next": true }, { "rank": 18, "model": "deepseek-flash@together", "score": 83.03, "se": 2.14, "evidence": -0.1281, "tied_with_next": true }, { "rank": 19, "model": "gemini-3.5-flash-lite", "score": 75.65, "se": 2.25, "evidence": -0.5594 }, { "rank": 20, "model": "mistral-medium-3-5", "rating": 1421.21, "evidence": -0.818 }, { "rank": 21, "model": "gpt-5.4-nano", "score": 82.84, "se": 2.14, "evidence": -1.021 }, { "rank": 22, "model": "claude-haiku-4-5", "rating": 1414.99, "evidence": -1.0738 }, { "rank": 23, "model": "gpt-4.1", "rating": 1403.29, "evidence": -1.422, "tied_with_next": true }, { "rank": 24, "model": "mistral-large-2512", "rating": 1400.45, "evidence": -1.4746 }, { "rank": 25, "model": "gpt-5-mini", "rating": 1373.26, "evidence": -2.0607, "tied_with_next": true }, { "rank": 26, "model": "gpt-4.1-mini", "rating": 1372.9, "evidence": -2.0714 }, { "rank": 27, "model": "mistral-small-2603", "rating": 1337.69, "evidence": -2.3977 }, { "rank": 28, "model": "gpt-oss-120b", "rating": 1323.37, "evidence": -2.5678 }, { "rank": 29, "model": "gpt-4o-mini", "rating": 1293.1, "evidence": -2.8918, "tied_with_next": true }, { "rank": 30, "model": "llama-3.3-70b-instruct-turbo", "rating": 1292.61, "evidence": -2.9013 } ] }, { "task": "classification", "band": "medium", "entries": [ { "rank": 1, "model": "gemini-3.8-flash", "score": 81.41, "se": 2.75, "evidence": 2.4908, "tied_with_next": true }, { "rank": 2, "model": "gemini-3.1-pro-preview", "score": 79.1, "se": 2.88, "evidence": 2.1835, "tied_with_next": true }, { "rank": 3, "model": "gemini-3.5-flash", "score": 75.6, "se": 3.04, "evidence": 1.5559, "tied_with_next": true }, { "rank": 4, "model": "gpt-6-astra", "score": 75.58, "se": 3.04, "evidence": 1.5514, "tied_with_next": true }, { "rank": 5, "model": "grok-4.7", "score": 75.28, "se": 3.05, "evidence": 1.4818, "tied_with_next": true }, { "rank": 6, "model": "claude-fable-5-1", "score": 72.99, "se": 3.14, "evidence": 0.9467, "tied_with_next": true }, { "rank": 7, "model": "claude-opus-4-8", "score": 72.03, "se": 3.17, "evidence": 0.7198, "tied_with_next": true }, { "rank": 8, "model": "gpt-5.5", "score": 70.73, "se": 3.22, "evidence": 0.4155, "tied_with_next": true }, { "rank": 9, "model": "gpt-5.4", "score": 70.22, "se": 3.23, "evidence": 0.2976, "tied_with_next": true }, { "rank": 10, "model": "gemini-3.5-flash-lite", "score": 67.24, "se": 3.32, "evidence": -0.3662, "tied_with_next": true }, { "rank": 11, "model": "gpt-5.4-nano", "score": 67.2, "se": 3.32, "evidence": -0.3748, "tied_with_next": true }, { "rank": 12, "model": "claude-opus-5-5", "score": 65.74, "se": 3.36, "evidence": -0.6836, "tied_with_next": true }, { "rank": 13, "model": "claude-sonnet-5", "score": 63.86, "se": 3.4, "evidence": -1.0575, "tied_with_next": true }, { "rank": 14, "model": "claude-opus-5", "score": 63.77, "se": 3.4, "evidence": -1.0756, "tied_with_next": true }, { "rank": 15, "model": "claude-sonnet-4-6", "score": 63.22, "se": 3.41, "evidence": -1.1843, "tied_with_next": true }, { "rank": 16, "model": "deepseek-flash", "score": 63.14, "se": 3.41, "evidence": -1.2003, "tied_with_next": true }, { "rank": 17, "model": "deepseek-flash@together", "score": 63.14, "se": 3.41, "evidence": -1.2003, "tied_with_next": true }, { "rank": 18, "model": "deepseek-v4-pro", "score": 62.35, "se": 3.43, "evidence": -1.3549, "tied_with_next": true }, { "rank": 19, "model": "deepseek-v4-pro@together", "score": 62.35, "se": 3.43, "evidence": -1.3549, "tied_with_next": true }, { "rank": 20, "model": "gpt-5.4-mini", "score": 59.8, "se": 3.47, "evidence": -1.7906 } ] }, { "task": "conversation", "band": "medium", "entries": [ { "rank": 1, "model": "gemini-3.1-pro-preview", "score": 73.5, "se": 1.4, "evidence": 3.3414, "tied_with_next": true }, { "rank": 2, "model": "gpt-6-astra", "score": 75.6, "se": 1.36, "evidence": 3.0952 }, { "rank": 3, "model": "gemini-3.8-flash", "score": 69.7, "se": 1.45, "evidence": 3.0565, "tied_with_next": true }, { "rank": 4, "model": "claude-opus-5-5", "score": 72.2, "se": 1.42, "evidence": 2.9415, "tied_with_next": true }, { "rank": 5, "model": "claude-fable-5-1", "score": 70.8, "se": 1.44, "evidence": 2.6739 }, { "rank": 6, "model": "gemini-3.5-flash", "score": 66.2, "se": 1.5, "evidence": 2.1147, "tied_with_next": true }, { "rank": 7, "model": "gpt-5.5", "score": 63, "se": 1.53, "evidence": 2.0384, "tied_with_next": true }, { "rank": 8, "model": "claude-opus-4-8", "score": 53, "se": 1.58, "evidence": 2.0162 }, { "rank": 9, "model": "claude-opus-5", "score": 59.9, "se": 1.55, "evidence": 1.995 }, { "rank": 10, "model": "deepseek-v4-pro", "score": 52.91, "se": 1.58, "evidence": 1.1135, "tied_with_next": true }, { "rank": 11, "model": "deepseek-v4-pro@together", "score": 52.91, "se": 1.58, "evidence": 1.1135 }, { "rank": 12, "model": "gpt-5.4", "score": 45.1, "se": 1.57, "evidence": 1.0502 }, { "rank": 13, "model": "claude-sonnet-4-6", "score": 32.8, "se": 1.49, "evidence": 0.5803, "tied_with_next": true }, { "rank": 14, "model": "claude-sonnet-5", "score": 33.7, "se": 1.5, "evidence": 0.2186 }, { "rank": 15, "model": "gemini-3.5-flash-lite", "rating": 1464.55, "evidence": 0.1278 }, { "rank": 16, "model": "deepseek-flash", "score": 33.63, "se": 1.5, "evidence": -0.4802, "tied_with_next": true }, { "rank": 17, "model": "deepseek-flash@together", "score": 33.63, "se": 1.5, "evidence": -0.4802, "tied_with_next": true }, { "rank": 18, "model": "gpt-5.4-mini", "score": 29.4, "se": 1.44, "evidence": -0.492 }, { "rank": 19, "model": "mistral-medium-3-5", "rating": 1432.88, "evidence": -0.9942, "tied_with_next": true }, { "rank": 20, "model": "gpt-4.1", "score": 31.1, "se": 1.46, "evidence": -1.1605, "tied_with_next": true }, { "rank": 21, "model": "grok-4.7", "rating": 1424.72, "evidence": -1.2058, "tied_with_next": true }, { "rank": 22, "model": "mistral-large-2512", "rating": 1419.64, "evidence": -1.3967 }, { "rank": 23, "model": "claude-haiku-4-5", "score": 13.2, "se": 1.07, "evidence": -2.0879 }, { "rank": 24, "model": "mistral-small-2603", "rating": 1364.93, "evidence": -2.3115 }, { "rank": 25, "model": "gpt-5.4-nano", "score": 11.7, "se": 1.02, "evidence": -2.4702 }, { "rank": 26, "model": "gpt-5-mini", "score": 21.6, "se": 1.3, "evidence": -2.4777 }, { "rank": 27, "model": "gpt-oss-120b", "rating": 1326.76, "evidence": -2.6343 }, { "rank": 28, "model": "gpt-4.1-mini", "score": 12.7, "se": 1.05, "evidence": -2.674 }, { "rank": 29, "model": "llama-3.3-70b-instruct-turbo", "rating": 1316.53, "evidence": -2.8168 }, { "rank": 30, "model": "gpt-4o-mini", "score": 8.3, "se": 0.87, "evidence": -3.7948 } ] }, { "task": "analysis", "band": "medium", "entries": [ { "rank": 1, "model": "claude-opus-5", "score": 80.6, "se": 2.8, "evidence": 2.5747, "tied_with_next": true }, { "rank": 2, "model": "gemini-3.1-pro-preview", "score": 79.6, "se": 2.85, "evidence": 2.4913, "tied_with_next": true }, { "rank": 3, "model": "gemini-3.5-flash", "score": 76.7, "se": 2.99, "evidence": 2.2267 }, { "rank": 4, "model": "gpt-5.5", "score": 69, "se": 3.27, "evidence": 1.1754, "tied_with_next": true }, { "rank": 5, "model": "claude-opus-4-8", "score": 64.8, "se": 3.38, "evidence": 0.6559, "tied_with_next": true }, { "rank": 6, "model": "deepseek-flash", "score": 61.1, "se": 3.45, "evidence": 0.3062, "tied_with_next": true }, { "rank": 7, "model": "deepseek-flash@together", "score": 61.1, "se": 3.45, "evidence": 0.3062, "tied_with_next": true }, { "rank": 8, "model": "claude-sonnet-5", "score": 60.6, "se": 3.46, "evidence": 0.2635 }, { "rank": 9, "model": "gpt-4.1", "score": 27, "se": 3.14, "evidence": -1.4277 }, { "rank": 10, "model": "mistral-large-2512", "score": 22.5, "se": 2.95, "evidence": -1.7759, "tied_with_next": true }, { "rank": 11, "model": "gpt-oss-120b", "score": 22.1, "se": 2.93, "evidence": -1.808, "tied_with_next": true }, { "rank": 12, "model": "llama-3.3-70b-instruct-turbo", "score": 19.9, "se": 2.82, "evidence": -2.0236 }, { "rank": 13, "model": "gpt-4o-mini", "score": 10.7, "se": 2.19, "evidence": -2.9647 } ] }, { "task": "analysis", "band": "hard", "entries": [ { "rank": 1, "model": "claude-opus-5-5", "score": 91.37, "se": 1.36, "evidence": 3.5239, "tied_with_next": true }, { "rank": 2, "model": "gpt-6-astra", "score": 94.21, "se": 1.15, "evidence": 3.4644, "tied_with_next": true }, { "rank": 3, "model": "gemini-3.8-flash", "score": 92.34, "se": 1.3, "evidence": 3.3752, "tied_with_next": true }, { "rank": 4, "model": "claude-opus-5", "score": 92.22, "se": 1.35, "evidence": 3.3463, "tied_with_next": true }, { "rank": 5, "model": "claude-fable-5-1", "score": 91.69, "se": 1.94, "evidence": 2.9054 }, { "rank": 6, "model": "gemini-3.1-pro-preview", "score": 89.46, "se": 1.5, "evidence": 2.6119 }, { "rank": 7, "model": "claude-opus-4-8", "score": 86.93, "se": 1.73, "evidence": 2.3282 }, { "rank": 8, "model": "gemini-3.5-flash", "score": 85.78, "se": 1.73, "evidence": 1.6176, "tied_with_next": true }, { "rank": 9, "model": "gpt-5.5", "score": 82.37, "se": 1.96, "evidence": 1.4793, "tied_with_next": true }, { "rank": 10, "model": "claude-sonnet-5", "score": 84.49, "se": 1.8, "evidence": 1.4387, "tied_with_next": true }, { "rank": 11, "model": "claude-sonnet-4-6", "score": 81.78, "se": 1.93, "evidence": 1.3207 }, { "rank": 12, "model": "gpt-5.4", "score": 80.26, "se": 2.05, "evidence": 0.8132 }, { "rank": 13, "model": "qwen3.5-9b", "score": 78.98, "se": 2.37, "evidence": 0.3085 }, { "rank": 14, "model": "deepseek-flash", "score": 79, "se": 2.02, "evidence": 0.265, "tied_with_next": true }, { "rank": 15, "model": "deepseek-flash@together", "score": 79, "se": 2.02, "evidence": 0.265 }, { "rank": 16, "model": "deepseek-v4-pro", "score": 77.96, "se": 2.06, "evidence": 0.0675, "tied_with_next": true }, { "rank": 17, "model": "deepseek-v4-pro@together", "score": 77.96, "se": 2.06, "evidence": 0.0675 }, { "rank": 18, "model": "grok-4.7", "score": 82.65, "se": 2.66, "evidence": 0.0017 }, { "rank": 19, "model": "gemini-3.5-flash-lite", "score": 68.73, "se": 2.28, "evidence": -0.8031 }, { "rank": 20, "model": "gpt-5-mini", "score": 71.72, "se": 3.21, "evidence": -1.3835, "tied_with_next": true }, { "rank": 21, "model": "gpt-oss-120b", "score": 75.76, "se": 2.74, "evidence": -1.5761, "tied_with_next": true }, { "rank": 22, "model": "gpt-4.1", "score": 66.92, "se": 2.82, "evidence": -1.6638, "tied_with_next": true }, { "rank": 23, "model": "gpt-5.4-mini", "score": 67.1, "se": 2.4, "evidence": -1.7314 }, { "rank": 24, "model": "mistral-medium-3-5", "score": 59.53, "se": 2.82, "evidence": -1.9953, "tied_with_next": true }, { "rank": 25, "model": "claude-haiku-4-5", "score": 60.48, "se": 2.82, "evidence": -2.005, "tied_with_next": true }, { "rank": 26, "model": "gpt-4.1-mini", "score": 65.85, "se": 2.71, "evidence": -2.0579, "tied_with_next": true }, { "rank": 27, "model": "gpt-5.4-nano", "score": 66.08, "se": 2.37, "evidence": -2.3969, "tied_with_next": true }, { "rank": 28, "model": "mistral-large-2512", "score": 51.33, "se": 2.69, "evidence": -2.7322 }, { "rank": 29, "model": "mistral-small-2603", "score": 47.47, "se": 2.76, "evidence": -3.3263 }, { "rank": 30, "model": "llama-3.3-70b-instruct-turbo", "score": 47.44, "se": 2.85, "evidence": -3.5597 }, { "rank": 31, "model": "gpt-4o-mini", "score": 37.72, "se": 2.41, "evidence": -3.9688 } ] }, { "task": "analysis", "band": "frontier", "entries": [ { "rank": 1, "model": "claude-opus-5", "score": 92.92, "se": 1.52, "evidence": 4.2112, "tied_with_next": true }, { "rank": 2, "model": "claude-opus-5-5", "score": 94.59, "se": 1.32, "evidence": 4.0254 }, { "rank": 3, "model": "claude-fable-5-1", "score": 73.18, "se": 1.55, "evidence": 3.2917, "tied_with_next": true }, { "rank": 4, "model": "gemini-3.1-pro-preview", "score": 73.84, "se": 1.46, "evidence": 3.2854 }, { "rank": 5, "model": "gpt-6-astra", "score": 66.79, "se": 1.73, "evidence": 2.9193, "tied_with_next": true }, { "rank": 6, "model": "claude-opus-4-8", "score": 75.11, "se": 2.36, "evidence": 2.8185 }, { "rank": 7, "model": "gemini-3.8-flash", "score": 44.52, "se": 1.96, "evidence": 2.3553 }, { "rank": 8, "model": "claude-sonnet-4-6", "score": 73.46, "se": 2.39, "evidence": 2.3277 }, { "rank": 9, "model": "gpt-5.5", "score": 54.75, "se": 2.4, "evidence": 2.0828, "tied_with_next": true }, { "rank": 10, "model": "claude-sonnet-5", "rating": 1511.75, "evidence": 1.8388 }, { "rank": 11, "model": "gpt-5.4", "score": 45.04, "se": 1.77, "evidence": 1.1894 }, { "rank": 12, "model": "gemini-3.5-flash", "score": 28.86, "se": 1.8, "evidence": 0.9036 }, { "rank": 13, "model": "grok-4.7", "rating": 1458.63, "evidence": -0.4622, "tied_with_next": true }, { "rank": 14, "model": "gemini-3.5-flash-lite", "score": 9.26, "se": 1.09, "evidence": -0.7295, "tied_with_next": true }, { "rank": 15, "model": "deepseek-v4-pro", "score": 6.92, "se": 0.94, "evidence": -0.8202, "tied_with_next": true }, { "rank": 16, "model": "deepseek-v4-pro@together", "score": 6.92, "se": 0.94, "evidence": -0.8202, "tied_with_next": true }, { "rank": 17, "model": "gpt-5.4-mini", "score": 7.06, "se": 0.97, "evidence": -0.8215 }, { "rank": 18, "model": "deepseek-flash", "score": 6.96, "se": 1.04, "evidence": -1.2837, "tied_with_next": true }, { "rank": 19, "model": "deepseek-flash@together", "score": 6.96, "se": 1.04, "evidence": -1.2837, "tied_with_next": true }, { "rank": 20, "model": "claude-haiku-4-5", "score": 7.79, "se": 1.01, "evidence": -1.38, "tied_with_next": true }, { "rank": 21, "model": "gpt-5.4-nano", "score": 9.93, "se": 1.12, "evidence": -1.3853 }, { "rank": 22, "model": "mistral-large-2512", "rating": 1427.35, "evidence": -1.423, "tied_with_next": true }, { "rank": 23, "model": "mistral-medium-3-5", "score": 4.52, "se": 0.81, "evidence": -1.569, "tied_with_next": true }, { "rank": 24, "model": "gpt-5-mini", "score": 15.75, "se": 0.94, "evidence": -1.7641 }, { "rank": 25, "model": "gpt-oss-120b", "rating": 1356.91, "evidence": -2.4067 }, { "rank": 26, "model": "mistral-small-2603", "rating": 1334.62, "evidence": -2.542 }, { "rank": 27, "model": "llama-3.3-70b-instruct-turbo", "rating": 1297.53, "evidence": -2.8421 }, { "rank": 28, "model": "gpt-4.1", "score": 3.77, "se": 0.52, "evidence": -3.1039 }, { "rank": 29, "model": "gpt-4.1-mini", "score": 1.75, "se": 0.51, "evidence": -3.2756 }, { "rank": 30, "model": "gpt-4o-mini", "score": 0, "se": 0.42, "evidence": -3.3363 } ] }, { "task": "math", "band": "medium", "entries": [ { "rank": 1, "model": "claude-opus-5", "rating": 1522.49, "evidence": 2.7023, "tied_with_next": true }, { "rank": 2, "model": "gemini-3.8-flash", "rating": 1516.35, "evidence": 2.5376, "tied_with_next": true }, { "rank": 3, "model": "claude-fable-5-1", "rating": 1517.46, "evidence": 2.4924, "tied_with_next": true }, { "rank": 4, "model": "claude-opus-5-5", "rating": 1510.21, "evidence": 2.1916, "tied_with_next": true }, { "rank": 5, "model": "gpt-5.5", "rating": 1499.39, "evidence": 2.1503, "tied_with_next": true }, { "rank": 6, "model": "gemini-3.5-flash", "rating": 1496.34, "evidence": 2.0288, "tied_with_next": true }, { "rank": 7, "model": "gpt-6-astra", "rating": 1490, "evidence": 1.7876, "tied_with_next": true }, { "rank": 8, "model": "gemini-3.1-pro-preview", "rating": 1487.87, "evidence": 1.7496 }, { "rank": 9, "model": "claude-opus-4-8", "rating": 1476.25, "evidence": 1.3243, "tied_with_next": true }, { "rank": 10, "model": "claude-sonnet-5", "rating": 1473.61, "evidence": 1.2434 }, { "rank": 11, "model": "claude-sonnet-4-6", "rating": 1463.24, "evidence": 0.8705, "tied_with_next": true }, { "rank": 12, "model": "gpt-5.4", "rating": 1459.91, "evidence": 0.7315, "tied_with_next": true }, { "rank": 13, "model": "grok-4.7", "rating": 1444.85, "evidence": 0.096, "tied_with_next": true }, { "rank": 14, "model": "deepseek-v4-pro", "rating": 1444.76, "evidence": 0.079, "tied_with_next": true }, { "rank": 15, "model": "deepseek-v4-pro@together", "rating": 1444.76, "evidence": 0.079, "tied_with_next": true }, { "rank": 16, "model": "gemini-3.5-flash-lite", "rating": 1441.85, "evidence": -0.0499, "tied_with_next": true }, { "rank": 17, "model": "gpt-5.4-mini", "rating": 1438.82, "evidence": -0.1686, "tied_with_next": true }, { "rank": 18, "model": "mistral-medium-3-5", "rating": 1428.35, "evidence": -0.5263, "tied_with_next": true }, { "rank": 19, "model": "deepseek-flash", "rating": 1424.75, "evidence": -0.6794, "tied_with_next": true }, { "rank": 20, "model": "deepseek-flash@together", "rating": 1424.75, "evidence": -0.6794, "tied_with_next": true }, { "rank": 21, "model": "gpt-5.4-nano", "rating": 1424.73, "evidence": -0.6906 }, { "rank": 22, "model": "gpt-5-mini", "rating": 1405.04, "evidence": -1.3611, "tied_with_next": true }, { "rank": 23, "model": "mistral-large-2512", "rating": 1401.5, "evidence": -1.524, "tied_with_next": true }, { "rank": 24, "model": "claude-haiku-4-5", "rating": 1400.23, "evidence": -1.5534 }, { "rank": 25, "model": "gpt-oss-120b", "rating": 1381.12, "evidence": -1.9778, "tied_with_next": true }, { "rank": 26, "model": "gpt-4.1", "rating": 1374.56, "evidence": -2.1042 }, { "rank": 27, "model": "gpt-4.1-mini", "rating": 1353.43, "evidence": -2.4232 }, { "rank": 28, "model": "mistral-small-2603", "rating": 1337.98, "evidence": -2.533 }, { "rank": 29, "model": "llama-3.3-70b-instruct-turbo", "rating": 1295.83, "evidence": -2.7931 }, { "rank": 30, "model": "gpt-4o-mini", "rating": 1275.35, "evidence": -3 } ] }, { "task": "math", "band": "hard", "entries": [ { "rank": 1, "model": "claude-opus-5-5", "score": 97.08, "se": 0.87, "evidence": 4.6644, "tied_with_next": true }, { "rank": 2, "model": "claude-fable-5-1", "score": 97.01, "se": 0.88, "evidence": 4.6233, "tied_with_next": true }, { "rank": 3, "model": "gpt-6-astra", "score": 96.81, "se": 0.9, "evidence": 4.5205, "tied_with_next": true }, { "rank": 4, "model": "gpt-5.5", "score": 95.86, "se": 1.02, "evidence": 3.9727, "tied_with_next": true }, { "rank": 5, "model": "claude-opus-5", "score": 95.73, "se": 1.04, "evidence": 3.8878, "tied_with_next": true }, { "rank": 6, "model": "grok-4.7", "score": 95.68, "se": 1.04, "evidence": 3.8662, "tied_with_next": true }, { "rank": 7, "model": "claude-opus-4-8", "score": 94.32, "se": 1.19, "evidence": 3.1144, "tied_with_next": true }, { "rank": 8, "model": "gpt-5.4", "score": 94.15, "se": 1.21, "evidence": 3.0219 }, { "rank": 9, "model": "gpt-5-mini", "score": 97.85, "se": 0.34, "evidence": 2.4559 }, { "rank": 10, "model": "claude-sonnet-5", "score": 92.94, "se": 1.32, "evidence": 2.42, "tied_with_next": true }, { "rank": 11, "model": "gemini-3.8-flash", "score": 91.56, "se": 1.43, "evidence": 1.8129, "tied_with_next": true }, { "rank": 12, "model": "gemini-3.1-pro-preview", "score": 91.04, "se": 1.47, "evidence": 1.6159, "tied_with_next": true }, { "rank": 13, "model": "gpt-5.4-nano", "score": 90.98, "se": 1.47, "evidence": 1.5964, "tied_with_next": true }, { "rank": 14, "model": "deepseek-v4-pro", "score": 90.68, "se": 1.49, "evidence": 1.4874, "tied_with_next": true }, { "rank": 15, "model": "deepseek-v4-pro@together", "score": 90.68, "se": 1.49, "evidence": 1.4874 }, { "rank": 16, "model": "gemini-3.5-flash", "score": 88.24, "se": 1.65, "evidence": 0.7551, "tied_with_next": true }, { "rank": 17, "model": "claude-sonnet-4-6", "score": 86.99, "se": 1.73, "evidence": 0.4351, "tied_with_next": true }, { "rank": 18, "model": "gpt-4.1-mini", "score": 87.29, "se": 0.73, "evidence": -0.4171, "tied_with_next": true }, { "rank": 19, "model": "claude-haiku-4-5", "score": 86.91, "se": 0.77, "evidence": -0.509 }, { "rank": 20, "model": "deepseek-flash", "score": 79.65, "se": 2.07, "evidence": -0.9827, "tied_with_next": true }, { "rank": 21, "model": "deepseek-flash@together", "score": 79.65, "se": 2.07, "evidence": -0.9827, "tied_with_next": true }, { "rank": 22, "model": "gpt-5.4-mini", "score": 78.46, "se": 2.11, "evidence": -1.1412 }, { "rank": 23, "model": "gemini-3.5-flash-lite", "score": 73.74, "se": 2.26, "evidence": -1.6941 }, { "rank": 24, "model": "gpt-4.1", "score": 83.01, "se": 0.85, "evidence": -1.7079 }, { "rank": 25, "model": "mistral-medium-3-5", "score": 81.63, "se": 0.9, "evidence": -2.0461 }, { "rank": 26, "model": "gpt-4o-mini", "score": 52.63, "se": 1.13, "evidence": -7.9632 }, { "rank": 27, "model": "mistral-large-2512", "score": 50.28, "se": 1.15, "evidence": -8.5033 }, { "rank": 28, "model": "mistral-small-2603", "score": 46.77, "se": 1.2, "evidence": -9.2312 }, { "rank": 29, "model": "llama-3.3-70b-instruct-turbo", "score": 41.6, "se": 1.18, "evidence": -10.5588 } ] }, { "task": "math", "band": "frontier", "entries": [ { "rank": 1, "model": "gpt-6-astra", "score": 96.84, "se": 2.6, "evidence": 3.2765, "tied_with_next": true }, { "rank": 2, "model": "claude-opus-5-5", "score": 95.62, "se": 2.64, "evidence": 3.1546, "tied_with_next": true }, { "rank": 3, "model": "claude-fable-5-1", "score": 95.09, "se": 2.65, "evidence": 3.1024 }, { "rank": 4, "model": "claude-opus-5", "score": 92.77, "se": 1.56, "evidence": 2.7932 }, { "rank": 5, "model": "claude-opus-4-8", "score": 72.3, "se": 2.53, "evidence": 2.2264, "tied_with_next": true }, { "rank": 6, "model": "gemini-3.8-flash", "score": 83.66, "se": 1.45, "evidence": 2.1984 }, { "rank": 7, "model": "gemini-3.1-pro-preview", "score": 64.05, "se": 1.7, "evidence": 1.7647, "tied_with_next": true }, { "rank": 8, "model": "gemini-3.5-flash", "score": 65.25, "se": 2.19, "evidence": 1.5565, "tied_with_next": true }, { "rank": 9, "model": "gpt-oss-120b", "score": 88.89, "se": 4.46, "evidence": 1.4673 }, { "rank": 10, "model": "deepseek-flash", "score": 75.99, "se": 1.8, "evidence": 1.2914, "tied_with_next": true }, { "rank": 11, "model": "deepseek-flash@together", "score": 75.99, "se": 1.8, "evidence": 1.2914, "tied_with_next": true }, { "rank": 12, "model": "gpt-5.4", "score": 60.88, "se": 3.29, "evidence": 1.0671, "tied_with_next": true }, { "rank": 13, "model": "claude-sonnet-5", "score": 72.8, "se": 3.33, "evidence": 0.9613, "tied_with_next": true }, { "rank": 14, "model": "gpt-5.5", "score": 69.1, "se": 4.46, "evidence": 0.7754 }, { "rank": 15, "model": "claude-sonnet-4-6", "score": 51.76, "se": 3.69, "evidence": 0.487 }, { "rank": 16, "model": "deepseek-v4-pro", "score": 54.04, "se": 4.58, "evidence": -0.5047, "tied_with_next": true }, { "rank": 17, "model": "deepseek-v4-pro@together", "score": 54.04, "se": 4.58, "evidence": -0.5047, "tied_with_next": true }, { "rank": 18, "model": "gemini-3.5-flash-lite", "score": 44.63, "se": 4.22, "evidence": -0.6698 }, { "rank": 19, "model": "gpt-5-mini", "score": 34.09, "se": 2.88, "evidence": -0.9588 }, { "rank": 20, "model": "qwen3.5-9b", "score": 42.78, "se": 6.53, "evidence": -1.0437, "tied_with_next": true }, { "rank": 21, "model": "claude-haiku-4-5", "score": 19.99, "se": 3.17, "evidence": -1.6603 }, { "rank": 22, "model": "gpt-5.4-nano", "score": 25.68, "se": 2.9, "evidence": -1.6835, "tied_with_next": true }, { "rank": 23, "model": "gpt-5.4-mini", "score": 23.58, "se": 2.66, "evidence": -1.9583 }, { "rank": 24, "model": "gpt-4.1-mini", "score": 18.62, "se": 2.15, "evidence": -1.9984 }, { "rank": 25, "model": "mistral-medium-3-5", "score": 16.29, "se": 2.99, "evidence": -2.1592 }, { "rank": 26, "model": "gpt-4.1", "score": 16.6, "se": 2.21, "evidence": -2.2063 }, { "rank": 27, "model": "mistral-small-2603", "score": 5.28, "se": 2.59, "evidence": -2.6614, "tied_with_next": true }, { "rank": 28, "model": "llama-3.3-70b-instruct-turbo", "score": 5.14, "se": 2.36, "evidence": -2.6648 }, { "rank": 29, "model": "mistral-large-2512", "score": 1.14, "se": 0.5, "evidence": -3.2655 }, { "rank": 30, "model": "gpt-4o-mini", "score": 3.82, "se": 1.39, "evidence": -3.474 } ] }, { "task": "code_write", "band": "hard", "entries": [ { "rank": 1, "model": "claude-opus-5-5", "score": 79.46, "se": 3.42, "evidence": 4.3355 }, { "rank": 2, "model": "claude-fable-5-1", "score": 75.25, "se": 3.73, "evidence": 3.7782 }, { "rank": 3, "model": "claude-opus-5", "score": 73.33, "se": 3.33, "evidence": 3.1548, "tied_with_next": true }, { "rank": 4, "model": "gpt-6-astra", "score": 61.46, "se": 4.57, "evidence": 2.905 }, { "rank": 5, "model": "claude-sonnet-5", "score": 70.04, "se": 3.42, "evidence": 2.0575, "tied_with_next": true }, { "rank": 6, "model": "claude-opus-4-8", "score": 66.17, "se": 3.44, "evidence": 2.0552, "tied_with_next": true }, { "rank": 7, "model": "gemini-3.8-flash", "score": 63.36, "se": 3.59, "evidence": 1.5472, "tied_with_next": true }, { "rank": 8, "model": "gpt-5.5", "score": 48.71, "se": 3.05, "evidence": 1.421, "tied_with_next": true }, { "rank": 9, "model": "claude-sonnet-4-6", "score": 60.95, "se": 3.47, "evidence": 1.3337, "tied_with_next": true }, { "rank": 10, "model": "grok-4.7", "score": 65.58, "se": 3.52, "evidence": 1.1411, "tied_with_next": true }, { "rank": 11, "model": "gemini-3.5-flash", "score": 63.59, "se": 3.51, "evidence": 0.8132, "tied_with_next": true }, { "rank": 12, "model": "gpt-5.4", "score": 48.98, "se": 3.48, "evidence": 0.5974 }, { "rank": 13, "model": "gemini-3.1-pro-preview", "score": 43.16, "se": 2.8, "evidence": 0.5591 }, { "rank": 14, "model": "gemini-3.5-flash-lite", "score": 60.66, "se": 3.53, "evidence": 0.085, "tied_with_next": true }, { "rank": 15, "model": "deepseek-v4-pro", "score": 56.31, "se": 3.6, "evidence": -0.4157, "tied_with_next": true }, { "rank": 16, "model": "deepseek-v4-pro@together", "score": 56.31, "se": 3.6, "evidence": -0.4157, "tied_with_next": true }, { "rank": 17, "model": "deepseek-flash", "score": 53.43, "se": 3.57, "evidence": -0.5864, "tied_with_next": true }, { "rank": 18, "model": "deepseek-flash@together", "score": 53.43, "se": 3.57, "evidence": -0.5864, "tied_with_next": true }, { "rank": 19, "model": "gpt-5.4-mini", "score": 56.65, "se": 3.58, "evidence": -0.6957, "tied_with_next": true }, { "rank": 20, "model": "gpt-5.4-nano", "score": 58.81, "se": 3.61, "evidence": -1.1148 }, { "rank": 21, "model": "claude-haiku-4-5", "rating": 1480.83, "evidence": -1.5103 }, { "rank": 22, "model": "gpt-4.1", "rating": 1456.1, "evidence": -1.6855 }, { "rank": 23, "model": "mistral-medium-3-5", "rating": 1478.63, "evidence": -1.7226 }, { "rank": 24, "model": "gpt-4.1-mini", "rating": 1432.98, "evidence": -2.0583, "tied_with_next": true }, { "rank": 25, "model": "gpt-5-mini", "rating": 1431.19, "evidence": -2.085 }, { "rank": 26, "model": "mistral-large-2512", "rating": 1469.27, "evidence": -2.1545 }, { "rank": 27, "model": "mistral-small-2603", "rating": 1412.14, "evidence": -2.3739 }, { "rank": 28, "model": "gpt-oss-120b", "rating": 1389.67, "evidence": -2.5862 }, { "rank": 29, "model": "gpt-4o-mini", "rating": 1347.98, "evidence": -2.8802, "tied_with_next": true }, { "rank": 30, "model": "llama-3.3-70b-instruct-turbo", "rating": 1345.74, "evidence": -2.9129 } ] }, { "task": "code_fix", "band": "hard", "entries": [ { "rank": 1, "model": "claude-opus-5-5", "score": 71.72, "se": 5.31, "evidence": 2.9518, "tied_with_next": true }, { "rank": 2, "model": "claude-fable-5-1", "score": 66.06, "se": 5.58, "evidence": 2.3957, "tied_with_next": true }, { "rank": 3, "model": "claude-opus-5", "score": 65.2, "se": 5.61, "evidence": 2.2966 }, { "rank": 4, "model": "gpt-5.4", "score": 69.68, "se": 2.34, "evidence": 1.9421, "tied_with_next": true }, { "rank": 5, "model": "claude-sonnet-5", "score": 59.39, "se": 5.79, "evidence": 1.6167 }, { "rank": 6, "model": "gemini-3.5-flash", "score": 64.16, "se": 3.09, "evidence": 1.5525 }, { "rank": 7, "model": "gpt-6-astra", "score": 57.32, "se": 5.83, "evidence": 1.3729, "tied_with_next": true }, { "rank": 8, "model": "gpt-5.5", "score": 59.02, "se": 3.58, "evidence": 1.3669, "tied_with_next": true }, { "rank": 9, "model": "gemini-3.8-flash", "score": 54.24, "se": 5.87, "evidence": 1.0165, "tied_with_next": true }, { "rank": 10, "model": "grok-4.7", "score": 53.99, "se": 5.87, "evidence": 0.9881 }, { "rank": 11, "model": "deepseek-v4-pro", "score": 57.05, "se": 3.52, "evidence": 0.6462, "tied_with_next": true }, { "rank": 12, "model": "deepseek-v4-pro@together", "score": 57.05, "se": 3.52, "evidence": 0.6462, "tied_with_next": true }, { "rank": 13, "model": "claude-opus-4-8", "score": 50.51, "se": 5.89, "evidence": 0.5911, "tied_with_next": true }, { "rank": 14, "model": "claude-sonnet-4-6", "score": 59.89, "se": 2.15, "evidence": 0.5335, "tied_with_next": true }, { "rank": 15, "model": "gemini-3.1-pro-preview", "score": 49.78, "se": 4, "evidence": 0.2285, "tied_with_next": true }, { "rank": 16, "model": "gpt-5.4-nano", "score": 46.77, "se": 5.88, "evidence": 0.1732, "tied_with_next": true }, { "rank": 17, "model": "gpt-4.1", "score": 49.83, "se": 1.88, "evidence": 0.0331, "tied_with_next": true }, { "rank": 18, "model": "gemini-3.5-flash-lite", "score": 45.25, "se": 5.87, "evidence": 0.0083, "tied_with_next": true }, { "rank": 19, "model": "gpt-5-mini", "score": 47, "se": 1.67, "evidence": -0.0076, "tied_with_next": true }, { "rank": 20, "model": "gpt-5.4-mini", "score": 41.67, "se": 5.81, "evidence": -0.3802, "tied_with_next": true }, { "rank": 21, "model": "deepseek-flash", "score": 37.63, "se": 5.71, "evidence": -0.7906, "tied_with_next": true }, { "rank": 22, "model": "deepseek-flash@together", "score": 37.63, "se": 5.71, "evidence": -0.7906, "tied_with_next": true }, { "rank": 23, "model": "gpt-4.1-mini", "score": 32.4, "se": 3.12, "evidence": -1.4148 }, { "rank": 24, "model": "gpt-oss-120b", "score": 25.56, "se": 1.91, "evidence": -1.8361, "tied_with_next": true }, { "rank": 25, "model": "claude-haiku-4-5", "score": 13.9, "se": 2.7, "evidence": -2.7578 }, { "rank": 26, "model": "qwen3.5-9b", "score": 9.21, "se": 2.35, "evidence": -3.3725, "tied_with_next": true }, { "rank": 27, "model": "codestral-2508", "score": 11.1, "se": 2.09, "evidence": -3.8252 }, { "rank": 28, "model": "gpt-4o-mini", "score": 3.6, "se": 1.24, "evidence": -5.1843 } ] }, { "task": "query", "band": "medium", "entries": [ { "rank": 1, "model": "gpt-6-astra", "score": 58.37, "se": 6.84, "evidence": 0.9752, "tied_with_next": true }, { "rank": 2, "model": "claude-fable-5-1", "score": 56.15, "se": 6.88, "evidence": 0.7326, "tied_with_next": true }, { "rank": 3, "model": "gpt-5.5", "score": 55.9, "se": 6.89, "evidence": 0.7051, "tied_with_next": true }, { "rank": 4, "model": "gemini-3.1-pro-preview", "score": 52.35, "se": 6.93, "evidence": 0.321, "tied_with_next": true }, { "rank": 5, "model": "claude-opus-5-5", "score": 52.33, "se": 6.93, "evidence": 0.3189, "tied_with_next": true }, { "rank": 6, "model": "claude-opus-5", "score": 51.96, "se": 6.93, "evidence": 0.2791, "tied_with_next": true }, { "rank": 7, "model": "gpt-5.4", "score": 51.75, "se": 6.93, "evidence": 0.2565, "tied_with_next": true }, { "rank": 8, "model": "claude-opus-4-8", "score": 51.08, "se": 6.93, "evidence": 0.1844, "tied_with_next": true }, { "rank": 9, "model": "gpt-5.4-nano", "score": 50.59, "se": 6.93, "evidence": 0.1317, "tied_with_next": true }, { "rank": 10, "model": "gemini-3.5-flash", "score": 50.5, "se": 6.93, "evidence": 0.122, "tied_with_next": true }, { "rank": 11, "model": "gpt-5.4-mini", "score": 49.39, "se": 6.93, "evidence": 0.0025, "tied_with_next": true }, { "rank": 12, "model": "deepseek-v4-pro", "score": 48.19, "se": 6.93, "evidence": -0.1266, "tied_with_next": true }, { "rank": 13, "model": "deepseek-v4-pro@together", "score": 48.19, "se": 6.93, "evidence": -0.1266, "tied_with_next": true }, { "rank": 14, "model": "deepseek-flash", "score": 46.12, "se": 6.91, "evidence": -0.3498, "tied_with_next": true }, { "rank": 15, "model": "deepseek-flash@together", "score": 46.12, "se": 6.91, "evidence": -0.3498, "tied_with_next": true }, { "rank": 16, "model": "gemini-3.8-flash", "score": 45.33, "se": 6.9, "evidence": -0.4353, "tied_with_next": true }, { "rank": 17, "model": "claude-sonnet-4-6", "score": 45.08, "se": 6.9, "evidence": -0.4622, "tied_with_next": true }, { "rank": 18, "model": "grok-4.7", "score": 44.92, "se": 6.9, "evidence": -0.4795, "tied_with_next": true }, { "rank": 19, "model": "claude-sonnet-5", "score": 42.44, "se": 6.85, "evidence": -0.7496, "tied_with_next": true }, { "rank": 20, "model": "gemini-3.5-flash-lite", "score": 40.62, "se": 6.81, "evidence": -0.9494 } ] }, { "task": "tool_use", "band": "hard", "entries": [ { "rank": 1, "model": "gpt-5.4", "score": 81.8, "se": 2, "evidence": 3 }, { "rank": 2, "model": "gpt-5.5", "score": 66.07, "se": 2.49, "evidence": 1.9105 }, { "rank": 3, "model": "gemini-3.1-pro-preview", "score": 59.36, "se": 4.2, "evidence": 1.2581 }, { "rank": 4, "model": "claude-sonnet-4-6", "score": 62.75, "se": 1.82, "evidence": 0.6885 }, { "rank": 5, "model": "mistral-medium-3-5", "rating": -0.12, "evidence": 0 }, { "rank": 6, "model": "gpt-5-mini", "score": 22.25, "se": 2.62, "evidence": -0.6393 }, { "rank": 7, "model": "gpt-oss-120b", "score": 14.2, "se": 2.3, "evidence": -1.815, "tied_with_next": true }, { "rank": 8, "model": "claude-haiku-4-5", "score": 13.9, "se": 2.7, "evidence": -1.856 }, { "rank": 9, "model": "qwen3.5-9b", "score": 9.21, "se": 2.35, "evidence": -2.5467 } ] }, { "task": "vision", "band": "medium", "entries": [ { "rank": 1, "model": "claude-opus-5", "rating": 1320.86, "evidence": 2.729, "tied_with_next": true }, { "rank": 2, "model": "claude-fable-5-1", "rating": 1318.93, "evidence": 2.6437, "tied_with_next": true }, { "rank": 3, "model": "gemini-3.8-flash", "rating": 1312.02, "evidence": 2.3731, "tied_with_next": true }, { "rank": 4, "model": "gemini-3.5-flash", "rating": 1310.49, "evidence": 2.3606 }, { "rank": 5, "model": "gpt-5.5", "rating": 1297.26, "evidence": 1.5571, "tied_with_next": true }, { "rank": 6, "model": "gemini-3.1-pro-preview", "rating": 1296.21, "evidence": 1.4895 }, { "rank": 7, "model": "gpt-5.4", "rating": 1291.91, "evidence": 1.1757, "tied_with_next": true }, { "rank": 8, "model": "claude-opus-4-8", "rating": 1287.93, "evidence": 0.8991 }, { "rank": 9, "model": "claude-sonnet-4-6", "rating": 1282.75, "evidence": 0.5404, "tied_with_next": true }, { "rank": 10, "model": "gpt-6-astra", "rating": 1280.05, "evidence": 0.4636, "tied_with_next": true }, { "rank": 11, "model": "claude-sonnet-5", "rating": 1275.4, "evidence": 0.1449 }, { "rank": 12, "model": "gemini-3.5-flash-lite", "rating": 1266.09, "evidence": -0.1767 }, { "rank": 13, "model": "gpt-5.4-mini", "rating": 1244.85, "evidence": -0.6 }, { "rank": 14, "model": "mistral-medium-3-5", "rating": 1222.37, "evidence": -1.0911, "tied_with_next": true }, { "rank": 15, "model": "mistral-large-2512", "rating": 1221.46, "evidence": -1.1038 }, { "rank": 16, "model": "gpt-4.1", "rating": 1210.31, "evidence": -1.4754 }, { "rank": 17, "model": "gpt-5-mini", "rating": 1201.96, "evidence": -1.8223 }, { "rank": 18, "model": "gpt-5.4-nano", "rating": 1196.06, "evidence": -2.0098 }, { "rank": 19, "model": "gpt-4.1-mini", "rating": 1181.29, "evidence": -2.3977 }, { "rank": 20, "model": "mistral-small-2603", "rating": 1141.55, "evidence": -2.7 }, { "rank": 21, "model": "gpt-4o-mini", "rating": 1065.48, "evidence": -3 } ] }, { "task": "document_qa", "band": "medium", "entries": [ { "rank": 1, "model": "claude-opus-5-5", "rating": 1527.81, "evidence": 2.9319 }, { "rank": 2, "model": "claude-fable-5-1", "rating": 1511.39, "evidence": 2.6357, "tied_with_next": true }, { "rank": 3, "model": "gemini-3.8-flash", "rating": 1507.93, "evidence": 2.5444, "tied_with_next": true }, { "rank": 4, "model": "claude-opus-5", "rating": 1504.8, "evidence": 2.4232 }, { "rank": 5, "model": "claude-opus-4-8", "rating": 1499.75, "evidence": 2.1502, "tied_with_next": true }, { "rank": 6, "model": "gemini-3.1-pro-preview", "rating": 1498.89, "evidence": 2.0962 }, { "rank": 7, "model": "claude-sonnet-4-6", "rating": 1492.39, "evidence": 1.7065 }, { "rank": 8, "model": "gpt-5.5", "rating": 1486.64, "evidence": 1.3706, "tied_with_next": true }, { "rank": 9, "model": "gpt-6-astra", "rating": 1485.58, "evidence": 1.2869, "tied_with_next": true }, { "rank": 10, "model": "claude-sonnet-5", "rating": 1481.83, "evidence": 1.0956, "tied_with_next": true }, { "rank": 11, "model": "gemini-3.5-flash", "rating": 1481.48, "evidence": 1.0726 }, { "rank": 12, "model": "gpt-5.4", "rating": 1477.36, "evidence": 0.8229 }, { "rank": 13, "model": "deepseek-v4-pro", "rating": 1471.65, "evidence": 0.4986, "tied_with_next": true }, { "rank": 14, "model": "deepseek-v4-pro@together", "rating": 1471.65, "evidence": 0.4986 }, { "rank": 15, "model": "gemini-3.5-flash-lite", "rating": 1461.62, "evidence": 0.0922 }, { "rank": 16, "model": "grok-4.7", "rating": 1450.54, "evidence": -0.3571, "tied_with_next": true }, { "rank": 17, "model": "deepseek-flash", "rating": 1447.41, "evidence": -0.4232, "tied_with_next": true }, { "rank": 18, "model": "deepseek-flash@together", "rating": 1447.41, "evidence": -0.4232, "tied_with_next": true }, { "rank": 19, "model": "gpt-5.4-mini", "rating": 1446.73, "evidence": -0.4447 }, { "rank": 20, "model": "claude-haiku-4-5", "rating": 1436.32, "evidence": -0.9762 }, { "rank": 21, "model": "mistral-medium-3-5", "rating": 1429.62, "evidence": -1.1737, "tied_with_next": true }, { "rank": 22, "model": "gpt-4.1", "rating": 1426.9, "evidence": -1.2569 }, { "rank": 23, "model": "mistral-large-2512", "rating": 1416.95, "evidence": -1.516 }, { "rank": 24, "model": "gpt-5.4-nano", "rating": 1402.6, "evidence": -1.7689 }, { "rank": 25, "model": "gpt-4.1-mini", "rating": 1389.88, "evidence": -1.9552 }, { "rank": 26, "model": "gpt-5-mini", "rating": 1370.25, "evidence": -2.2161 }, { "rank": 27, "model": "mistral-small-2603", "rating": 1358.76, "evidence": -2.3356 }, { "rank": 28, "model": "gpt-4o-mini", "rating": 1326.18, "evidence": -2.6932, "tied_with_next": true }, { "rank": 29, "model": "gpt-oss-120b", "rating": 1324.42, "evidence": -2.7316 }, { "rank": 30, "model": "llama-3.3-70b-instruct-turbo", "rating": 1313.38, "evidence": -2.9546 } ] }, { "task": "document_qa", "band": "hard", "entries": [ { "rank": 1, "model": "claude-opus-5", "rating": 1516.27, "evidence": 2.7432, "tied_with_next": true }, { "rank": 2, "model": "claude-fable-5-1", "rating": 1512.58, "evidence": 2.6568 }, { "rank": 3, "model": "gpt-5-mini", "score": 69.4, "se": 5, "evidence": 2.4445, "tied_with_next": true }, { "rank": 4, "model": "gpt-4.1", "score": 63.9, "se": 5, "evidence": 1.995 }, { "rank": 5, "model": "gpt-5.5", "rating": 1485.72, "evidence": 1.5087, "tied_with_next": true }, { "rank": 6, "model": "claude-sonnet-4-6", "rating": 1482.24, "evidence": 1.2688 }, { "rank": 7, "model": "gpt-5.4", "rating": 1471, "evidence": -0.1193, "tied_with_next": true }, { "rank": 8, "model": "gpt-6-astra", "rating": 1467.52, "evidence": -0.3496, "tied_with_next": true }, { "rank": 9, "model": "claude-sonnet-5", "rating": 1466.14, "evidence": -0.6239 }, { "rank": 10, "model": "gpt-4.1-mini", "score": 44.4, "se": 5, "evidence": -0.7535 }, { "rank": 11, "model": "claude-opus-4-8", "rating": 1464.07, "evidence": -0.8144, "tied_with_next": true }, { "rank": 12, "model": "gemini-3.5-flash", "rating": 1463.27, "evidence": -0.8703 }, { "rank": 13, "model": "gpt-oss-120b", "score": 36.1, "se": 5, "evidence": -1.6945, "tied_with_next": true }, { "rank": 14, "model": "llama-3.3-70b-instruct-turbo", "score": 33.3, "se": 5, "evidence": -1.9914 }, { "rank": 15, "model": "gemini-3.1-pro-preview", "rating": 1443.92, "evidence": -2.4 }, { "rank": 16, "model": "claude-haiku-4-5", "rating": 1419.97, "evidence": -3 } ] } ], "models_without_results": null, "caveats_en": [ "These numbers come from the sources named on each row, not from Zabio's own tests. Zabio's tests are shown separately and never combined with them into one figure.", "Sources test models at their own settings. A settings mismatch against how Zabio calls the model is marked on the row and lowers its weight.", "Scores are compared only inside one benchmark. Task scores combine benchmarks by evidence in standard errors, so a task score is a summary, not a benchmark result.", "A model missing from every source is listed as such rather than left out." ], "caveats_es": [ "Estas cifras vienen de las fuentes que indica cada fila, no de las pruebas propias de Zabio. Las pruebas de Zabio se muestran aparte y nunca se combinan con ellas en una sola cifra.", "Las fuentes prueban los modelos con sus propios ajustes. Una diferencia de ajustes respecto a cómo Zabio llama al modelo se marca en la fila y reduce su peso.", "Las puntuaciones solo se comparan dentro de un mismo benchmark. Las puntuaciones por tarea combinan benchmarks por evidencia en errores estándar, así que son un resumen, no un resultado de benchmark.", "Un modelo ausente de todas las fuentes aparece como tal en lugar de omitirse." ] } }