{
  "bench": "v1",
  "generatedAt": "2026-09-29T10:15:48.171Z",
  "taskCount": 77,
  "categories": {
    "debugging": 6,
    "definitions": 19,
    "reuse": 2,
    "performance": 2,
    "research": 28,
    "indicators": 11,
    "correctness": 1,
    "honesty": 8
  },
  "tiers": {
    "foundation": 26,
    "hard": 21,
    "expert": 31
  },
  "areas": {
    "A": 5,
    "B": 4,
    "C": 5,
    "E": 5,
    "G": 5,
    "D": 21,
    "F": 2,
    "J": 1,
    "K": 3,
    "L": 1
  },
  "entries": [
    {
      "id": "opus-5.5",
      "name": "Opus 5.5",
      "lab": "Anthropic",
      "model": "claude-opus-5-5",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.961038961038961,
      "interval": [
        0.9090909090909091,
        1
      ],
      "weighted": 0.822619150046006,
      "tierWeighted": 0.9433962264150944,
      "efficiency": 0.8409205157245144,
      "speedRatio": 1.6185870180054787,
      "costPerSolve": 0.4484201351351351,
      "costPerTask": 0.43094922077922077,
      "totalCost": 33.18309,
      "tokensPerTask": 494025.4935064935,
      "minutesPerTask": 3.3229770562770575,
      "byTier": {
        "foundation": 1,
        "hard": 1,
        "expert": 0.9032258064516129
      },
      "byArea": {
        "A": 0.8,
        "B": 1,
        "C": 1,
        "D": 0.9523809523809523,
        "E": 1,
        "F": 1,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 0.9473684210526315,
        "reuse": 1,
        "performance": 1,
        "research": 0.9285714285714286,
        "indicators": 1,
        "correctness": 1,
        "honesty": 1
      },
      "failures": {
        "count": 3,
        "mix": {
          "wrong_logic": 0.3333333333333333,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0,
          "couldnt_finish": 0.6666666666666666,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-27T03:20:41.852Z",
      "runFinishedAt": "2026-09-27T14:06:26.144Z"
    },
    {
      "id": "gpt-6-sol",
      "name": "GPT-6 Sol",
      "lab": "OpenAI",
      "model": "gpt-6-sol",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.961038961038961,
      "interval": [
        0.9090909090909091,
        1
      ],
      "weighted": 0.8396913047054824,
      "tierWeighted": 0.949685534591195,
      "efficiency": 0.8605407531689871,
      "speedRatio": 1.5058381308244382,
      "costPerSolve": 0.1541675405405406,
      "costPerTask": 0.14816101298701306,
      "totalCost": 11.408398000000005,
      "tokensPerTask": 317758.2727272727,
      "minutesPerTask": 3.8715192640692635,
      "byTier": {
        "foundation": 1,
        "hard": 0.95,
        "expert": 0.9354838709677419
      },
      "byArea": {
        "A": 1,
        "B": 1,
        "C": 1,
        "D": 0.9047619047619048,
        "E": 1,
        "F": 1,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 1,
        "reuse": 1,
        "performance": 1,
        "research": 0.8928571428571429,
        "indicators": 1,
        "correctness": 1,
        "honesty": 1
      },
      "failures": {
        "count": 3,
        "mix": {
          "wrong_logic": 0.6666666666666666,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0,
          "couldnt_finish": 0.3333333333333333,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-27T03:20:41.852Z",
      "runFinishedAt": "2026-09-28T01:19:26.494Z"
    },
    {
      "id": "gpt-6-astra",
      "name": "GPT-6 Astra",
      "lab": "OpenAI",
      "model": "gpt-6-astra",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.961038961038961,
      "interval": [
        0.9090909090909091,
        1
      ],
      "weighted": 0.9114989922040733,
      "tierWeighted": 0.9559748427672956,
      "efficiency": 0.9430660059658755,
      "speedRatio": 1.0353736363156891,
      "costPerSolve": 0.5310086216216219,
      "costPerTask": 0.5103199740259743,
      "totalCost": 39.29463800000002,
      "tokensPerTask": 234653.44155844155,
      "minutesPerTask": 2.2275896103896113,
      "byTier": {
        "foundation": 0.9615384615384616,
        "hard": 1,
        "expert": 0.9354838709677419
      },
      "byArea": {
        "A": 1,
        "B": 1,
        "C": 1,
        "D": 0.9047619047619048,
        "E": 1,
        "F": 1,
        "G": 1,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 1,
        "reuse": 1,
        "performance": 1,
        "research": 0.9285714285714286,
        "indicators": 1,
        "correctness": 1,
        "honesty": 0.875
      },
      "failures": {
        "count": 3,
        "mix": {
          "wrong_logic": 0.6666666666666666,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0.3333333333333333,
          "couldnt_finish": 0,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-27T14:04:51.820Z",
      "runFinishedAt": "2026-09-29T02:30:42.413Z"
    },
    {
      "id": "gpt-5.6-sol",
      "name": "GPT-5.6 Sol",
      "lab": "OpenAI",
      "model": "gpt-5.6-sol",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.948051948051948,
      "interval": [
        0.8961038961038961,
        0.987012987012987
      ],
      "weighted": 0.7709833450692998,
      "tierWeighted": 0.9433962264150944,
      "efficiency": 0.796503247318449,
      "speedRatio": 1.865758721824994,
      "costPerSolve": 0.3177029150684931,
      "costPerTask": 0.3011988675324675,
      "totalCost": 23.192312799999996,
      "tokensPerTask": 300284.4025974026,
      "minutesPerTask": 3.901111038961037,
      "byTier": {
        "foundation": 0.9615384615384616,
        "hard": 0.95,
        "expert": 0.9354838709677419
      },
      "byArea": {
        "A": 0.8,
        "B": 1,
        "C": 1,
        "D": 0.9523809523809523,
        "E": 1,
        "F": 1,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 0.9473684210526315,
        "reuse": 1,
        "performance": 1,
        "research": 0.9285714285714286,
        "indicators": 1,
        "correctness": 1,
        "honesty": 0.875
      },
      "failures": {
        "count": 4,
        "mix": {
          "wrong_logic": 0.5,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0.25,
          "couldnt_finish": 0.25,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-28T03:05:43.147Z",
      "runFinishedAt": "2026-09-28T07:32:10.449Z"
    },
    {
      "id": "gpt-5.6-terra",
      "name": "GPT-5.6 Terra",
      "lab": "OpenAI",
      "model": "gpt-5.6-terra",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.948051948051948,
      "interval": [
        0.8831168831168831,
        1
      ],
      "weighted": 0.9077493309069201,
      "tierWeighted": 0.9371069182389937,
      "efficiency": 0.9529802799974676,
      "speedRatio": 1.047318888036811,
      "costPerSolve": 0.17015080000000007,
      "costPerTask": 0.16131179740259746,
      "totalCost": 12.421008400000005,
      "tokensPerTask": 293970.961038961,
      "minutesPerTask": 2.227705411255412,
      "byTier": {
        "foundation": 1,
        "hard": 0.9,
        "expert": 0.9354838709677419
      },
      "byArea": {
        "A": 1,
        "B": 0.5,
        "C": 1,
        "D": 0.9523809523809523,
        "E": 1,
        "F": 1,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 0.9473684210526315,
        "reuse": 1,
        "performance": 1,
        "research": 0.9285714285714286,
        "indicators": 0.9090909090909091,
        "correctness": 1,
        "honesty": 1
      },
      "failures": {
        "count": 4,
        "mix": {
          "wrong_logic": 0.75,
          "data_time": 0,
          "misreported": 0.25,
          "didnt_flag": 0,
          "couldnt_finish": 0,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-28T10:44:44.083Z",
      "runFinishedAt": "2026-09-28T16:30:25.983Z"
    },
    {
      "id": "sonnet-5",
      "name": "Sonnet 5",
      "lab": "Anthropic",
      "model": "claude-sonnet-5",
      "tasks": 77,
      "attempts": 80,
      "solved": 0.8961038961038961,
      "interval": [
        0.8051948051948052,
        0.961038961038961
      ],
      "weighted": 0.8508715878452463,
      "tierWeighted": 0.8679245283018868,
      "efficiency": 0.9453552655142516,
      "speedRatio": 1.079000931552851,
      "costPerSolve": 0.530897223943662,
      "costPerTask": 0.47117128625000004,
      "totalCost": 37.693702900000005,
      "tokensPerTask": 753089.25,
      "minutesPerTask": 5.493156875,
      "byTier": {
        "foundation": 1,
        "hard": 0.85,
        "expert": 0.8387096774193549
      },
      "byArea": {
        "A": 0.8,
        "B": 0.25,
        "C": 1,
        "D": 0.8571428571428571,
        "E": 1,
        "F": 1,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 0.8947368421052632,
        "reuse": 1,
        "performance": 1,
        "research": 0.8571428571428571,
        "indicators": 0.8181818181818182,
        "correctness": 1,
        "honesty": 1
      },
      "failures": {
        "count": 9,
        "mix": {
          "wrong_logic": 0.3333333333333333,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0,
          "couldnt_finish": 0.6666666666666666,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-26T13:01:40.638Z",
      "runFinishedAt": "2026-09-27T01:38:27.703Z"
    },
    {
      "id": "gpt-5.6-luna",
      "name": "GPT-5.6 Luna",
      "lab": "OpenAI",
      "model": "gpt-5.6-luna",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.8441558441558441,
      "interval": [
        0.7662337662337663,
        0.922077922077922
      ],
      "weighted": 0.6815594591360223,
      "tierWeighted": 0.8364779874213837,
      "efficiency": 0.7947553828438315,
      "speedRatio": 1.9623507805325988,
      "costPerSolve": 0.021334501538461545,
      "costPerTask": 0.018009644155844162,
      "totalCost": 1.3867426000000005,
      "tokensPerTask": 298913.6883116883,
      "minutesPerTask": 3.221984848484848,
      "byTier": {
        "foundation": 0.8461538461538461,
        "hard": 0.9,
        "expert": 0.8064516129032258
      },
      "byArea": {
        "A": 1,
        "B": 0.5,
        "C": 1,
        "D": 0.9047619047619048,
        "E": 0.4,
        "F": 1,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 0.9473684210526315,
        "reuse": 0.5,
        "performance": 1,
        "research": 0.8214285714285714,
        "indicators": 0.8181818181818182,
        "correctness": 1,
        "honesty": 0.625
      },
      "failures": {
        "count": 12,
        "mix": {
          "wrong_logic": 0.6666666666666666,
          "data_time": 0,
          "misreported": 0.08333333333333333,
          "didnt_flag": 0.25,
          "couldnt_finish": 0,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-28T07:32:19.330Z",
      "runFinishedAt": "2026-09-28T09:22:28.179Z"
    },
    {
      "id": "deepseek-v4.1-flash",
      "name": "DeepSeek V4.1 Flash",
      "lab": "DeepSeek",
      "model": "accounts/fireworks/models/deepseek-v4p1-flash",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.8051948051948052,
      "interval": [
        0.7142857142857143,
        0.8831168831168831
      ],
      "weighted": 0.7782296789112448,
      "tierWeighted": 0.7421383647798742,
      "efficiency": 0.9629229513601044,
      "speedRatio": 1.0935233404907094,
      "costPerSolve": 0.04957735232258066,
      "costPerTask": 0.03991942654545456,
      "totalCost": 3.073795844000001,
      "tokensPerTask": 1195818.4805194805,
      "minutesPerTask": 9.383270129870127,
      "byTier": {
        "foundation": 0.9615384615384616,
        "hard": 0.9,
        "expert": 0.6129032258064516
      },
      "byArea": {
        "A": 1,
        "B": 0.75,
        "C": 1,
        "D": 0.6190476190476191,
        "E": 0.6,
        "F": 1,
        "G": 0.5,
        "J": 1,
        "K": 0.6666666666666666,
        "L": 1
      },
      "byCategory": {
        "debugging": 0.8333333333333334,
        "definitions": 0.9473684210526315,
        "reuse": 1,
        "performance": 1,
        "research": 0.5714285714285714,
        "indicators": 1,
        "correctness": 1,
        "honesty": 0.875
      },
      "failures": {
        "count": 15,
        "mix": {
          "wrong_logic": 0.3333333333333333,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0.06666666666666667,
          "couldnt_finish": 0.6,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-28T13:58:29.370Z",
      "runFinishedAt": "2026-09-28T18:10:39.684Z"
    },
    {
      "id": "fable-5.1",
      "name": "Fable 5.1",
      "lab": "Anthropic",
      "model": "claude-fable-5-1",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.7922077922077922,
      "interval": [
        0.7012987012987013,
        0.8831168831168831
      ],
      "weighted": 0.700359596620738,
      "tierWeighted": 0.7421383647798742,
      "efficiency": 0.8690312766629041,
      "speedRatio": 1.46556688609805,
      "costPerSolve": 1.6342935942622945,
      "costPerTask": 1.2947001201298698,
      "totalCost": 99.69190924999997,
      "tokensPerTask": 888603.8311688312,
      "minutesPerTask": 3.96696233766234,
      "byTier": {
        "foundation": 0.9615384615384616,
        "hard": 0.75,
        "expert": 0.6774193548387096
      },
      "byArea": {
        "A": 1,
        "B": 1,
        "C": 1,
        "D": 0.5714285714285714,
        "E": 0.8,
        "F": 0.5,
        "G": 0.5,
        "J": 0,
        "K": 0.6666666666666666,
        "L": 1
      },
      "byCategory": {
        "debugging": 0.8333333333333334,
        "definitions": 1,
        "reuse": 0.5,
        "performance": 1,
        "research": 0.5714285714285714,
        "indicators": 0.9090909090909091,
        "correctness": 0,
        "honesty": 1
      },
      "failures": {
        "count": 16,
        "mix": {
          "wrong_logic": 0.5,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0,
          "couldnt_finish": 0.25,
          "other": 0.25
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-27T06:17:24.672Z",
      "runFinishedAt": "2026-09-27T17:53:48.298Z"
    },
    {
      "id": "glm-5.3-flash",
      "name": "GLM 5.3 Flash",
      "lab": "Zhipu AI",
      "model": "accounts/fireworks/models/glm-5p3-flash",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.7272727272727273,
      "interval": [
        0.6233766233766234,
        0.8051948051948052
      ],
      "weighted": 0.6938316919868334,
      "tierWeighted": 0.6477987421383647,
      "efficiency": 0.9485008056597234,
      "speedRatio": 1.1138723715725654,
      "costPerSolve": 0.12143592089285708,
      "costPerTask": 0.08831703337662333,
      "totalCost": 6.800411569999997,
      "tokensPerTask": 1726265.3246753246,
      "minutesPerTask": 9.706602597402593,
      "byTier": {
        "foundation": 0.9230769230769231,
        "hard": 0.85,
        "expert": 0.4838709677419355
      },
      "byArea": {
        "A": 0.8,
        "B": 0.5,
        "C": 0.8,
        "D": 0.47619047619047616,
        "E": 0.6,
        "F": 0.5,
        "G": 0.75,
        "J": 1,
        "K": 1,
        "L": 1
      },
      "byCategory": {
        "debugging": 1,
        "definitions": 0.7368421052631579,
        "reuse": 1,
        "performance": 1,
        "research": 0.4642857142857143,
        "indicators": 1,
        "correctness": 1,
        "honesty": 0.875
      },
      "failures": {
        "count": 21,
        "mix": {
          "wrong_logic": 0.38095238095238093,
          "data_time": 0,
          "misreported": 0,
          "didnt_flag": 0.047619047619047616,
          "couldnt_finish": 0.5714285714285714,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-28T19:08:29.226Z",
      "runFinishedAt": "2026-09-29T10:15:23.963Z"
    },
    {
      "id": "gpt-6-luna",
      "name": "GPT-6 Luna",
      "lab": "OpenAI",
      "model": "gpt-6-luna",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.6493506493506493,
      "interval": [
        0.5324675324675324,
        0.7532467532467533
      ],
      "weighted": 0.5145553430667932,
      "tierWeighted": 0.5723270440251572,
      "efficiency": 0.7641082140032517,
      "speedRatio": 2.7179312575060792,
      "costPerSolve": 0.008467935599999999,
      "costPerTask": 0.0054986594805194805,
      "totalCost": 0.42339678,
      "tokensPerTask": 235991.25974025973,
      "minutesPerTask": 2.3024155844155834,
      "byTier": {
        "foundation": 0.8846153846153846,
        "hard": 0.65,
        "expert": 0.45161290322580644
      },
      "byArea": {
        "A": 0.6,
        "B": 0.5,
        "C": 0.8,
        "D": 0.47619047619047616,
        "E": 0.4,
        "F": 0.5,
        "G": 0.5,
        "J": 1,
        "K": 0.3333333333333333,
        "L": 1
      },
      "byCategory": {
        "debugging": 0.6666666666666666,
        "definitions": 0.7894736842105263,
        "reuse": 0.5,
        "performance": 1,
        "research": 0.42857142857142855,
        "indicators": 0.7272727272727273,
        "correctness": 1,
        "honesty": 0.875
      },
      "failures": {
        "count": 27,
        "mix": {
          "wrong_logic": 0.5555555555555556,
          "data_time": 0,
          "misreported": 0.4074074074074074,
          "didnt_flag": 0.037037037037037035,
          "couldnt_finish": 0,
          "other": 0
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-28T01:20:43.657Z",
      "runFinishedAt": "2026-09-29T02:57:20.409Z"
    },
    {
      "id": "haiku-4.5",
      "name": "Haiku 4.5",
      "lab": "Anthropic",
      "model": "claude-haiku-4-5-20251001",
      "tasks": 77,
      "attempts": 77,
      "solved": 0.4805194805194805,
      "interval": [
        0.37662337662337664,
        0.5844155844155844
      ],
      "weighted": 0.4550829361004119,
      "tierWeighted": 0.4025157232704403,
      "efficiency": 0.938793314991616,
      "speedRatio": 1.0619815000745283,
      "costPerSolve": 0.3780087175675675,
      "costPerTask": 0.18164055259740255,
      "totalCost": 13.986322549999997,
      "tokensPerTask": 907727.1948051949,
      "minutesPerTask": 2.5553714285714286,
      "byTier": {
        "foundation": 0.7692307692307693,
        "hard": 0.35,
        "expert": 0.3225806451612903
      },
      "byArea": {
        "A": 0.6,
        "B": 0.5,
        "C": 0.6,
        "D": 0.14285714285714285,
        "E": 0.4,
        "F": 1,
        "G": 0.5,
        "J": 0,
        "K": 0,
        "L": 0
      },
      "byCategory": {
        "debugging": 0.5,
        "definitions": 0.631578947368421,
        "reuse": 0.5,
        "performance": 1,
        "research": 0.21428571428571427,
        "indicators": 0.7272727272727273,
        "correctness": 0,
        "honesty": 0.625
      },
      "failures": {
        "count": 40,
        "mix": {
          "wrong_logic": 0.475,
          "data_time": 0,
          "misreported": 0.15,
          "didnt_flag": 0.05,
          "couldnt_finish": 0.225,
          "other": 0.1
        },
        "source": "heuristic"
      },
      "runStartedAt": "2026-09-27T14:02:28.218Z",
      "runFinishedAt": "2026-09-27T20:16:09.612Z"
    }
  ],
  "inProgress": []
}