{
 "suite": "SWE-bench Verified, the 10 tasks of OpenHuman's harness benchmark",
 "grader": "official SWE-bench evaluator",
 "runs": [
  {
   "harness": "Letronna",
   "label": "round one",
   "model": "DeepSeek V4.1 Flash",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 12,
     "prompt_tokens": 120760,
     "cached_tokens": 110720,
     "output_tokens": 3059,
     "cost_usd": 0.003674
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 9,
     "prompt_tokens": 98297,
     "cached_tokens": 91264,
     "output_tokens": 1546,
     "cost_usd": 0.002256
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 64282,
     "cached_tokens": 60672,
     "output_tokens": 1392,
     "cost_usd": 0.001559
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 9,
     "prompt_tokens": 101076,
     "cached_tokens": 92160,
     "output_tokens": 3312,
     "cost_usd": 0.003601
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 15,
     "prompt_tokens": 184869,
     "cached_tokens": 177024,
     "output_tokens": 5851,
     "cost_usd": 0.005218
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 78234,
     "cached_tokens": 57856,
     "output_tokens": 2048,
     "cost_usd": 0.004459
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 19,
     "prompt_tokens": 372907,
     "cached_tokens": 345984,
     "output_tokens": 9894,
     "cost_usd": 0.011013
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 51356,
     "cached_tokens": 48128,
     "output_tokens": 1087,
     "cost_usd": 0.001281
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 74,
     "prompt_tokens": 2486386,
     "cached_tokens": 2377600,
     "output_tokens": 34691,
     "cost_usd": 0.044265
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 13,
     "prompt_tokens": 132128,
     "cached_tokens": 124928,
     "output_tokens": 3079,
     "cost_usd": 0.003302
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "baseline",
   "model": "DeepSeek V4.1 Flash",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 38,
     "prompt_tokens": 558945,
     "cached_tokens": 488960,
     "output_tokens": 12422,
     "cost_usd": 0.019418
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 10,
     "prompt_tokens": 101137,
     "cached_tokens": 82560,
     "output_tokens": 2106,
     "cost_usd": 0.004298
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 59155,
     "cached_tokens": 51072,
     "output_tokens": 1053,
     "cost_usd": 0.001997
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 24,
     "prompt_tokens": 370212,
     "cached_tokens": 322944,
     "output_tokens": 9086,
     "cost_usd": 0.013511
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 33,
     "prompt_tokens": 746932,
     "cached_tokens": 644352,
     "output_tokens": 21436,
     "cost_usd": 0.030182
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 12,
     "prompt_tokens": 125095,
     "cached_tokens": 104576,
     "output_tokens": 2937,
     "cost_usd": 0.005154
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": false,
     "calls": 115,
     "prompt_tokens": 3512868,
     "cached_tokens": 3338752,
     "output_tokens": 43830,
     "cost_usd": 0.062432
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 53333,
     "cached_tokens": 46592,
     "output_tokens": 869,
     "cost_usd": 0.001672
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 60,
     "prompt_tokens": 1300012,
     "cached_tokens": 1169536,
     "output_tokens": 24952,
     "cost_usd": 0.038051
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 19,
     "prompt_tokens": 259402,
     "cached_tokens": 224896,
     "output_tokens": 6725,
     "cost_usd": 0.009886
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "round two (a later build)",
   "model": "DeepSeek V4.1 Flash",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 16,
     "prompt_tokens": 202024,
     "cached_tokens": 183296,
     "output_tokens": 4350,
     "cost_usd": 0.005969
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 11,
     "prompt_tokens": 102833,
     "cached_tokens": 96256,
     "output_tokens": 1755,
     "cost_usd": 0.002328
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 52925,
     "cached_tokens": 49408,
     "output_tokens": 1359,
     "cost_usd": 0.001491
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 18,
     "prompt_tokens": 332021,
     "cached_tokens": 309504,
     "output_tokens": 11941,
     "cost_usd": 0.011471
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 15,
     "prompt_tokens": 187501,
     "cached_tokens": 173184,
     "output_tokens": 5683,
     "cost_usd": 0.006077
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 75407,
     "cached_tokens": 69120,
     "output_tokens": 1661,
     "cost_usd": 0.002147
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 98,
     "prompt_tokens": 4934847,
     "cached_tokens": 4832896,
     "output_tokens": 57093,
     "cost_usd": 0.064047
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 50169,
     "cached_tokens": 47488,
     "output_tokens": 778,
     "cost_usd": 0.001011
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 29,
     "prompt_tokens": 612796,
     "cached_tokens": 579968,
     "output_tokens": 14982,
     "cost_usd": 0.015653
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 31,
     "prompt_tokens": 733109,
     "cached_tokens": 699776,
     "output_tokens": 16129,
     "cost_usd": 0.016777
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "Claude Sonnet 5.5",
   "model": "Claude Sonnet 5.5",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 62659,
     "cached_tokens": 54756,
     "output_tokens": 1392,
     "cost_usd": 0.05098
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 71924,
     "cached_tokens": 67243,
     "output_tokens": 1118,
     "cost_usd": 0.036592
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 60955,
     "cached_tokens": 57753,
     "output_tokens": 1148,
     "cost_usd": 0.030027
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 60721,
     "cached_tokens": 43861,
     "output_tokens": 1218,
     "cost_usd": 0.08397
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 86361,
     "cached_tokens": 73557,
     "output_tokens": 2053,
     "cost_usd": 0.079062
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 42541,
     "cached_tokens": 39613,
     "output_tokens": 836,
     "cost_usd": 0.024009
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": false,
     "calls": 5,
     "prompt_tokens": 45899,
     "cached_tokens": 41211,
     "output_tokens": 1193,
     "cost_usd": 0.034779
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 44531,
     "cached_tokens": 42892,
     "output_tokens": 772,
     "cost_usd": 0.018533
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 10,
     "prompt_tokens": 116380,
     "cached_tokens": 107993,
     "output_tokens": 3510,
     "cost_usd": 0.079403
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 39607,
     "cached_tokens": 36933,
     "output_tokens": 1183,
     "cost_usd": 0.026195
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "Claude Haiku 5.5",
   "model": "Claude Haiku 5.5",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 70724,
     "cached_tokens": 62479,
     "output_tokens": 3546,
     "cost_usd": 0.020226
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 74999,
     "cached_tokens": 63145,
     "output_tokens": 1198,
     "cost_usd": 0.017997
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 44284,
     "cached_tokens": 34457,
     "output_tokens": 1367,
     "cost_usd": 0.014961
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 4,
     "prompt_tokens": 32842,
     "cached_tokens": 30290,
     "output_tokens": 1385,
     "cost_usd": 0.007524
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 4,
     "prompt_tokens": 33875,
     "cached_tokens": 30719,
     "output_tokens": 1453,
     "cost_usd": 0.008319
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 41787,
     "cached_tokens": 32523,
     "output_tokens": 1337,
     "cost_usd": 0.014227
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": false,
     "calls": 5,
     "prompt_tokens": 45360,
     "cached_tokens": 40826,
     "output_tokens": 1246,
     "cost_usd": 0.009684
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 37552,
     "cached_tokens": 29172,
     "output_tokens": 951,
     "cost_usd": 0.01221
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 11,
     "prompt_tokens": 127698,
     "cached_tokens": 117754,
     "output_tokens": 7057,
     "cost_usd": 0.033462
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 46236,
     "cached_tokens": 35674,
     "output_tokens": 1539,
     "cost_usd": 0.016187
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "Kimi K2.7 Code",
   "model": "Kimi K2.7 Code",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 43,
     "prompt_tokens": 789613,
     "cached_tokens": 758784,
     "output_tokens": 7094,
     "cost_usd": 0.201833
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 22,
     "prompt_tokens": 266570,
     "cached_tokens": 251904,
     "output_tokens": 2682,
     "cost_usd": 0.072522
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 15,
     "prompt_tokens": 131324,
     "cached_tokens": 120064,
     "output_tokens": 1476,
     "cost_usd": 0.039413
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 10,
     "prompt_tokens": 73564,
     "cached_tokens": 67072,
     "output_tokens": 2560,
     "cost_usd": 0.029151
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 16,
     "prompt_tokens": 138453,
     "cached_tokens": 127744,
     "output_tokens": 3924,
     "cost_usd": 0.050141
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 32,
     "prompt_tokens": 426518,
     "cached_tokens": 404736,
     "output_tokens": 7204,
     "cost_usd": 0.126409
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": false,
     "calls": 41,
     "prompt_tokens": 918169,
     "cached_tokens": 882176,
     "output_tokens": 9835,
     "cost_usd": 0.241147
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 45105,
     "cached_tokens": 38912,
     "output_tokens": 884,
     "cost_usd": 0.016813
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 61,
     "prompt_tokens": 1334687,
     "cached_tokens": 1296640,
     "output_tokens": 15713,
     "cost_usd": 0.345358
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 30670,
     "cached_tokens": 28672,
     "output_tokens": 670,
     "cost_usd": 0.010026
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "GLM-5.3 Flash",
   "model": "GLM-5.3 Flash",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 44897,
     "cached_tokens": 14592,
     "output_tokens": 800,
     "cost_usd": 0.005384
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 58823,
     "cached_tokens": 26752,
     "output_tokens": 791,
     "cost_usd": 0.006009
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 28495,
     "cached_tokens": 9728,
     "output_tokens": 265,
     "cost_usd": 0.003239
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 4,
     "prompt_tokens": 16480,
     "cached_tokens": 4864,
     "output_tokens": 278,
     "cost_usd": 0.002027
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 43992,
     "cached_tokens": 21888,
     "output_tokens": 660,
     "cost_usd": 0.004302
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 28,
     "prompt_tokens": 220862,
     "cached_tokens": 111872,
     "output_tokens": 5450,
     "cost_usd": 0.02243
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": false,
     "calls": 17,
     "prompt_tokens": 133120,
     "cached_tokens": 63232,
     "output_tokens": 1407,
     "cost_usd": 0.013084
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 26648,
     "cached_tokens": 7296,
     "output_tokens": 185,
     "cost_usd": 0.003214
    },
    {
     "task": "sympy__sympy-13031",
     "solved": false,
     "calls": 11,
     "prompt_tokens": 69051,
     "cached_tokens": 31616,
     "output_tokens": 2763,
     "cost_usd": 0.007945
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 33350,
     "cached_tokens": 12160,
     "output_tokens": 466,
     "cost_usd": 0.003776
    }
   ]
  },
  {
   "harness": "Letronna",
   "label": "GPT-6 Luna",
   "model": "GPT-6 Luna",
   "network": "offline",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 17,
     "prompt_tokens": 198451,
     "cached_tokens": 73920,
     "output_tokens": 2241,
     "cost_usd": 0.034289
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 14,
     "prompt_tokens": 155314,
     "cached_tokens": 143110,
     "output_tokens": 1293,
     "cost_usd": 0.006881
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 71248,
     "cached_tokens": 64490,
     "output_tokens": 1080,
     "cost_usd": 0.003788
    },
    {
     "task": "django__django-15572",
     "solved": false,
     "calls": 10,
     "prompt_tokens": 75074,
     "cached_tokens": 69389,
     "output_tokens": 1080,
     "cost_usd": 0.003618
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 77948,
     "cached_tokens": 69776,
     "output_tokens": 1146,
     "cost_usd": 0.004297
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 71308,
     "cached_tokens": 63003,
     "output_tokens": 1829,
     "cost_usd": 0.004707
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": false,
     "calls": 13,
     "prompt_tokens": 165217,
     "cached_tokens": 152243,
     "output_tokens": 1469,
     "cost_usd": 0.007388
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 57884,
     "cached_tokens": 52146,
     "output_tokens": 790,
     "cost_usd": 0.003069
    },
    {
     "task": "sympy__sympy-13031",
     "solved": false,
     "calls": 11,
     "prompt_tokens": 103581,
     "cached_tokens": 47947,
     "output_tokens": 1940,
     "cost_usd": 0.016321
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 39538,
     "cached_tokens": 33259,
     "output_tokens": 626,
     "cost_usd": 0.002704
    }
   ]
  },
  {
   "harness": "Claude Code",
   "label": "published",
   "model": "DeepSeek V4.1 Flash",
   "network": "online",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 32,
     "prompt_tokens": 889240,
     "cached_tokens": 465536,
     "output_tokens": 8791,
     "cost_usd": 0.070227
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 108841,
     "cached_tokens": 74112,
     "output_tokens": 1071,
     "cost_usd": 0.006074
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 138352,
     "cached_tokens": 103808,
     "output_tokens": 988,
     "cost_usd": 0.006086
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 15,
     "prompt_tokens": 279797,
     "cached_tokens": 182272,
     "output_tokens": 4529,
     "cost_usd": 0.038893
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 12,
     "prompt_tokens": 255528,
     "cached_tokens": 163456,
     "output_tokens": 6569,
     "cost_usd": 0.018243
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 149368,
     "cached_tokens": 103808,
     "output_tokens": 2400,
     "cost_usd": 0.008585
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 24,
     "prompt_tokens": 517338,
     "cached_tokens": 299392,
     "output_tokens": 7056,
     "cost_usd": 0.044824
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 100058,
     "cached_tokens": 74112,
     "output_tokens": 673,
     "cost_usd": 0.004518
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 44,
     "prompt_tokens": 1347351,
     "cached_tokens": 619648,
     "output_tokens": 16115,
     "cost_usd": 0.127683
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 20,
     "prompt_tokens": 437371,
     "cached_tokens": 283520,
     "output_tokens": 4020,
     "cost_usd": 0.02634
    }
   ]
  },
  {
   "harness": "Codex",
   "label": "published",
   "model": "DeepSeek V4.1 Flash",
   "network": "online",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": false,
     "calls": 6,
     "prompt_tokens": 78696,
     "cached_tokens": 61568,
     "output_tokens": 1510,
     "cost_usd": 0.01066
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 84867,
     "cached_tokens": 72192,
     "output_tokens": 1495,
     "cost_usd": 0.003015
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 10,
     "prompt_tokens": 113275,
     "cached_tokens": 100736,
     "output_tokens": 1562,
     "cost_usd": 0.00312
    },
    {
     "task": "django__django-15572",
     "solved": false,
     "calls": 3,
     "prompt_tokens": 19876,
     "cached_tokens": 9600,
     "output_tokens": 828,
     "cost_usd": 0.002067
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 14,
     "prompt_tokens": 260850,
     "cached_tokens": 237696,
     "output_tokens": 5744,
     "cost_usd": 0.014633
    },
    {
     "task": "psf__requests-5414",
     "solved": false,
     "calls": 4,
     "prompt_tokens": 31758,
     "cached_tokens": 20352,
     "output_tokens": 715,
     "cost_usd": 0.002201
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 30,
     "prompt_tokens": 681604,
     "cached_tokens": 646144,
     "output_tokens": 8772,
     "cost_usd": 0.033521
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 64697,
     "cached_tokens": 52864,
     "output_tokens": 791,
     "cost_usd": 0.002408
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 36,
     "prompt_tokens": 930588,
     "cached_tokens": 890624,
     "output_tokens": 13906,
     "cost_usd": 0.04501
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 18,
     "prompt_tokens": 283948,
     "cached_tokens": 265728,
     "output_tokens": 2593,
     "cost_usd": 0.005086
    }
   ]
  },
  {
   "harness": "OpenCode",
   "label": "published",
   "model": "DeepSeek V4.1 Flash",
   "network": "online",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 12,
     "prompt_tokens": 130389,
     "cached_tokens": 115968,
     "output_tokens": 2492,
     "cost_usd": 0.004006
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 46961,
     "cached_tokens": 32384,
     "output_tokens": 410,
     "cost_usd": 0.00253
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 5,
     "prompt_tokens": 35725,
     "cached_tokens": 25216,
     "output_tokens": 387,
     "cost_usd": 0.001884
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 11,
     "prompt_tokens": 110956,
     "cached_tokens": 98560,
     "output_tokens": 4270,
     "cost_usd": 0.004717
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 20,
     "prompt_tokens": 635163,
     "cached_tokens": 582784,
     "output_tokens": 7888,
     "cost_usd": 0.014338
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 15,
     "prompt_tokens": 247108,
     "cached_tokens": 223616,
     "output_tokens": 4290,
     "cost_usd": 0.006769
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 27,
     "prompt_tokens": 804182,
     "cached_tokens": 755328,
     "output_tokens": 8478,
     "cost_usd": 0.014681
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 7,
     "prompt_tokens": 55462,
     "cached_tokens": 44672,
     "output_tokens": 665,
     "cost_usd": 0.002152
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 46,
     "prompt_tokens": 1328365,
     "cached_tokens": 1282816,
     "output_tokens": 12627,
     "cost_usd": 0.018257
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 29,
     "prompt_tokens": 460491,
     "cached_tokens": 440192,
     "output_tokens": 5287,
     "cost_usd": 0.007538
    }
   ]
  },
  {
   "harness": "OpenClaw",
   "label": "published",
   "model": "DeepSeek V4.1 Flash",
   "network": "online",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 20,
     "prompt_tokens": 272479,
     "cached_tokens": 257408,
     "output_tokens": 7622,
     "cost_usd": 0.007606
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 14,
     "prompt_tokens": 175893,
     "cached_tokens": 166528,
     "output_tokens": 2546,
     "cost_usd": 0.003432
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 11,
     "prompt_tokens": 119534,
     "cached_tokens": 114688,
     "output_tokens": 1689,
     "cost_usd": 0.002084
    },
    {
     "task": "django__django-15572",
     "solved": false,
     "calls": 11,
     "prompt_tokens": 113729,
     "cached_tokens": 109184,
     "output_tokens": 3323,
     "cost_usd": 0.003003
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 26,
     "prompt_tokens": 683909,
     "cached_tokens": 638080,
     "output_tokens": 14089,
     "cost_usd": 0.017242
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 13,
     "prompt_tokens": 164936,
     "cached_tokens": 155136,
     "output_tokens": 4944,
     "cost_usd": 0.004902
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 39,
     "prompt_tokens": 803543,
     "cached_tokens": 776448,
     "output_tokens": 18194,
     "cost_usd": 0.01731
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 60351,
     "cached_tokens": 56448,
     "output_tokens": 793,
     "cost_usd": 0.001231
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 38,
     "prompt_tokens": 1154209,
     "cached_tokens": 1094400,
     "output_tokens": 34543,
     "cost_usd": 0.03298
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 20,
     "prompt_tokens": 340192,
     "cached_tokens": 324096,
     "output_tokens": 5111,
     "cost_usd": 0.006453
    }
   ]
  },
  {
   "harness": "Hermes",
   "label": "published",
   "model": "DeepSeek V4.1 Flash",
   "network": "online",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 29,
     "prompt_tokens": 721312,
     "cached_tokens": 691968,
     "output_tokens": 6166,
     "cost_usd": 0.010177
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 14,
     "prompt_tokens": 242917,
     "cached_tokens": 224000,
     "output_tokens": 2269,
     "cost_usd": 0.004871
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 17,
     "prompt_tokens": 286268,
     "cached_tokens": 269056,
     "output_tokens": 2104,
     "cost_usd": 0.004651
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 18,
     "prompt_tokens": 387493,
     "cached_tokens": 365312,
     "output_tokens": 4690,
     "cost_usd": 0.007237
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 31,
     "prompt_tokens": 997590,
     "cached_tokens": 959872,
     "output_tokens": 10229,
     "cost_usd": 0.014675
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 14,
     "prompt_tokens": 273221,
     "cached_tokens": 251392,
     "output_tokens": 2848,
     "cost_usd": 0.005737
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 17,
     "prompt_tokens": 474992,
     "cached_tokens": 436608,
     "output_tokens": 3466,
     "cost_usd": 0.009147
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 118203,
     "cached_tokens": 102016,
     "output_tokens": 1300,
     "cost_usd": 0.003514
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 30,
     "prompt_tokens": 839958,
     "cached_tokens": 806400,
     "output_tokens": 9973,
     "cost_usd": 0.013437
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 17,
     "prompt_tokens": 426940,
     "cached_tokens": 400000,
     "output_tokens": 5085,
     "cost_usd": 0.008292
    }
   ]
  },
  {
   "harness": "DeepSeek harness",
   "label": "published",
   "model": "DeepSeek V4.1 Flash",
   "network": "online",
   "tasks": [
    {
     "task": "django__django-13569",
     "solved": true,
     "calls": 21,
     "prompt_tokens": 386001,
     "cached_tokens": 361216,
     "output_tokens": 7384,
     "cost_usd": 0.009232
    },
    {
     "task": "django__django-14672",
     "solved": true,
     "calls": 8,
     "prompt_tokens": 81369,
     "cached_tokens": 77440,
     "output_tokens": 1281,
     "cost_usd": 0.00159
    },
    {
     "task": "django__django-15277",
     "solved": true,
     "calls": 6,
     "prompt_tokens": 55619,
     "cached_tokens": 52736,
     "output_tokens": 1795,
     "cost_usd": 0.001668
    },
    {
     "task": "django__django-15572",
     "solved": true,
     "calls": 14,
     "prompt_tokens": 234298,
     "cached_tokens": 222080,
     "output_tokens": 9067,
     "cost_usd": 0.007939
    },
    {
     "task": "matplotlib__matplotlib-24570",
     "solved": true,
     "calls": 23,
     "prompt_tokens": 411110,
     "cached_tokens": 399616,
     "output_tokens": 12688,
     "cost_usd": 0.010536
    },
    {
     "task": "psf__requests-5414",
     "solved": true,
     "calls": 12,
     "prompt_tokens": 194770,
     "cached_tokens": 181248,
     "output_tokens": 6654,
     "cost_usd": 0.006564
    },
    {
     "task": "pylint-dev__pylint-4970",
     "solved": true,
     "calls": 27,
     "prompt_tokens": 770987,
     "cached_tokens": 734208,
     "output_tokens": 14496,
     "cost_usd": 0.016417
    },
    {
     "task": "scikit-learn__scikit-learn-14141",
     "solved": true,
     "calls": 10,
     "prompt_tokens": 137963,
     "cached_tokens": 128512,
     "output_tokens": 2471,
     "cost_usd": 0.003286
    },
    {
     "task": "sympy__sympy-13031",
     "solved": true,
     "calls": 68,
     "prompt_tokens": 4534162,
     "cached_tokens": 4400128,
     "output_tokens": 47743,
     "cost_usd": 0.061951
    },
    {
     "task": "sympy__sympy-16766",
     "solved": true,
     "calls": 23,
     "prompt_tokens": 527095,
     "cached_tokens": 494976,
     "output_tokens": 6470,
     "cost_usd": 0.010185
    }
   ]
  }
 ]
}