{
  "version": 1,
  "started_at": "2026-09-28T17:45:57.324Z",
  "finished_at": "2026-09-28T17:47:39.248Z",
  "dataset_sha256": "bd4071e587737a965e0c59877bb33ebae5bad56b1293cd414ae71f6fe7cff102",
  "methodology": {
    "suite": "math-v1",
    "cases": 24,
    "repeats": 3,
    "warmups_per_target": 1,
    "runner": "Huawei VPS, Hong Kong",
    "order": "Sequential requests, alternating provider order by case and repeat",
    "jevify": "Public HTTPS Jevify API → Huawei MaaS DeepSeek V4.1 Flash, thinking disabled, temperature 0",
    "jev": "OpenRouter Decisions API → TypeSafe Jev 1.13",
    "cache": "Demo cache bypassed; provider-side caching may still apply",
    "scoring": "Exact integer reference labels from scripts/math-reference.py: exact Choice, Noul >= 0.5, Score rounded to nearest rubric level. Errors count as incorrect. Latency is successful end-to-end requests; no retries by benchmark client.",
    "limitations": "24 authored mathematics problems, repeated 3 times, not 72 independent examples. Exact reference answers generated before execution; no prompt tuning after results. This is a deliberately difficult mathematics stress test, not a representative classification workload or a general intelligence ranking. Jevify thinking is disabled. Network paths and providers differ. The earlier routing benchmark is retained separately."
  },
  "summary": {
    "jevify": {
      "total": 72,
      "correct": 29,
      "errors": 0,
      "accuracy": 0.4027777777777778,
      "median_ms": 953,
      "p95_ms": 1653,
      "noul_brier": 0.4375,
      "score_mae": 4.625,
      "by_type": {
        "Choice": {
          "correct": 12,
          "total": 24
        },
        "Score": {
          "correct": 3,
          "total": 24
        },
        "Noul": {
          "correct": 14,
          "total": 24
        }
      }
    },
    "jev": {
      "total": 72,
      "correct": 30,
      "errors": 0,
      "accuracy": 0.4166666666666667,
      "median_ms": 295,
      "p95_ms": 444,
      "noul_brier": 0.24257916666666676,
      "score_mae": 2.3895833333333334,
      "by_type": {
        "Choice": {
          "correct": 12,
          "total": 24
        },
        "Score": {
          "correct": 6,
          "total": 24
        },
        "Noul": {
          "correct": 12,
          "total": 24
        }
      }
    }
  },
  "cases": [
    {
      "id": "m01",
      "category": "Choice",
      "title": "Modular exponentiation",
      "request": {
        "model": "jev-latest",
        "state": "What is the remainder when 7^2025 is divided by 1000?",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "790": null,
              "804": null,
              "807": null,
              "808": null,
              "836": null
            }
          }
        }
      },
      "expected": "807",
      "rationale": "Exact modular exponentiation: pow(7, 2025, 1000)."
    },
    {
      "id": "m02",
      "category": "Choice",
      "title": "Chinese remainder system",
      "request": {
        "model": "jev-latest",
        "state": "Find the least nonnegative integer x such that x ≡ 17 (mod 23), x ≡ 29 (mod 31), and x ≡ 31 (mod 37).",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "15295": null,
              "15309": null,
              "15312": null,
              "15313": null,
              "15341": null
            }
          }
        }
      },
      "expected": "15312",
      "rationale": "Exhaustively checked 0 ≤ x < 23×31×37; pairwise coprime moduli give uniqueness."
    },
    {
      "id": "m03",
      "category": "Choice",
      "title": "Forbidden substring count",
      "request": {
        "model": "jev-latest",
        "state": "How many binary strings of length 30 contain no occurrence of three consecutive ones (111)?",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "98950079": null,
              "98950093": null,
              "98950096": null,
              "98950097": null,
              "98950125": null
            }
          }
        }
      },
      "expected": "98950096",
      "rationale": "Integer DP over suffixes containing zero, one or two trailing ones; checked against enumeration for lengths 1–8."
    },
    {
      "id": "m04",
      "category": "Choice",
      "title": "Polynomial coefficient",
      "request": {
        "model": "jev-latest",
        "state": "What is the coefficient of x^18 in (1 + x + x^2)^20?",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "326527333": null,
              "326527347": null,
              "326527350": null,
              "326527351": null,
              "326527379": null
            }
          }
        }
      },
      "expected": "326527350",
      "rationale": "Twenty exact polynomial convolutions; smaller powers independently checked by exhaustive enumeration."
    },
    {
      "id": "m05",
      "category": "Choice",
      "title": "Surjection count",
      "request": {
        "model": "jev-latest",
        "state": "How many functions from a labeled 9-element set to a labeled 4-element set are onto?",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "186463": null,
              "186477": null,
              "186480": null,
              "186481": null,
              "186509": null
            }
          }
        }
      },
      "expected": "186480",
      "rationale": "Inclusion-exclusion: 4^9 − 4×3^9 + 6×2^9 − 4."
    },
    {
      "id": "m06",
      "category": "Choice",
      "title": "Fibonacci residue",
      "request": {
        "model": "jev-latest",
        "state": "F_0=0, F_1=1, and F_n=F_(n−1)+F_(n−2). What is F_100 modulo 1000?",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "58": null,
              "72": null,
              "75": null,
              "76": null,
              "104": null
            }
          }
        }
      },
      "expected": "75",
      "rationale": "Exact 100-step integer recurrence, then reduction modulo 1000."
    },
    {
      "id": "m07",
      "category": "Choice",
      "title": "Constrained subset sum",
      "request": {
        "model": "jev-latest",
        "state": "How many 8-element subsets of {1,2,...,24} have elements summing to exactly 100?",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "17558": null,
              "17572": null,
              "17575": null,
              "17576": null,
              "17604": null
            }
          }
        }
      },
      "expected": "17575",
      "rationale": "Integer DP indexed by subset size and total; an independent small case was checked by enumeration."
    },
    {
      "id": "m08",
      "category": "Choice",
      "title": "Five-by-five determinant",
      "request": {
        "model": "jev-latest",
        "state": "Compute the determinant of this matrix: [[7, -2, 3, 1, 5], [4, 6, -1, 2, 0], [3, 1, 8, -4, 2], [0, 5, 2, 9, -3], [6, -1, 4, 2, 7]]",
        "questions": {
          "decision": {
            "type": "choice",
            "instructions": "Solve the mathematical problem exactly. Select the correct answer.",
            "criteria": {
              "12183": null,
              "12197": null,
              "12200": null,
              "12201": null,
              "12229": null
            }
          }
        }
      },
      "expected": "12200",
      "rationale": "Exact recursive cofactor expansion using integer arithmetic."
    },
    {
      "id": "m09",
      "category": "Noul",
      "title": "Large-power claim",
      "request": {
        "model": "jev-latest",
        "state": "Claim: the last six decimal digits of 123456789^12345 are 338549.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": true,
      "rationale": "Verified by exact modular exponentiation modulo 1,000,000."
    },
    {
      "id": "m10",
      "category": "Noul",
      "title": "Derangement near miss",
      "request": {
        "model": "jev-latest",
        "state": "Claim: exactly 176214842 permutations of 12 labeled objects have no fixed points.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": false,
      "rationale": "Derangement recurrence D_n=(n−1)(D_(n−1)+D_(n−2)) gives 176214841."
    },
    {
      "id": "m11",
      "category": "Noul",
      "title": "Separated binary positions",
      "request": {
        "model": "jev-latest",
        "state": "Claim: the number of length-40 binary strings with exactly twelve ones and no adjacent ones is 51895934.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": false,
      "rationale": "The spacing bijection gives C(40−12+1,12) = 51895935."
    },
    {
      "id": "m12",
      "category": "Noul",
      "title": "Constrained lattice paths",
      "request": {
        "model": "jev-latest",
        "state": "Claim: exactly 208012 paths from (0,0) to (12,12), using only unit right/up steps, never go strictly above y=x.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": true,
      "rationale": "Catalan number C_12 = C(24,12)/13."
    },
    {
      "id": "m13",
      "category": "Noul",
      "title": "Prime counting",
      "request": {
        "model": "jev-latest",
        "state": "Claim: the closed interval [10000,11000] contains exactly 106 prime numbers.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": true,
      "rationale": "Trial division of every candidate through its integer square root."
    },
    {
      "id": "m14",
      "category": "Noul",
      "title": "Dice-sum probability",
      "request": {
        "model": "jev-latest",
        "state": "Seven independent fair six-sided dice are rolled. Claim: the probability that their sum is exactly 28 is greater than 1/12.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": false,
      "rationale": "Exact convolution of seven die distributions gives 6109/93312; compared as a rational number."
    },
    {
      "id": "m15",
      "category": "Noul",
      "title": "Exactly three fixed points",
      "request": {
        "model": "jev-latest",
        "state": "Claim: exactly 222486 permutations of 10 labeled objects have exactly three fixed points.",
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": false,
      "rationale": "Choose the three fixed points, then derange the other seven: C(10,3)D_7 = 222480."
    },
    {
      "id": "m16",
      "category": "Noul",
      "title": "Weighted shortest path",
      "request": {
        "model": "jev-latest",
        "state": {
          "edges": [
            {
              "from": 0,
              "to": 1,
              "cost": 3
            },
            {
              "from": 0,
              "to": 2,
              "cost": 11
            },
            {
              "from": 0,
              "to": 3,
              "cost": 19
            },
            {
              "from": 1,
              "to": 2,
              "cost": 4
            },
            {
              "from": 1,
              "to": 3,
              "cost": 25
            },
            {
              "from": 1,
              "to": 4,
              "cost": 15
            },
            {
              "from": 2,
              "to": 3,
              "cost": 31
            },
            {
              "from": 2,
              "to": 4,
              "cost": 3
            },
            {
              "from": 2,
              "to": 5,
              "cost": 6
            },
            {
              "from": 3,
              "to": 4,
              "cost": 22
            },
            {
              "from": 3,
              "to": 5,
              "cost": 7
            },
            {
              "from": 3,
              "to": 6,
              "cost": 23
            },
            {
              "from": 4,
              "to": 5,
              "cost": 8
            },
            {
              "from": 4,
              "to": 6,
              "cost": 6
            },
            {
              "from": 4,
              "to": 7,
              "cost": 4
            },
            {
              "from": 5,
              "to": 6,
              "cost": 20
            },
            {
              "from": 5,
              "to": 7,
              "cost": 31
            },
            {
              "from": 5,
              "to": 8,
              "cost": 11
            },
            {
              "from": 6,
              "to": 7,
              "cost": 27
            },
            {
              "from": 6,
              "to": 8,
              "cost": 20
            },
            {
              "from": 6,
              "to": 9,
              "cost": 13
            },
            {
              "from": 7,
              "to": 8,
              "cost": 29
            },
            {
              "from": 7,
              "to": 9,
              "cost": 4
            },
            {
              "from": 8,
              "to": 9,
              "cost": 26
            }
          ],
          "claim": "The minimum total cost from vertex 0 to vertex 9 is 18.",
          "rules": "Edges are directed; costs add; only listed edges exist."
        },
        "questions": {
          "decision": {
            "type": "noul",
            "instructions": "Is the mathematical claim in the context true? Evaluate exactly; do not assume a stated number is correct."
          }
        }
      },
      "expected": true,
      "rationale": "DAG shortest-path DP in vertex order gives 18."
    },
    {
      "id": "m17",
      "category": "Score",
      "title": "Exponent tower digit",
      "request": {
        "model": "jev-latest",
        "state": "What is the last decimal digit of 7^(7^7)? Exponentiation is right-associative.",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 3,
      "rationale": "The powers of 7 cycle modulo 10 with period 4. 7^7 mod 4 = 3, so 7^3 mod 10 = 3."
    },
    {
      "id": "m18",
      "category": "Score",
      "title": "Binomial final digit",
      "request": {
        "model": "jev-latest",
        "state": "What is the last decimal digit of the binomial coefficient C(100,50)?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 6,
      "rationale": "Exact factorial quotient, reduced modulo 10."
    },
    {
      "id": "m19",
      "category": "Score",
      "title": "Congruence-constrained triples",
      "request": {
        "model": "jev-latest",
        "state": "Let N count ordered triples of nonnegative integers (x,y,z) with x+y+z=120, x ≡ 1 (mod 4), y ≡ 2 (mod 5), and z ≡ 3 (mod 6). What is N modulo 10?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 4,
      "rationale": "Exhaustive enumeration of all admissible x and y, with z=120−x−y, gives N=64."
    },
    {
      "id": "m20",
      "category": "Score",
      "title": "Permanent final digit",
      "request": {
        "model": "jev-latest",
        "state": "For A=[[2, 1, 3, 0, 2, 4], [1, 3, 2, 1, 0, 2], [4, 0, 1, 2, 3, 1], [2, 1, 0, 3, 1, 2], [3, 2, 1, 4, 2, 0], [0, 4, 2, 1, 3, 2]], define perm(A) = sum over all permutations p of {0,...,5} of product A[i][p(i)] for i=0,...,5 (no signs). What is perm(A) modulo 10?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 7,
      "rationale": "Exact enumeration of all 720 permutations gives permanent 29337."
    },
    {
      "id": "m21",
      "category": "Score",
      "title": "Factorial zeros in base twelve",
      "request": {
        "model": "jev-latest",
        "state": "Let Z be the number of trailing zero digits in the base-12 representation of 1000!. What is Z modulo 10?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 7,
      "rationale": "Since 12=2²×3, Z=min(floor(v_2(1000!)/2),v_3(1000!))=497."
    },
    {
      "id": "m22",
      "category": "Score",
      "title": "Sum of squares residue",
      "request": {
        "model": "jev-latest",
        "state": "What is (1^2 + 2^2 + ... + 1000003^2) modulo 10?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 4,
      "rationale": "Exact formula n(n+1)(2n+1)/6 before reduction modulo 10."
    },
    {
      "id": "m23",
      "category": "Score",
      "title": "Distinct integer partitions",
      "request": {
        "model": "jev-latest",
        "state": "Let P be the number of ways to write 80 as a sum of distinct positive integers, ignoring order. What is P modulo 10?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 2,
      "rationale": "Descending subset-sum DP over parts 1 through 80 gives P=77312."
    },
    {
      "id": "m24",
      "category": "Score",
      "title": "Recurrence residue",
      "request": {
        "model": "jev-latest",
        "state": "a_0=2, a_1=3 and a_n=3a_(n−1)+2a_(n−2) for n≥2. What is a_100 modulo 10?",
        "questions": {
          "decision": {
            "type": "score",
            "instructions": "Solve the problem exactly. Select the rubric level matching the resulting digit. Put all probability on that digit.",
            "criteria": [
              "The answer is the digit 0",
              "The answer is the digit 1",
              "The answer is the digit 2",
              "The answer is the digit 3",
              "The answer is the digit 4",
              "The answer is the digit 5",
              "The answer is the digit 6",
              "The answer is the digit 7",
              "The answer is the digit 8",
              "The answer is the digit 9"
            ]
          }
        }
      },
      "expected": 1,
      "rationale": "Exact 100-term integer recurrence, then reduction modulo 10."
    }
  ],
  "runs": [
    {
      "case_id": "m01",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 871,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "807",
        "probabilities": {
          "790": 0,
          "804": 0,
          "807": 1,
          "808": 0,
          "836": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 464,
        "output_tokens": 17
      },
      "prediction": "807",
      "correct": true
    },
    {
      "case_id": "m01",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 645,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "807",
        "probabilities": {
          "790": 0.01,
          "804": 0.06,
          "807": 0.8,
          "808": 0.1,
          "836": 0.02
        },
        "confidence": 0.75
      },
      "usage": {
        "input_tokens": 343,
        "output_tokens": 64,
        "cost": 0.000014406
      },
      "prediction": "807",
      "correct": true
    },
    {
      "case_id": "m02",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 277,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "15313",
        "probabilities": {
          "15295": 0.13,
          "15309": 0.28,
          "15312": 0.1,
          "15313": 0.36,
          "15341": 0.13
        },
        "confidence": 0.2
      },
      "usage": {
        "input_tokens": 379,
        "output_tokens": 76,
        "cost": 0.000015918
      },
      "prediction": "15313",
      "correct": false
    },
    {
      "case_id": "m02",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1142,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "15312",
        "probabilities": {
          "15295": 0,
          "15309": 0,
          "15312": 1,
          "15313": 0,
          "15341": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 489,
        "output_tokens": 17
      },
      "prediction": "15312",
      "correct": true
    },
    {
      "case_id": "m03",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1209,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "98950079",
        "probabilities": {
          "98950079": 1,
          "98950093": 0,
          "98950096": 0,
          "98950097": 0,
          "98950125": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 475,
        "output_tokens": 17
      },
      "prediction": "98950079",
      "correct": false
    },
    {
      "case_id": "m03",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 290,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "98950097",
        "probabilities": {
          "98950079": 0.09,
          "98950093": 0.12,
          "98950096": 0.13,
          "98950097": 0.64,
          "98950125": 0.02
        },
        "confidence": 0.54
      },
      "usage": {
        "input_tokens": 368,
        "output_tokens": 94,
        "cost": 0.000015456
      },
      "prediction": "98950097",
      "correct": false
    },
    {
      "case_id": "m04",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 309,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "326527350",
        "probabilities": {
          "326527333": 0.06,
          "326527347": 0.07,
          "326527350": 0.72,
          "326527351": 0.11,
          "326527379": 0.04
        },
        "confidence": 0.64
      },
      "usage": {
        "input_tokens": 374,
        "output_tokens": 100,
        "cost": 0.000015708
      },
      "prediction": "326527350",
      "correct": true
    },
    {
      "case_id": "m04",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1113,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "326527333",
        "probabilities": {
          "326527333": 1,
          "326527347": 0,
          "326527350": 0,
          "326527351": 0,
          "326527379": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 477,
        "output_tokens": 17
      },
      "prediction": "326527333",
      "correct": false
    },
    {
      "case_id": "m05",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 910,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "186480",
        "probabilities": {
          "186463": 0,
          "186477": 0,
          "186480": 1,
          "186481": 0,
          "186509": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 472,
        "output_tokens": 17
      },
      "prediction": "186480",
      "correct": true
    },
    {
      "case_id": "m05",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 320,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "186480",
        "probabilities": {
          "186463": 0.01,
          "186477": 0.01,
          "186480": 0.96,
          "186481": 0.02,
          "186509": 0
        },
        "confidence": 0.95
      },
      "usage": {
        "input_tokens": 357,
        "output_tokens": 82,
        "cost": 0.000014994
      },
      "prediction": "186480",
      "correct": true
    },
    {
      "case_id": "m06",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 300,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "75",
        "probabilities": {
          "58": 0.17,
          "72": 0.12,
          "75": 0.3,
          "76": 0.23,
          "104": 0.18
        },
        "confidence": 0.12
      },
      "usage": {
        "input_tokens": 360,
        "output_tokens": 59,
        "cost": 0.00001512
      },
      "prediction": "75",
      "correct": true
    },
    {
      "case_id": "m06",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1319,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "75",
        "probabilities": {
          "58": 0,
          "72": 0,
          "75": 1,
          "76": 0,
          "104": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 484,
        "output_tokens": 17
      },
      "prediction": "75",
      "correct": true
    },
    {
      "case_id": "m07",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 934,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "17558",
        "probabilities": {
          "17558": 0.2,
          "17572": 0.2,
          "17575": 0.2,
          "17576": 0.2,
          "17604": 0.2
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 474,
        "output_tokens": 43
      },
      "prediction": "17558",
      "correct": false
    },
    {
      "case_id": "m07",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 455,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "17576",
        "probabilities": {
          "17558": 0.06,
          "17572": 0.08,
          "17575": 0.1,
          "17576": 0.72,
          "17604": 0.04
        },
        "confidence": 0.65
      },
      "usage": {
        "input_tokens": 359,
        "output_tokens": 76,
        "cost": 0.000015078
      },
      "prediction": "17576",
      "correct": false
    },
    {
      "case_id": "m08",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 325,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "12201",
        "probabilities": {
          "12183": 0.13,
          "12197": 0.32,
          "12200": 0.06,
          "12201": 0.35,
          "12229": 0.14
        },
        "confidence": 0.2
      },
      "usage": {
        "input_tokens": 414,
        "output_tokens": 76,
        "cost": 0.000017388
      },
      "prediction": "12201",
      "correct": false
    },
    {
      "case_id": "m08",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1668,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "12183",
        "probabilities": {
          "12183": 0.2,
          "12197": 0.2,
          "12200": 0.2,
          "12201": 0.2,
          "12229": 0.2
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 534,
        "output_tokens": 43
      },
      "prediction": "12183",
      "correct": false
    },
    {
      "case_id": "m09",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1038,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 419,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m09",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 254,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.44
      },
      "usage": {
        "input_tokens": 320,
        "output_tokens": 20,
        "cost": 0.00001344
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m10",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 284,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.28
      },
      "usage": {
        "input_tokens": 312,
        "output_tokens": 20,
        "cost": 0.000013104
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m10",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 904,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 417,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m11",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1653,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 423,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m11",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 302,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.3
      },
      "usage": {
        "input_tokens": 317,
        "output_tokens": 20,
        "cost": 0.000013314
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m12",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 252,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.43
      },
      "usage": {
        "input_tokens": 326,
        "output_tokens": 20,
        "cost": 0.000013692
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m12",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1089,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 433,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m13",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 827,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0.5
      },
      "usage": {
        "input_tokens": 418,
        "output_tokens": 17
      },
      "prediction": true,
      "correct": true
    },
    {
      "case_id": "m13",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 323,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.37
      },
      "usage": {
        "input_tokens": 314,
        "output_tokens": 20,
        "cost": 0.000013188
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m14",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 455,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.26
      },
      "usage": {
        "input_tokens": 317,
        "output_tokens": 20,
        "cost": 0.000013314
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m14",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 960,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 427,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m15",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 856,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 417,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m15",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 296,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.44
      },
      "usage": {
        "input_tokens": 310,
        "output_tokens": 20,
        "cost": 0.00001302
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m16",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 288,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.39
      },
      "usage": {
        "input_tokens": 830,
        "output_tokens": 20,
        "cost": 0.00003486
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m16",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1149,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 726,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m17",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1040,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 3,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 1,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 567,
        "output_tokens": 17
      },
      "prediction": 3,
      "correct": true
    },
    {
      "case_id": "m17",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 265,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 5.52,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0,
          "1": 0.01,
          "2": 0.03,
          "3": 0.34,
          "4": 0.05,
          "5": 0,
          "6": 0.03,
          "7": 0.36,
          "8": 0.06,
          "9": 0.12
        },
        "confidence": 0.17
      },
      "usage": {
        "input_tokens": 439,
        "output_tokens": 17,
        "cost": 0.000018438
      },
      "prediction": 6,
      "correct": false
    },
    {
      "case_id": "m18",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 316,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 2.13,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.55,
          "1": 0.03,
          "2": 0.11,
          "3": 0.02,
          "4": 0.03,
          "5": 0.05,
          "6": 0.11,
          "7": 0.01,
          "8": 0.05,
          "9": 0.04
        },
        "confidence": 0.15
      },
      "usage": {
        "input_tokens": 435,
        "output_tokens": 17,
        "cost": 0.00001827
      },
      "prediction": 2,
      "correct": false
    },
    {
      "case_id": "m18",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 936,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 563,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m19",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 891,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 604,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m19",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 250,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 3.86,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.19999999999999998,
          "1": 0.1,
          "2": 0.09,
          "3": 0.09,
          "4": 0.08,
          "5": 0.13,
          "6": 0.08,
          "7": 0.07,
          "8": 0.08,
          "9": 0.08
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 479,
        "output_tokens": 17,
        "cost": 0.000020118
      },
      "prediction": 4,
      "correct": true
    },
    {
      "case_id": "m20",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 281,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 2.98,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.39999999999999997,
          "1": 0.04,
          "2": 0.09,
          "3": 0.04,
          "4": 0.07,
          "5": 0.11,
          "6": 0.07,
          "7": 0.05,
          "8": 0.07,
          "9": 0.06
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 571,
        "output_tokens": 17,
        "cost": 0.000023982
      },
      "prediction": 3,
      "correct": false
    },
    {
      "case_id": "m20",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 1423,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 701,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m21",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 950,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 574,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m21",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 313,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.33,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.15000000000000002,
          "1": 0.07,
          "2": 0.1,
          "3": 0.09,
          "4": 0.11,
          "5": 0.11,
          "6": 0.1,
          "7": 0.07,
          "8": 0.1,
          "9": 0.1
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 446,
        "output_tokens": 17,
        "cost": 0.000018732
      },
      "prediction": 4,
      "correct": false
    },
    {
      "case_id": "m22",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 366,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.48,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.09,
          "1": 0.07,
          "2": 0.07,
          "3": 0.19,
          "4": 0.08,
          "5": 0.14,
          "6": 0.08,
          "7": 0.12,
          "8": 0.09,
          "9": 0.07
        },
        "confidence": 0.04
      },
      "usage": {
        "input_tokens": 445,
        "output_tokens": 17,
        "cost": 0.00001869
      },
      "prediction": 4,
      "correct": true
    },
    {
      "case_id": "m22",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 871,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 572,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m23",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 865,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 6,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 1,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 576,
        "output_tokens": 17
      },
      "prediction": 6,
      "correct": false
    },
    {
      "case_id": "m23",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 295,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 3.69,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.28,
          "1": 0.08,
          "2": 0.07,
          "3": 0.07,
          "4": 0.07,
          "5": 0.09,
          "6": 0.08,
          "7": 0.07,
          "8": 0.08,
          "9": 0.1
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 446,
        "output_tokens": 17,
        "cost": 0.000018732
      },
      "prediction": 4,
      "correct": false
    },
    {
      "case_id": "m24",
      "target": "jev",
      "repeat": 0,
      "elapsed_ms": 300,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 5.03,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.07,
          "1": 0.07,
          "2": 0.12,
          "3": 0.11,
          "4": 0.07,
          "5": 0.04,
          "6": 0.11,
          "7": 0.15,
          "8": 0.14,
          "9": 0.12
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 461,
        "output_tokens": 17,
        "cost": 0.000019362
      },
      "prediction": 5,
      "correct": false
    },
    {
      "case_id": "m24",
      "target": "jevify",
      "repeat": 0,
      "elapsed_ms": 897,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 6,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 1,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 590,
        "output_tokens": 17
      },
      "prediction": 6,
      "correct": false
    },
    {
      "case_id": "m01",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 307,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "807",
        "probabilities": {
          "790": 0.01,
          "804": 0.06,
          "807": 0.8099999999999999,
          "808": 0.09,
          "836": 0.03
        },
        "confidence": 0.77
      },
      "usage": {
        "input_tokens": 343,
        "output_tokens": 64,
        "cost": 0.000014406
      },
      "prediction": "807",
      "correct": true
    },
    {
      "case_id": "m01",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 829,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "807",
        "probabilities": {
          "790": 0,
          "804": 0,
          "807": 1,
          "808": 0,
          "836": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 464,
        "output_tokens": 17
      },
      "prediction": "807",
      "correct": true
    },
    {
      "case_id": "m02",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 962,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "15312",
        "probabilities": {
          "15295": 0,
          "15309": 0,
          "15312": 1,
          "15313": 0,
          "15341": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 489,
        "output_tokens": 17
      },
      "prediction": "15312",
      "correct": true
    },
    {
      "case_id": "m02",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 261,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "15313",
        "probabilities": {
          "15295": 0.13,
          "15309": 0.28,
          "15312": 0.1,
          "15313": 0.37,
          "15341": 0.12
        },
        "confidence": 0.21
      },
      "usage": {
        "input_tokens": 379,
        "output_tokens": 76,
        "cost": 0.000015918
      },
      "prediction": "15313",
      "correct": false
    },
    {
      "case_id": "m03",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 281,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "98950097",
        "probabilities": {
          "98950079": 0.08,
          "98950093": 0.12,
          "98950096": 0.13,
          "98950097": 0.64,
          "98950125": 0.03
        },
        "confidence": 0.55
      },
      "usage": {
        "input_tokens": 368,
        "output_tokens": 94,
        "cost": 0.000015456
      },
      "prediction": "98950097",
      "correct": false
    },
    {
      "case_id": "m03",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 843,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "98950079",
        "probabilities": {
          "98950079": 1,
          "98950093": 0,
          "98950096": 0,
          "98950097": 0,
          "98950125": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 475,
        "output_tokens": 17
      },
      "prediction": "98950079",
      "correct": false
    },
    {
      "case_id": "m04",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 878,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "326527333",
        "probabilities": {
          "326527333": 1,
          "326527347": 0,
          "326527350": 0,
          "326527351": 0,
          "326527379": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 477,
        "output_tokens": 17
      },
      "prediction": "326527333",
      "correct": false
    },
    {
      "case_id": "m04",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 324,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "326527350",
        "probabilities": {
          "326527333": 0.06,
          "326527347": 0.07,
          "326527350": 0.74,
          "326527351": 0.1,
          "326527379": 0.03
        },
        "confidence": 0.68
      },
      "usage": {
        "input_tokens": 374,
        "output_tokens": 100,
        "cost": 0.000015708
      },
      "prediction": "326527350",
      "correct": true
    },
    {
      "case_id": "m05",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 337,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "186480",
        "probabilities": {
          "186463": 0.02,
          "186477": 0.01,
          "186480": 0.94,
          "186481": 0.03,
          "186509": 0
        },
        "confidence": 0.93
      },
      "usage": {
        "input_tokens": 357,
        "output_tokens": 82,
        "cost": 0.000014994
      },
      "prediction": "186480",
      "correct": true
    },
    {
      "case_id": "m05",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1012,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "186480",
        "probabilities": {
          "186463": 0,
          "186477": 0,
          "186480": 1,
          "186481": 0,
          "186509": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 472,
        "output_tokens": 17
      },
      "prediction": "186480",
      "correct": true
    },
    {
      "case_id": "m06",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1032,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "75",
        "probabilities": {
          "58": 0,
          "72": 0,
          "75": 1,
          "76": 0,
          "104": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 484,
        "output_tokens": 17
      },
      "prediction": "75",
      "correct": true
    },
    {
      "case_id": "m06",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 294,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "75",
        "probabilities": {
          "58": 0.18,
          "72": 0.12,
          "75": 0.27999999999999997,
          "76": 0.27,
          "104": 0.15
        },
        "confidence": 0.11
      },
      "usage": {
        "input_tokens": 360,
        "output_tokens": 59,
        "cost": 0.00001512
      },
      "prediction": "75",
      "correct": true
    },
    {
      "case_id": "m07",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 318,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "17576",
        "probabilities": {
          "17558": 0.06,
          "17572": 0.08,
          "17575": 0.1,
          "17576": 0.72,
          "17604": 0.04
        },
        "confidence": 0.65
      },
      "usage": {
        "input_tokens": 359,
        "output_tokens": 76,
        "cost": 0.000015078
      },
      "prediction": "17576",
      "correct": false
    },
    {
      "case_id": "m07",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1434,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "17558",
        "probabilities": {
          "17558": 0.2,
          "17572": 0.2,
          "17575": 0.2,
          "17576": 0.2,
          "17604": 0.2
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 474,
        "output_tokens": 43
      },
      "prediction": "17558",
      "correct": false
    },
    {
      "case_id": "m08",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 3026,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "12183",
        "probabilities": {
          "12183": 0.2,
          "12197": 0.2,
          "12200": 0.2,
          "12201": 0.2,
          "12229": 0.2
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 534,
        "output_tokens": 43
      },
      "prediction": "12183",
      "correct": false
    },
    {
      "case_id": "m08",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 375,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "12201",
        "probabilities": {
          "12183": 0.12,
          "12197": 0.31,
          "12200": 0.05,
          "12201": 0.37,
          "12229": 0.15
        },
        "confidence": 0.21
      },
      "usage": {
        "input_tokens": 414,
        "output_tokens": 76,
        "cost": 0.000017388
      },
      "prediction": "12201",
      "correct": false
    },
    {
      "case_id": "m09",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 333,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.42
      },
      "usage": {
        "input_tokens": 320,
        "output_tokens": 20,
        "cost": 0.00001344
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m09",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1122,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 419,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m10",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 885,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 417,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m10",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 273,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.3
      },
      "usage": {
        "input_tokens": 312,
        "output_tokens": 20,
        "cost": 0.000013104
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m11",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 349,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.3
      },
      "usage": {
        "input_tokens": 317,
        "output_tokens": 20,
        "cost": 0.000013314
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m11",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1026,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 423,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m12",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 936,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 433,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m12",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 263,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.42
      },
      "usage": {
        "input_tokens": 326,
        "output_tokens": 20,
        "cost": 0.000013692
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m13",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 260,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.32
      },
      "usage": {
        "input_tokens": 314,
        "output_tokens": 20,
        "cost": 0.000013188
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m13",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1135,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 418,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m14",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 998,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 427,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m14",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 279,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.26
      },
      "usage": {
        "input_tokens": 317,
        "output_tokens": 20,
        "cost": 0.000013314
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m15",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 259,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.44
      },
      "usage": {
        "input_tokens": 310,
        "output_tokens": 20,
        "cost": 0.00001302
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m15",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 900,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 417,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m16",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1103,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 726,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m16",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 271,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.34
      },
      "usage": {
        "input_tokens": 830,
        "output_tokens": 20,
        "cost": 0.00003486
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m17",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 277,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 5.38,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0,
          "1": 0.02,
          "2": 0.03,
          "3": 0.36,
          "4": 0.05,
          "5": 0.01,
          "6": 0.02,
          "7": 0.35,
          "8": 0.05,
          "9": 0.11
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 439,
        "output_tokens": 17,
        "cost": 0.000018438
      },
      "prediction": 5,
      "correct": false
    },
    {
      "case_id": "m17",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1046,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 3,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 1,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 567,
        "output_tokens": 17
      },
      "prediction": 3,
      "correct": true
    },
    {
      "case_id": "m18",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 1620,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 563,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m18",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 267,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 2.53,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.5,
          "1": 0.03,
          "2": 0.09,
          "3": 0.03,
          "4": 0.03,
          "5": 0.06,
          "6": 0.13,
          "7": 0.02,
          "8": 0.06,
          "9": 0.05
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 435,
        "output_tokens": 17,
        "cost": 0.00001827
      },
      "prediction": 3,
      "correct": false
    },
    {
      "case_id": "m19",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 253,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 3.93,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.21000000000000002,
          "1": 0.09,
          "2": 0.09,
          "3": 0.09,
          "4": 0.08,
          "5": 0.12,
          "6": 0.08,
          "7": 0.07,
          "8": 0.08,
          "9": 0.09
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 479,
        "output_tokens": 17,
        "cost": 0.000020118
      },
      "prediction": 4,
      "correct": true
    },
    {
      "case_id": "m19",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 923,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 604,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m20",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 874,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 701,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m20",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 330,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 2.91,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.42,
          "1": 0.04,
          "2": 0.09,
          "3": 0.04,
          "4": 0.06,
          "5": 0.11,
          "6": 0.08,
          "7": 0.04,
          "8": 0.06,
          "9": 0.06
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 571,
        "output_tokens": 17,
        "cost": 0.000023982
      },
      "prediction": 3,
      "correct": false
    },
    {
      "case_id": "m21",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 310,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.22,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.16,
          "1": 0.08,
          "2": 0.1,
          "3": 0.09,
          "4": 0.12,
          "5": 0.1,
          "6": 0.09,
          "7": 0.07,
          "8": 0.09,
          "9": 0.1
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 446,
        "output_tokens": 17,
        "cost": 0.000018732
      },
      "prediction": 4,
      "correct": false
    },
    {
      "case_id": "m21",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 933,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 574,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m22",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 845,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 572,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m22",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 289,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.35,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.09,
          "1": 0.07,
          "2": 0.07,
          "3": 0.22,
          "4": 0.09,
          "5": 0.12,
          "6": 0.08,
          "7": 0.11,
          "8": 0.08,
          "9": 0.07
        },
        "confidence": 0.09
      },
      "usage": {
        "input_tokens": 445,
        "output_tokens": 17,
        "cost": 0.00001869
      },
      "prediction": 4,
      "correct": true
    },
    {
      "case_id": "m23",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 273,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 3.7,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.27999999999999997,
          "1": 0.08,
          "2": 0.07,
          "3": 0.07,
          "4": 0.07,
          "5": 0.09,
          "6": 0.08,
          "7": 0.07,
          "8": 0.09,
          "9": 0.1
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 446,
        "output_tokens": 17,
        "cost": 0.000018732
      },
      "prediction": 4,
      "correct": false
    },
    {
      "case_id": "m23",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 927,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 6,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 1,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 576,
        "output_tokens": 17
      },
      "prediction": 6,
      "correct": false
    },
    {
      "case_id": "m24",
      "target": "jevify",
      "repeat": 1,
      "elapsed_ms": 910,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 6,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 1,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 590,
        "output_tokens": 17
      },
      "prediction": 6,
      "correct": false
    },
    {
      "case_id": "m24",
      "target": "jev",
      "repeat": 1,
      "elapsed_ms": 319,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.96,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.07,
          "1": 0.06,
          "2": 0.13,
          "3": 0.11,
          "4": 0.07,
          "5": 0.04,
          "6": 0.11,
          "7": 0.14,
          "8": 0.15000000000000002,
          "9": 0.12
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 461,
        "output_tokens": 17,
        "cost": 0.000019362
      },
      "prediction": 5,
      "correct": false
    },
    {
      "case_id": "m01",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1249,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "807",
        "probabilities": {
          "790": 0,
          "804": 0,
          "807": 1,
          "808": 0,
          "836": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 464,
        "output_tokens": 17
      },
      "prediction": "807",
      "correct": true
    },
    {
      "case_id": "m01",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 256,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "807",
        "probabilities": {
          "790": 0.01,
          "804": 0.06,
          "807": 0.82,
          "808": 0.08,
          "836": 0.03
        },
        "confidence": 0.79
      },
      "usage": {
        "input_tokens": 343,
        "output_tokens": 64,
        "cost": 0.000014406
      },
      "prediction": "807",
      "correct": true
    },
    {
      "case_id": "m02",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 302,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "15313",
        "probabilities": {
          "15295": 0.13,
          "15309": 0.28,
          "15312": 0.1,
          "15313": 0.38,
          "15341": 0.11
        },
        "confidence": 0.22
      },
      "usage": {
        "input_tokens": 379,
        "output_tokens": 76,
        "cost": 0.000015918
      },
      "prediction": "15313",
      "correct": false
    },
    {
      "case_id": "m02",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 893,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "15312",
        "probabilities": {
          "15295": 0,
          "15309": 0,
          "15312": 1,
          "15313": 0,
          "15341": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 489,
        "output_tokens": 17
      },
      "prediction": "15312",
      "correct": true
    },
    {
      "case_id": "m03",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 2470,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "98950079",
        "probabilities": {
          "98950079": 1,
          "98950093": 0,
          "98950096": 0,
          "98950097": 0,
          "98950125": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 475,
        "output_tokens": 17
      },
      "prediction": "98950079",
      "correct": false
    },
    {
      "case_id": "m03",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 316,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "98950097",
        "probabilities": {
          "98950079": 0.08,
          "98950093": 0.09,
          "98950096": 0.12,
          "98950097": 0.6900000000000001,
          "98950125": 0.02
        },
        "confidence": 0.59
      },
      "usage": {
        "input_tokens": 368,
        "output_tokens": 94,
        "cost": 0.000015456
      },
      "prediction": "98950097",
      "correct": false
    },
    {
      "case_id": "m04",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 253,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "326527350",
        "probabilities": {
          "326527333": 0.07,
          "326527347": 0.08,
          "326527350": 0.69,
          "326527351": 0.12,
          "326527379": 0.04
        },
        "confidence": 0.61
      },
      "usage": {
        "input_tokens": 374,
        "output_tokens": 100,
        "cost": 0.000015708
      },
      "prediction": "326527350",
      "correct": true
    },
    {
      "case_id": "m04",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 863,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "326527333",
        "probabilities": {
          "326527333": 1,
          "326527347": 0,
          "326527350": 0,
          "326527351": 0,
          "326527379": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 477,
        "output_tokens": 17
      },
      "prediction": "326527333",
      "correct": false
    },
    {
      "case_id": "m05",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1023,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "186480",
        "probabilities": {
          "186463": 0,
          "186477": 0,
          "186480": 1,
          "186481": 0,
          "186509": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 472,
        "output_tokens": 17
      },
      "prediction": "186480",
      "correct": true
    },
    {
      "case_id": "m05",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 320,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "186480",
        "probabilities": {
          "186463": 0.02,
          "186477": 0.01,
          "186480": 0.94,
          "186481": 0.03,
          "186509": 0
        },
        "confidence": 0.92
      },
      "usage": {
        "input_tokens": 357,
        "output_tokens": 82,
        "cost": 0.000014994
      },
      "prediction": "186480",
      "correct": true
    },
    {
      "case_id": "m06",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 262,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "75",
        "probabilities": {
          "58": 0.13,
          "72": 0.1,
          "75": 0.37,
          "76": 0.26,
          "104": 0.14
        },
        "confidence": 0.21
      },
      "usage": {
        "input_tokens": 360,
        "output_tokens": 59,
        "cost": 0.00001512
      },
      "prediction": "75",
      "correct": true
    },
    {
      "case_id": "m06",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1063,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "75",
        "probabilities": {
          "58": 0,
          "72": 0,
          "75": 1,
          "76": 0,
          "104": 0
        },
        "confidence": 1
      },
      "usage": {
        "input_tokens": 484,
        "output_tokens": 17
      },
      "prediction": "75",
      "correct": true
    },
    {
      "case_id": "m07",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1166,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "17558",
        "probabilities": {
          "17558": 0.2,
          "17572": 0.2,
          "17575": 0.2,
          "17576": 0.2,
          "17604": 0.2
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 474,
        "output_tokens": 43
      },
      "prediction": "17558",
      "correct": false
    },
    {
      "case_id": "m07",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 333,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "17576",
        "probabilities": {
          "17558": 0.06,
          "17572": 0.07,
          "17575": 0.1,
          "17576": 0.73,
          "17604": 0.04
        },
        "confidence": 0.68
      },
      "usage": {
        "input_tokens": 359,
        "output_tokens": 76,
        "cost": 0.000015078
      },
      "prediction": "17576",
      "correct": false
    },
    {
      "case_id": "m08",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 341,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "choice",
        "choice": "12201",
        "probabilities": {
          "12183": 0.12,
          "12197": 0.3,
          "12200": 0.06,
          "12201": 0.37,
          "12229": 0.15
        },
        "confidence": 0.21
      },
      "usage": {
        "input_tokens": 414,
        "output_tokens": 76,
        "cost": 0.000017388
      },
      "prediction": "12201",
      "correct": false
    },
    {
      "case_id": "m08",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1031,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "choice",
        "choice": "12183",
        "probabilities": {
          "12183": 0.2,
          "12197": 0.2,
          "12200": 0.2,
          "12201": 0.2,
          "12229": 0.2
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 534,
        "output_tokens": 43
      },
      "prediction": "12183",
      "correct": false
    },
    {
      "case_id": "m09",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 987,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 419,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m09",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 255,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.41
      },
      "usage": {
        "input_tokens": 320,
        "output_tokens": 20,
        "cost": 0.00001344
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m10",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 320,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.33
      },
      "usage": {
        "input_tokens": 312,
        "output_tokens": 20,
        "cost": 0.000013104
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m10",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 940,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 417,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m11",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 960,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 423,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m11",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 341,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.25
      },
      "usage": {
        "input_tokens": 317,
        "output_tokens": 20,
        "cost": 0.000013314
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m12",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 381,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.43
      },
      "usage": {
        "input_tokens": 326,
        "output_tokens": 20,
        "cost": 0.000013692
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m12",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1335,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 433,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m13",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 953,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0.5
      },
      "usage": {
        "input_tokens": 418,
        "output_tokens": 17
      },
      "prediction": true,
      "correct": true
    },
    {
      "case_id": "m13",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 283,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.35
      },
      "usage": {
        "input_tokens": 314,
        "output_tokens": 20,
        "cost": 0.000013188
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m14",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 266,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.25
      },
      "usage": {
        "input_tokens": 317,
        "output_tokens": 20,
        "cost": 0.000013314
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m14",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 846,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 427,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m15",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 816,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 417,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m15",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 261,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.47
      },
      "usage": {
        "input_tokens": 310,
        "output_tokens": 20,
        "cost": 0.00001302
      },
      "prediction": false,
      "correct": true
    },
    {
      "case_id": "m16",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 444,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "noul",
        "noul": 0.35
      },
      "usage": {
        "input_tokens": 830,
        "output_tokens": 20,
        "cost": 0.00003486
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m16",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 904,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "noul",
        "noul": 0
      },
      "usage": {
        "input_tokens": 726,
        "output_tokens": 15
      },
      "prediction": false,
      "correct": false
    },
    {
      "case_id": "m17",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 838,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 3,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 1,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 567,
        "output_tokens": 17
      },
      "prediction": 3,
      "correct": true
    },
    {
      "case_id": "m17",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 291,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 5.47,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0,
          "1": 0.02,
          "2": 0.02,
          "3": 0.35,
          "4": 0.05,
          "5": 0.01,
          "6": 0.02,
          "7": 0.36,
          "8": 0.06,
          "9": 0.11
        },
        "confidence": 0.16
      },
      "usage": {
        "input_tokens": 439,
        "output_tokens": 17,
        "cost": 0.000018438
      },
      "prediction": 5,
      "correct": false
    },
    {
      "case_id": "m18",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 280,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 2.25,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.54,
          "1": 0.03,
          "2": 0.1,
          "3": 0.02,
          "4": 0.03,
          "5": 0.06,
          "6": 0.12,
          "7": 0.02,
          "8": 0.04,
          "9": 0.04
        },
        "confidence": 0.1
      },
      "usage": {
        "input_tokens": 435,
        "output_tokens": 17,
        "cost": 0.00001827
      },
      "prediction": 2,
      "correct": false
    },
    {
      "case_id": "m18",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1098,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 563,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m19",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 872,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 604,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m19",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 293,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 3.89,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.21,
          "1": 0.09,
          "2": 0.09,
          "3": 0.09,
          "4": 0.08,
          "5": 0.12,
          "6": 0.08,
          "7": 0.07,
          "8": 0.08,
          "9": 0.09
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 479,
        "output_tokens": 17,
        "cost": 0.000020118
      },
      "prediction": 4,
      "correct": true
    },
    {
      "case_id": "m20",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 300,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 2.93,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.41,
          "1": 0.04,
          "2": 0.09,
          "3": 0.04,
          "4": 0.07,
          "5": 0.11,
          "6": 0.07,
          "7": 0.05,
          "8": 0.07,
          "9": 0.05
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 571,
        "output_tokens": 17,
        "cost": 0.000023982
      },
      "prediction": 3,
      "correct": false
    },
    {
      "case_id": "m20",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1012,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 701,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m21",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 879,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 574,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m21",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 268,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.29,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.16,
          "1": 0.07,
          "2": 0.1,
          "3": 0.09,
          "4": 0.11,
          "5": 0.1,
          "6": 0.09,
          "7": 0.08,
          "8": 0.1,
          "9": 0.1
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 446,
        "output_tokens": 17,
        "cost": 0.000018732
      },
      "prediction": 4,
      "correct": false
    },
    {
      "case_id": "m22",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 350,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.46,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.08,
          "1": 0.07,
          "2": 0.06,
          "3": 0.23,
          "4": 0.08,
          "5": 0.13,
          "6": 0.08,
          "7": 0.12,
          "8": 0.08,
          "9": 0.07
        },
        "confidence": 0.06
      },
      "usage": {
        "input_tokens": 445,
        "output_tokens": 17,
        "cost": 0.00001869
      },
      "prediction": 4,
      "correct": true
    },
    {
      "case_id": "m22",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 1064,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 0,
        "confidence": 1,
        "probabilities": {
          "0": 1,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 0,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 572,
        "output_tokens": 17
      },
      "prediction": 0,
      "correct": false
    },
    {
      "case_id": "m23",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 877,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 6,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 1,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 576,
        "output_tokens": 17
      },
      "prediction": 6,
      "correct": false
    },
    {
      "case_id": "m23",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 310,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 3.67,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.29,
          "1": 0.08,
          "2": 0.07,
          "3": 0.07,
          "4": 0.07,
          "5": 0.09,
          "6": 0.08,
          "7": 0.07,
          "8": 0.08,
          "9": 0.1
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 446,
        "output_tokens": 17,
        "cost": 0.000018732
      },
      "prediction": 4,
      "correct": false
    },
    {
      "case_id": "m24",
      "target": "jev",
      "repeat": 2,
      "elapsed_ms": 259,
      "status": 200,
      "model": "typesafe/jev-1.13-20260917",
      "answer": {
        "type": "score",
        "score": 4.89,
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        },
        "probabilities": {
          "0": 0.08,
          "1": 0.07,
          "2": 0.13,
          "3": 0.11,
          "4": 0.07,
          "5": 0.04,
          "6": 0.1,
          "7": 0.13,
          "8": 0.15000000000000002,
          "9": 0.12
        },
        "confidence": 0
      },
      "usage": {
        "input_tokens": 461,
        "output_tokens": 17,
        "cost": 0.000019362
      },
      "prediction": 5,
      "correct": false
    },
    {
      "case_id": "m24",
      "target": "jevify",
      "repeat": 2,
      "elapsed_ms": 928,
      "status": 200,
      "model": "deepseek-v4.1-flash",
      "answer": {
        "type": "score",
        "score": 6,
        "confidence": 1,
        "probabilities": {
          "0": 0,
          "1": 0,
          "2": 0,
          "3": 0,
          "4": 0,
          "5": 0,
          "6": 1,
          "7": 0,
          "8": 0,
          "9": 0
        },
        "legend": {
          "0": "The answer is the digit 0",
          "1": "The answer is the digit 1",
          "2": "The answer is the digit 2",
          "3": "The answer is the digit 3",
          "4": "The answer is the digit 4",
          "5": "The answer is the digit 5",
          "6": "The answer is the digit 6",
          "7": "The answer is the digit 7",
          "8": "The answer is the digit 8",
          "9": "The answer is the digit 9"
        }
      },
      "usage": {
        "input_tokens": 590,
        "output_tokens": 17
      },
      "prediction": 6,
      "correct": false
    }
  ]
}