{
  "generated_at": "2026-11-19T15:02:05Z",
  "a": {
    "slug": "gemma3n-e2b-it",
    "display_name": "Gemma3n E2B IT",
    "model_version_id": 104,
    "run_id": "r20261016-bv1-473145",
    "overall": 79.8536,
    "ranked": true,
    "ci_low": 75.22,
    "ci_high": 84.66,
    "rank_lo": 64,
    "rank_hi": 127,
    "hard_score": 55.7108,
    "hard_n": 71,
    "size_class": "4-9",
    "params_b": 5.0,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.40.4, flash attention, 16k context; 8 parallel slots)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 13.816,
      "vram_gb": 3.3972,
      "size_gb": 3.3972
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "b": {
    "slug": "google-gemma-3n-e4b-it",
    "display_name": "Gemma 3n E4B IT",
    "model_version_id": 9,
    "run_id": "r20261006-bv1-53e146",
    "overall": 86.1364,
    "ranked": true,
    "ci_low": 82.37,
    "ci_high": 89.57,
    "rank_lo": 44,
    "rank_hi": 97,
    "hard_score": 50.7309,
    "hard_n": 38,
    "size_class": "4-9",
    "params_b": 8.0,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.34.4, flash attention, 16k context; 4 parallel slots until 2026-10-07 00:45Z, 8 after)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 12.0698,
      "vram_gb": 4.2095,
      "size_gb": 4.2095
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "overall": {
    "higher": "google-gemma-3n-e4b-it",
    "difference": 6.2,
    "clearly_ahead": "tie"
  },
  "categories": [
    {
      "id": "reasoning",
      "label": "Reasoning",
      "overall": true,
      "a": 68.5714,
      "b": 82.8571,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 14.3
    },
    {
      "id": "math",
      "label": "Math",
      "overall": true,
      "a": 76.0,
      "b": 92.0,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 16.0
    },
    {
      "id": "coding",
      "label": "Coding",
      "overall": true,
      "a": 61.3886,
      "b": 65.2591,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 3.9
    },
    {
      "id": "instruction",
      "label": "Instruction following",
      "overall": true,
      "a": 89.6667,
      "b": 83.4444,
      "higher": "gemma3n-e2b-it",
      "difference": 6.3
    },
    {
      "id": "knowledge",
      "label": "Knowledge & honesty",
      "overall": true,
      "a": 83.3333,
      "b": 96.6667,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 13.4
    },
    {
      "id": "long_context",
      "label": "Long context",
      "overall": true,
      "a": 72.2222,
      "b": 66.6667,
      "higher": "gemma3n-e2b-it",
      "difference": 5.5
    },
    {
      "id": "multilingual",
      "label": "Multilingual",
      "overall": true,
      "a": 80.0,
      "b": 93.3333,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 13.3
    },
    {
      "id": "tool_use",
      "label": "Tool use",
      "overall": true,
      "a": 87.5,
      "b": 95.0,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 7.5
    },
    {
      "id": "safety",
      "label": "Safety",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "sycophancy",
      "label": "Pushback resistance",
      "overall": false,
      "a": 91.6667,
      "b": 91.6667,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "gotchas",
      "label": "Famous gotchas",
      "overall": false,
      "a": 53.3333,
      "b": 73.3333,
      "higher": "google-gemma-3n-e4b-it",
      "difference": 20.0
    },
    {
      "id": "writing",
      "label": "Writing",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    },
    {
      "id": "identity",
      "label": "Self-description",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    }
  ],
  "same_backend": false,
  "same_battery": true,
  "items": {
    "compared": 259,
    "a_wins": 16,
    "b_wins": 36,
    "ties": 207,
    "tie_margin": 0.001,
    "full_marks_at": 0.999,
    "by_category": [
      {
        "id": "reasoning",
        "label": "Reasoning",
        "compared": 35,
        "a_wins": 0,
        "b_wins": 5,
        "ties": 30
      },
      {
        "id": "math",
        "label": "Math",
        "compared": 25,
        "a_wins": 0,
        "b_wins": 4,
        "ties": 21
      },
      {
        "id": "coding",
        "label": "Coding",
        "compared": 30,
        "a_wins": 4,
        "b_wins": 9,
        "ties": 17
      },
      {
        "id": "instruction",
        "label": "Instruction following",
        "compared": 30,
        "a_wins": 5,
        "b_wins": 2,
        "ties": 23
      },
      {
        "id": "knowledge",
        "label": "Knowledge & honesty",
        "compared": 30,
        "a_wins": 0,
        "b_wins": 4,
        "ties": 26
      },
      {
        "id": "long_context",
        "label": "Long context",
        "compared": 18,
        "a_wins": 3,
        "b_wins": 2,
        "ties": 13
      },
      {
        "id": "multilingual",
        "label": "Multilingual",
        "compared": 20,
        "a_wins": 1,
        "b_wins": 3,
        "ties": 16
      },
      {
        "id": "tool_use",
        "label": "Tool use",
        "compared": 20,
        "a_wins": 2,
        "b_wins": 3,
        "ties": 15
      },
      {
        "id": "safety",
        "label": "Safety",
        "compared": 24,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 24
      },
      {
        "id": "sycophancy",
        "label": "Pushback resistance",
        "compared": 12,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 12
      },
      {
        "id": "gotchas",
        "label": "Famous gotchas",
        "compared": 15,
        "a_wins": 1,
        "b_wins": 4,
        "ties": 10
      }
    ],
    "differing": [
      {
        "item_id": "reasoning-004",
        "category": "reasoning",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "The movie starts at 19:45.\nThe movie runs for 2 hours 20 minutes.\nWe need to add 2 hours 20 minutes to 19:45.\n\n19:45 + 2 hours = 21:45\n21:45 + 20 minutes = 21:45 + 20 minutes = 21:65\nSince there are 60 minutes in an hour, we can write 65 minutes as 1 hour and 5 minutes.\nSo, 21:65 = 21:00 + 1 hour + 5 minutes = 22:05.\n\nAlternatively, we can add 2 hours and 20 minutes to 19:45:\n19:45 + 2 hours = 21:45\n21:45 + 20 minutes = 21:45 + 20 minutes = 21:65\nSince 65 minutes is 1 hour and 5 minutes, we can add 1 hour to 21:45, which gives us 22:45.\nThen, we add 5 minutes to 22:45, which gives us 22:50.\nHowever, we need to consider that 65 minutes is 1 hour and 5 minutes.\nSo, 19:45 + 2 hours 20 minutes = 19:45 + 2 hours + 20 minutes = 21:45 + 20 minutes = 21:65.\nSince 65 minutes is 1 hour and 5 minutes, we can add 1 hour to 21:45, which gives us 22:45.\nThen, we add 5 minutes to 22:45, which gives us 22:50.\nHowever, we can also calculate the total minutes:\n19:45 is 19 * 60 + 45 = 1140 + 45 = 1185 minutes.\nAdding 2 hours 20 minutes, we get 1185 + 2 * 60 + 20 = 1185 + 120 + 20 = 1185 + 140 = 1325 minutes.\nNow, we convert 1325 minutes to hours and minutes:\n1325 / 60 = 22 with a remainder of 5.\nSo, 1325 minutes is 22 hours and 5 minutes.\nSince the movie starts at 19:45, the movie ends at 19:45 + 2 hours 20 minutes = 21:45 + 20 minutes = 21:65.\nSince 65 minutes is 1 hour and 5 minutes, we add 1 hour to 21:45, which gives us 22:45.\nThen, we add 5 minutes to 22:45, which gives us 22:50.\nHowever, we can also calculate the end time as follows:\n19:45 + 2 hours 20 minutes = 19:45 + 2:20 = 21:65.\nSince 65 minutes is 1 hour and 5 minutes, we can add 1 hour to 21:65, which gives us 22:65.\nSince 65 minutes is 1 hour and 5 minutes, we can add 1 hour and 5 minutes to 22:65, which gives us 22:65 + 1:5 = 22:70.\nSince 70 minutes is 1 hour and 10 minutes, we can add 1 hour and 10 minutes to 22:70, which gives us 22:70 + 1:10 = 23:80.\nSince 80 minutes is 1 hour and 20 minutes, we can add 1 hour and 20",
          "output_truncated": true
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "The movie starts at 19:45.\nThe movie runs for 2 hours 20 minutes.\nWe need to add 2 hours 20 minutes to 19:45.\n19:45 + 2 hours = 21:45\n21:45 + 20 minutes = 21:65\nSince there are 60 minutes in an hour, we can rewrite 65 minutes as 1 hour and 5 minutes.\nSo, 21:65 = 21:00 + 1 hour 5 minutes = 22:05\nAlternatively, we can add 2 hours and 20 minutes to 19:45:\n19:45 + 2 hours 20 minutes = (19 + 2) hours + (45 + 20) minutes\n= 21 hours + 65 minutes\nSince 65 minutes = 1 hour 5 minutes, we have\n21 hours + 1 hour 5 minutes = 22 hours 5 minutes\nSo the movie ends at 22:05.\n\nFinal Answer: The final answer is $\\boxed{22:05}$",
          "output_truncated": false
        }
      },
      {
        "item_id": "reasoning-006",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "reasoning-015",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "reasoning-022",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "reasoning-025",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-008",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-018",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-023",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-024",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-002",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.7143,
          "verdict": "71%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-014",
        "category": "coding",
        "public": true,
        "a": {
          "score": 0.4286,
          "verdict": "43%",
          "full_marks": false,
          "output_text": "```python\ndef reverse_words(s: str) -> str:\n    s = s.strip()\n    s = ' '.join(s.split())\n    return s[::-1]\n```",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "```python\ndef reverse_words(s: str) -> str:\n▁▁▁▁\"\"\"Reverses the order of words in a string.\n\n▁▁▁▁Words are separated by whitespace; collapse any run of whitespace\n▁▁▁▁between words to a single space and strip leading/trailing whitespace.\n\n▁▁▁▁Examples:\n▁▁▁▁reverse_words(\" the sky▁▁is blue \") -> \"blue is sky the\"\n▁▁▁▁reverse_words(\"hello\") -> \"hello\"\n▁▁▁▁\"\"\"\n▁▁▁▁words = s.split()\n▁▁▁▁words = [word for word in words if word]\n▁▁▁▁words.reverse()\n▁▁▁▁return \" \".join(words)\n```",
          "output_truncated": false
        }
      },
      {
        "item_id": "coding-024",
        "category": "coding",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.375,
          "verdict": "38%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-008",
        "category": "instruction",
        "public": true,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "{\"name\": \"Elara Vance\", \"age\": 28}\n",
          "output_truncated": false
        },
        "b": {
          "score": 0.3333,
          "verdict": "33%",
          "full_marks": false,
          "output_text": "```json\n{\n▁▁\"name\": \"Elara Vance\",\n▁▁\"age\": 28\n}\n```",
          "output_truncated": false
        }
      },
      {
        "item_id": "instruction-017",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-022",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-024",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.3333,
          "verdict": "33%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-026",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-029",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        }
      },
      {
        "item_id": "knowledge-007",
        "category": "knowledge",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "Andrew Wiles published the accepted proof of Fermat's Last Theorem in 1995. While he presented the proof in 1994, it was published in the Journal of the Mathematical Society of Japan in 1995.\n\nANSWER: 1995",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "Andrew Wiles published the accepted proof of Fermat's Last Theorem in 1995.\n\nANSWER: Andrew Wiles",
          "output_truncated": false
        }
      },
      {
        "item_id": "knowledge-010",
        "category": "knowledge",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "knowledge-015",
        "category": "knowledge",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "knowledge-020",
        "category": "knowledge",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-005",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-006",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "long_context-008",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "long_context-010",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-016",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "multilingual-003",
        "category": "multilingual",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "Let's calculate the distance Anna travels in the first part of her ride.\nTime = 2 hours\nSpeed = 18 km/h\nDistance = Speed * Time = 18 km/h * 2 h = 36 km\n\nNext, she rides for 90 minutes with a speed of 12 km/h.\nTime = 90 minutes = 1.5 hours\nSpeed = 12 km/h\nDistance = Speed * Time = 12 km/h * 1.5 h = 18 km\n\nTotal distance traveled so far = 36 km + 18 km = 54 km\n\nShe then pushes her bicycle 3 km. This doesn't add to the distance she has traveled by bike.\n\nThe total distance of the route is 100 km.\nDistance remaining = Total distance - Distance traveled = 100 km - 54 km = 46 km\n\nTherefore, Anna still has 46 km to go.\n\nFinal Answer: The final answer is 46",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "First, let's calculate the distance Anna travels during the first 2 hours.\nDistance = Speed × Time\nDistance = 18 km/h × 2 h = 36 km\n\nNext, let's calculate the distance Anna travels during the next 90 minutes.\n90 minutes = 1.5 hours\nDistance = Speed × Time\nDistance = 12 km/h × 1.5 h = 18 km\n\nNow, let's calculate the total distance Anna has traveled so far.\nTotal distance = 36 km + 18 km = 54 km\n\nAnna then pushes her bike for 3 km.\nTotal distance traveled = 54 km + 3 km = 57 km\n\nThe total distance of the route is 100 km.\nRemaining distance = Total distance - Distance traveled\nRemaining distance = 100 km - 57 km = 43 km\n\nTherefore, Anna has 43 km left to travel to reach her destination.\n\nFinal Answer: The final answer is 43",
          "output_truncated": false
        }
      },
      {
        "item_id": "multilingual-004",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "multilingual-008",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "multilingual-020",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        }
      },
      {
        "item_id": "tool_use-003",
        "category": "tool_use",
        "public": true,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "{\"name\": \"create_calendar_event\", \"arguments\": {\"title\": \"Quarterly planning\", \"date\": \"2026-11-03\", \"start_time\": \"14:30\", \"duration_minutes\": 90}}",
          "output_truncated": false
        },
        "b": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false,
          "output_text": "{\"name\": \"create_calendar_event\", \"arguments\": {\"title\": \"Quaertlry planning\", \"date\": \"2026-11-03\", \"start_time\": \"14:30\", \"duration_minutes\": 90}}",
          "output_truncated": false
        }
      },
      {
        "item_id": "tool_use-004",
        "category": "tool_use",
        "public": false,
        "a": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "tool_use-006",
        "category": "tool_use",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        }
      },
      {
        "item_id": "tool_use-018",
        "category": "tool_use",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "I can't directly answer that question. I need to search my files for information about France.\n\n```json\n{\"name\": \"search_files\", \"arguments\": {\"query\": \"capital of France\", \"file_type\": \"any\"}}\n```",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "The capital of France is Paris.",
          "output_truncated": false
        }
      },
      {
        "item_id": "tool_use-020",
        "category": "tool_use",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "gotchas-001",
        "category": "gotchas",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "The word \"strawberry\" has 2 'r's.\n\nANSWER: 2\n",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "The word \"strawberry\" has the following letters: s, t, r, a, w, b, e, r, r, y.\nThe letter 'r' appears 3 times in the word \"strawberry\".\n\nANSWER: 3",
          "output_truncated": false
        }
      },
      {
        "item_id": "gotchas-002",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "gotchas-009",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "gotchas-012",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "gotchas-015",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      }
    ]
  },
  "note": "Speed and memory were measured on one laptop with this server's settings; speeds compare only between models on the same backend, and memory can differ from what one user would need, in either direction. Answer text appears only for public items."
}
