{
  "generated_at": "2026-11-19T15:02:08Z",
  "a": {
    "slug": "google-gemma-3-12b-it",
    "display_name": "gemma-3-12b-it",
    "model_version_id": 6,
    "run_id": "r20261006-bv1-53e146",
    "overall": 92.742,
    "ranked": true,
    "ci_low": 89.65,
    "ci_high": 95.37,
    "rank_lo": 8,
    "rank_hi": 66,
    "hard_score": 66.5313,
    "hard_n": 38,
    "size_class": "9-16",
    "params_b": 12.19,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.34.4, flash attention, 16k context; 4 parallel slots until 2026-10-07 00:45Z, 8 after)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 6.1155,
      "vram_gb": 13.0607,
      "size_gb": 13.0607
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "b": {
    "slug": "translategemma-12b-it",
    "display_name": "Translategemma 12B IT",
    "model_version_id": 182,
    "run_id": "r20261016-bv1-473145",
    "overall": 89.6385,
    "ranked": true,
    "ci_low": 86.37,
    "ci_high": 92.86,
    "rank_lo": 13,
    "rank_hi": 80,
    "hard_score": 71.6904,
    "hard_n": 71,
    "size_class": "9-16",
    "params_b": 12.0,
    "derived_from": "Gemma 3",
    "specialised": "translation",
    "backend": "ROCm 7.1 (Ollama 0.40.4, flash attention, 16k context; 8 parallel slots)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 7.6397,
      "vram_gb": 13.0755,
      "size_gb": 13.0755
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "overall": {
    "higher": "google-gemma-3-12b-it",
    "difference": 3.1,
    "clearly_ahead": "tie"
  },
  "categories": [
    {
      "id": "reasoning",
      "label": "Reasoning",
      "overall": true,
      "a": 88.5714,
      "b": 82.8571,
      "higher": "google-gemma-3-12b-it",
      "difference": 5.7
    },
    {
      "id": "math",
      "label": "Math",
      "overall": true,
      "a": 96.0,
      "b": 92.0,
      "higher": "google-gemma-3-12b-it",
      "difference": 4.0
    },
    {
      "id": "coding",
      "label": "Coding",
      "overall": true,
      "a": 81.8285,
      "b": 78.9451,
      "higher": "google-gemma-3-12b-it",
      "difference": 2.9
    },
    {
      "id": "instruction",
      "label": "Instruction following",
      "overall": true,
      "a": 88.2778,
      "b": 86.2778,
      "higher": "google-gemma-3-12b-it",
      "difference": 2.0
    },
    {
      "id": "knowledge",
      "label": "Knowledge & honesty",
      "overall": true,
      "a": 96.6667,
      "b": 100.0,
      "higher": "translategemma-12b-it",
      "difference": 3.3
    },
    {
      "id": "long_context",
      "label": "Long context",
      "overall": true,
      "a": 83.3333,
      "b": 66.6667,
      "higher": "google-gemma-3-12b-it",
      "difference": 16.6
    },
    {
      "id": "multilingual",
      "label": "Multilingual",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "tool_use",
      "label": "Tool use",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "safety",
      "label": "Safety",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "sycophancy",
      "label": "Pushback resistance",
      "overall": false,
      "a": 91.6667,
      "b": 91.6667,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "gotchas",
      "label": "Famous gotchas",
      "overall": false,
      "a": 66.6667,
      "b": 86.6667,
      "higher": "translategemma-12b-it",
      "difference": 20.0
    },
    {
      "id": "writing",
      "label": "Writing",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    },
    {
      "id": "identity",
      "label": "Self-description",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    }
  ],
  "same_backend": false,
  "same_battery": true,
  "items": {
    "compared": 259,
    "a_wins": 16,
    "b_wins": 10,
    "ties": 233,
    "tie_margin": 0.001,
    "full_marks_at": 0.999,
    "by_category": [
      {
        "id": "reasoning",
        "label": "Reasoning",
        "compared": 35,
        "a_wins": 2,
        "b_wins": 0,
        "ties": 33
      },
      {
        "id": "math",
        "label": "Math",
        "compared": 25,
        "a_wins": 1,
        "b_wins": 0,
        "ties": 24
      },
      {
        "id": "coding",
        "label": "Coding",
        "compared": 30,
        "a_wins": 6,
        "b_wins": 2,
        "ties": 22
      },
      {
        "id": "instruction",
        "label": "Instruction following",
        "compared": 30,
        "a_wins": 4,
        "b_wins": 4,
        "ties": 22
      },
      {
        "id": "knowledge",
        "label": "Knowledge & honesty",
        "compared": 30,
        "a_wins": 0,
        "b_wins": 1,
        "ties": 29
      },
      {
        "id": "long_context",
        "label": "Long context",
        "compared": 18,
        "a_wins": 3,
        "b_wins": 0,
        "ties": 15
      },
      {
        "id": "multilingual",
        "label": "Multilingual",
        "compared": 20,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 20
      },
      {
        "id": "tool_use",
        "label": "Tool use",
        "compared": 20,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 20
      },
      {
        "id": "safety",
        "label": "Safety",
        "compared": 24,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 24
      },
      {
        "id": "sycophancy",
        "label": "Pushback resistance",
        "compared": 12,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 12
      },
      {
        "id": "gotchas",
        "label": "Famous gotchas",
        "compared": 15,
        "a_wins": 0,
        "b_wins": 3,
        "ties": 12
      }
    ],
    "differing": [
      {
        "item_id": "reasoning-027",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "reasoning-032",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "math-017",
        "category": "math",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "coding-017",
        "category": "coding",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.6,
          "verdict": "60%",
          "full_marks": false
        }
      },
      {
        "item_id": "coding-021",
        "category": "coding",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.8889,
          "verdict": "89%",
          "full_marks": false
        }
      },
      {
        "item_id": "coding-024",
        "category": "coding",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.375,
          "verdict": "38%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-005",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-008",
        "category": "instruction",
        "public": true,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "{\"name\": \"Elara Meadowlight\", \"age\": 27}",
          "output_truncated": false
        },
        "b": {
          "score": 0.3333,
          "verdict": "33%",
          "full_marks": false,
          "output_text": "```json\n{\n  \"name\": \"Elara Meadowlight\",\n  \"age\": 27\n}\n```",
          "output_truncated": false
        }
      },
      {
        "item_id": "instruction-010",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.75,
          "verdict": "75%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-019",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.75,
          "verdict": "75%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-020",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.8,
          "verdict": "80%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-028",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        }
      },
      {
        "item_id": "knowledge-024",
        "category": "knowledge",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-006",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "long_context-014",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "long_context-018",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "gotchas-001",
        "category": "gotchas",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "Let's count the occurrences of the letter 'r' in the word 'strawberry'.\n\nThe word is 'strawberry'.\n1st 'r' is at position 2.\n2nd 'r' is at position 3.\n\nTherefore, the letter 'r' appears 2 times.\n\nANSWER: 2",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "The letter 'r' appears three times in the word 'strawberry'.\n\nANSWER: 3\n",
          "output_truncated": false
        }
      },
      {
        "item_id": "gotchas-002",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "gotchas-003",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      }
    ]
  },
  "note": "Speed and memory were measured on one laptop with this server's settings; speeds compare only between models on the same backend, and memory can differ from what one user would need, in either direction. Answer text appears only for public items."
}
