{
  "generated_at": "2026-11-19T15:02:09Z",
  "a": {
    "slug": "qwen-qwen3-14b",
    "display_name": "Qwen3-14B",
    "model_version_id": 44,
    "run_id": "r20261006-bv1-53e146",
    "overall": 91.707,
    "ranked": true,
    "ci_low": 88.48,
    "ci_high": 94.73,
    "rank_lo": 11,
    "rank_hi": 71,
    "hard_score": 52.9304,
    "hard_n": 38,
    "size_class": "9-16",
    "params_b": 14.77,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.34.4, flash attention, 16k context; 4 parallel slots until 2026-10-07 00:45Z, 8 after)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 4.9961,
      "vram_gb": 30.9283,
      "size_gb": 30.9283
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "b": {
    "slug": "qwen-qwen3-32b",
    "display_name": "Qwen3-32B",
    "model_version_id": 47,
    "run_id": "r20261006-bv1-53e146",
    "overall": 93.6304,
    "ranked": true,
    "ci_low": 90.91,
    "ci_high": 96.3,
    "rank_lo": 5,
    "rank_hi": 58,
    "hard_score": 64.7483,
    "hard_n": 38,
    "size_class": "16-35",
    "params_b": 32.76,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.34.4, flash attention, 16k context; 4 parallel slots until 2026-10-07 00:45Z, 8 after)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 2.2793,
      "vram_gb": 54.9647,
      "size_gb": 54.9647
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "overall": {
    "higher": "qwen-qwen3-32b",
    "difference": 1.9,
    "clearly_ahead": "tie"
  },
  "categories": [
    {
      "id": "reasoning",
      "label": "Reasoning",
      "overall": true,
      "a": 91.4286,
      "b": 85.7143,
      "higher": "qwen-qwen3-14b",
      "difference": 5.7
    },
    {
      "id": "math",
      "label": "Math",
      "overall": true,
      "a": 88.0,
      "b": 88.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "coding",
      "label": "Coding",
      "overall": true,
      "a": 73.4897,
      "b": 83.7924,
      "higher": "qwen-qwen3-32b",
      "difference": 10.3
    },
    {
      "id": "instruction",
      "label": "Instruction following",
      "overall": true,
      "a": 93.5556,
      "b": 96.2778,
      "higher": "qwen-qwen3-32b",
      "difference": 2.7
    },
    {
      "id": "knowledge",
      "label": "Knowledge & honesty",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "long_context",
      "label": "Long context",
      "overall": true,
      "a": 88.8889,
      "b": 88.8889,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "multilingual",
      "label": "Multilingual",
      "overall": true,
      "a": 90.0,
      "b": 100.0,
      "higher": "qwen-qwen3-32b",
      "difference": 10.0
    },
    {
      "id": "tool_use",
      "label": "Tool use",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "safety",
      "label": "Safety",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "sycophancy",
      "label": "Pushback resistance",
      "overall": false,
      "a": 91.6667,
      "b": 100.0,
      "higher": "qwen-qwen3-32b",
      "difference": 8.3
    },
    {
      "id": "gotchas",
      "label": "Famous gotchas",
      "overall": false,
      "a": 80.0,
      "b": 86.6667,
      "higher": "qwen-qwen3-32b",
      "difference": 6.7
    },
    {
      "id": "writing",
      "label": "Writing",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    },
    {
      "id": "identity",
      "label": "Self-description",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    }
  ],
  "same_backend": true,
  "same_battery": true,
  "items": {
    "compared": 259,
    "a_wins": 13,
    "b_wins": 22,
    "ties": 224,
    "tie_margin": 0.001,
    "full_marks_at": 0.999,
    "by_category": [
      {
        "id": "reasoning",
        "label": "Reasoning",
        "compared": 35,
        "a_wins": 2,
        "b_wins": 0,
        "ties": 33
      },
      {
        "id": "math",
        "label": "Math",
        "compared": 25,
        "a_wins": 2,
        "b_wins": 2,
        "ties": 21
      },
      {
        "id": "coding",
        "label": "Coding",
        "compared": 30,
        "a_wins": 5,
        "b_wins": 10,
        "ties": 15
      },
      {
        "id": "instruction",
        "label": "Instruction following",
        "compared": 30,
        "a_wins": 3,
        "b_wins": 5,
        "ties": 22
      },
      {
        "id": "knowledge",
        "label": "Knowledge & honesty",
        "compared": 30,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 30
      },
      {
        "id": "long_context",
        "label": "Long context",
        "compared": 18,
        "a_wins": 1,
        "b_wins": 1,
        "ties": 16
      },
      {
        "id": "multilingual",
        "label": "Multilingual",
        "compared": 20,
        "a_wins": 0,
        "b_wins": 2,
        "ties": 18
      },
      {
        "id": "tool_use",
        "label": "Tool use",
        "compared": 20,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 20
      },
      {
        "id": "safety",
        "label": "Safety",
        "compared": 24,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 24
      },
      {
        "id": "sycophancy",
        "label": "Pushback resistance",
        "compared": 12,
        "a_wins": 0,
        "b_wins": 1,
        "ties": 11
      },
      {
        "id": "gotchas",
        "label": "Famous gotchas",
        "compared": 15,
        "a_wins": 0,
        "b_wins": 1,
        "ties": 14
      }
    ],
    "differing": [
      {
        "item_id": "reasoning-032",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "reasoning-033",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "math-012",
        "category": "math",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "math-017",
        "category": "math",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "math-022",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-023",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-021",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.3333,
          "verdict": "33%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-023",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.8889,
          "verdict": "89%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-024",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.625,
          "verdict": "63%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-025",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.2222,
          "verdict": "22%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-028",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-029",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-005",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-018",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.8,
          "verdict": "80%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-019",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.75,
          "verdict": "75%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-021",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-025",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-028",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-029",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-005",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-014",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "multilingual-004",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "multilingual-005",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "sycophancy-002",
        "category": "sycophancy",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "gotchas-007",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      }
    ]
  },
  "note": "Speed and memory were measured on one laptop with this server's settings; speeds compare only between models on the same backend, and memory can differ from what one user would need, in either direction. Answer text appears only for public items."
}
