{
  "generated_at": "2026-11-19T15:02:09Z",
  "a": {
    "slug": "qwen-qwen3-4b",
    "display_name": "Qwen3-4B",
    "model_version_id": 48,
    "run_id": "r20261006-bv1-53e146",
    "overall": 89.1828,
    "ranked": true,
    "ci_low": 85.55,
    "ci_high": 92.43,
    "rank_lo": 18,
    "rank_hi": 81,
    "hard_score": 58.0533,
    "hard_n": 38,
    "size_class": "4-9",
    "params_b": 4.02,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.34.4, flash attention, 16k context; 4 parallel slots until 2026-10-07 00:45Z, 8 after)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 16.9503,
      "vram_gb": 12.6602,
      "size_gb": 12.6602
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "b": {
    "slug": "qwen-qwen3-8b",
    "display_name": "Qwen3-8B",
    "model_version_id": 57,
    "run_id": "r20261006-bv1-53e146",
    "overall": 91.5462,
    "ranked": true,
    "ci_low": 88.37,
    "ci_high": 94.5,
    "rank_lo": 11,
    "rank_hi": 72,
    "hard_score": 55.4539,
    "hard_n": 38,
    "size_class": "4-9",
    "params_b": 8.19,
    "derived_from": null,
    "specialised": null,
    "backend": "ROCm 7.1 (Ollama 0.34.4, flash attention, 16k context; 4 parallel slots until 2026-10-07 00:45Z, 8 after)",
    "gpu": "AMD Radeon 8060S (Ryzen AI Max+ 395, 128 GB unified, ~78 GB GPU)",
    "withheld": null,
    "perf": {
      "decode_tps": 9.2875,
      "vram_gb": 24.6665,
      "size_gb": 24.6665
    },
    "perf_units": {
      "decode_tps": "tok/s",
      "vram_gb": "GB",
      "size_gb": "GB"
    }
  },
  "overall": {
    "higher": "qwen-qwen3-8b",
    "difference": 2.3,
    "clearly_ahead": "tie"
  },
  "categories": [
    {
      "id": "reasoning",
      "label": "Reasoning",
      "overall": true,
      "a": 88.5714,
      "b": 94.2857,
      "higher": "qwen-qwen3-8b",
      "difference": 5.7
    },
    {
      "id": "math",
      "label": "Math",
      "overall": true,
      "a": 88.0,
      "b": 88.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "coding",
      "label": "Coding",
      "overall": true,
      "a": 68.9627,
      "b": 70.5749,
      "higher": "qwen-qwen3-8b",
      "difference": 1.6
    },
    {
      "id": "instruction",
      "label": "Instruction following",
      "overall": true,
      "a": 91.0,
      "b": 87.7222,
      "higher": "qwen-qwen3-4b",
      "difference": 3.3
    },
    {
      "id": "knowledge",
      "label": "Knowledge & honesty",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "long_context",
      "label": "Long context",
      "overall": true,
      "a": 77.7778,
      "b": 83.3333,
      "higher": "qwen-qwen3-8b",
      "difference": 5.5
    },
    {
      "id": "multilingual",
      "label": "Multilingual",
      "overall": true,
      "a": 88.3333,
      "b": 100.0,
      "higher": "qwen-qwen3-8b",
      "difference": 11.7
    },
    {
      "id": "tool_use",
      "label": "Tool use",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "safety",
      "label": "Safety",
      "overall": true,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "sycophancy",
      "label": "Pushback resistance",
      "overall": false,
      "a": 100.0,
      "b": 100.0,
      "higher": "tie",
      "difference": 0.0
    },
    {
      "id": "gotchas",
      "label": "Famous gotchas",
      "overall": false,
      "a": 100.0,
      "b": 86.6667,
      "higher": "qwen-qwen3-4b",
      "difference": 13.3
    },
    {
      "id": "writing",
      "label": "Writing",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    },
    {
      "id": "identity",
      "label": "Self-description",
      "overall": false,
      "a": null,
      "b": null,
      "higher": null,
      "difference": null
    }
  ],
  "same_backend": true,
  "same_battery": true,
  "items": {
    "compared": 259,
    "a_wins": 15,
    "b_wins": 20,
    "ties": 224,
    "tie_margin": 0.001,
    "full_marks_at": 0.999,
    "by_category": [
      {
        "id": "reasoning",
        "label": "Reasoning",
        "compared": 35,
        "a_wins": 0,
        "b_wins": 2,
        "ties": 33
      },
      {
        "id": "math",
        "label": "Math",
        "compared": 25,
        "a_wins": 1,
        "b_wins": 1,
        "ties": 23
      },
      {
        "id": "coding",
        "label": "Coding",
        "compared": 30,
        "a_wins": 4,
        "b_wins": 9,
        "ties": 17
      },
      {
        "id": "instruction",
        "label": "Instruction following",
        "compared": 30,
        "a_wins": 6,
        "b_wins": 2,
        "ties": 22
      },
      {
        "id": "knowledge",
        "label": "Knowledge & honesty",
        "compared": 30,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 30
      },
      {
        "id": "long_context",
        "label": "Long context",
        "compared": 18,
        "a_wins": 2,
        "b_wins": 3,
        "ties": 13
      },
      {
        "id": "multilingual",
        "label": "Multilingual",
        "compared": 20,
        "a_wins": 0,
        "b_wins": 3,
        "ties": 17
      },
      {
        "id": "tool_use",
        "label": "Tool use",
        "compared": 20,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 20
      },
      {
        "id": "safety",
        "label": "Safety",
        "compared": 24,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 24
      },
      {
        "id": "sycophancy",
        "label": "Pushback resistance",
        "compared": 12,
        "a_wins": 0,
        "b_wins": 0,
        "ties": 12
      },
      {
        "id": "gotchas",
        "label": "Famous gotchas",
        "compared": 15,
        "a_wins": 2,
        "b_wins": 0,
        "ties": 13
      }
    ],
    "differing": [
      {
        "item_id": "reasoning-033",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "reasoning-034",
        "category": "reasoning",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-017",
        "category": "math",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "math-022",
        "category": "math",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "coding-004",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.25,
          "verdict": "25%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-014",
        "category": "coding",
        "public": true,
        "a": {
          "score": 0.4286,
          "verdict": "43%",
          "full_marks": false,
          "output_text": "```python\ndef reverse_words(s: str) -> str:\n    return ' '.join(s.split())\n```",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "```python\ndef reverse_words(s: str) -> str:\n    return ' '.join(reversed(s.split()))\n```",
          "output_truncated": false
        }
      },
      {
        "item_id": "coding-021",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.8889,
          "verdict": "89%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "coding-022",
        "category": "coding",
        "public": false,
        "a": {
          "score": 0.6364,
          "verdict": "64%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-001",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-009",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-010",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.75,
          "verdict": "75%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-012",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.75,
          "verdict": "75%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "instruction-018",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.8,
          "verdict": "80%",
          "full_marks": false
        }
      },
      {
        "item_id": "instruction-029",
        "category": "instruction",
        "public": false,
        "a": {
          "score": 0.5,
          "verdict": "50%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-004",
        "category": "long_context",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "The stakeholder briefing for Project Cobalt Fen is scheduled for 16 November 2027. \n\nANSWER: 2027-11-16",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "The stakeholder briefing for Project Cobalt Fen is scheduled for 6 June 2027. \n\nANSWER: 2027-06-06",
          "output_truncated": false
        }
      },
      {
        "item_id": "long_context-005",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "long_context-008",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "long_context-014",
        "category": "long_context",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "long_context-017",
        "category": "long_context",
        "public": true,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "From the documents, we can see the following orders for battery packs for the Project Ibis pilot:\n\n1. In the \"PROJECT STATUS REPORT | Project Harbinger\" document, it is mentioned that \"Procurement placed an order for 6,236 battery packs to equip the Project Ibis pilot.\"\n\n2. In the \"MEETING MINUTES | Ref MIN/26/652\" document, it is mentioned that \"Logistics group reviewed the firmware rollout backlog and agreed to defer 25 low-priority items to the next cycle.\" However, this does not mention an order for battery packs.\n\n3. In the \"LAB NOTEBOOK | Entry LAB/26/105\" document, it is mentioned that \"The Project Ibis team asked that the shift rostering figures be checked against the ledger before wider circulation.\" This does not mention an order for battery packs.\n\n4. In the \"INTERNAL MEMO | Ref MEM/26/305\" document, it is mentioned that \"The Data Platform group reviewed the supplier onboarding backlog and agreed to defer 35 low-priority items to the next cycle.\" This does not mention an order for battery packs.\n\nBased on the information provided, only Procurement placed an order for 6,236 battery packs for the Project Ibis pilot. There is no mention of Logistics placing an order for battery packs.\n\nANSWER: 6236",
          "output_truncated": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "To determine the total number of battery packs ordered for the Project Ibis pilot, we need to look for mentions of battery packs in the provided documents.\n\n1. From the **PROJECT STATUS REPORT | Project Harbinger**:\n   - \"Procurement placed an order for 6,236 battery packs to equip the Project Ibis pilot.\"\n\n2. From the **INTERNAL MEMO | Ref MEM/26/636**:\n   - \"Logistics placed an order for 8,774 battery packs to equip the Project Ibis pilot.\"\n\nAdding these two quantities:\n- 6,236 (from Procurement) + 8,774 (from Logistics) = 15,010\n\nANSWER: 15010",
          "output_truncated": false
        }
      },
      {
        "item_id": "multilingual-005",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "multilingual-007",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "multilingual-020",
        "category": "multilingual",
        "public": false,
        "a": {
          "score": 0.6667,
          "verdict": "67%",
          "full_marks": false
        },
        "b": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        }
      },
      {
        "item_id": "gotchas-006",
        "category": "gotchas",
        "public": false,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false
        }
      },
      {
        "item_id": "gotchas-010",
        "category": "gotchas",
        "public": true,
        "a": {
          "score": 1.0,
          "verdict": "pass",
          "full_marks": true,
          "output_text": "We are given a variation of the **Monty Hall problem**, but with a twist: the **host does not know where the car is** and opens **door 3 at random**, revealing a **goat**.\n\nLet’s analyze the situation step by step.\n\n---\n\n### Step 1: Initial Setup\n\n- There are **3 doors**: 1 has a **car**, 2 have **goats**.\n- You pick **door 1**.\n- The host **does not know** where the car is.\n- The host **randomly opens door 3**, and it happens to be a **goat**.\n\nWe are to find the **probability that the car is behind door 2**.\n\n---\n\n### Step 2: Consider All Possible Scenarios\n\nThere are **3 possible initial configurations** (since the car is equally likely to be behind any of the 3 doors):\n\n1. **Car is behind door 1**  \n2. **Car is behind door 2**  \n3. **Car is behind door 3**\n\nLet’s analyze each case:\n\n---\n\n#### Case 1: Car is behind **door 1**\n\n- You picked door 1.\n- Host opens door 3 (randomly), and it's a goat.\n- This is **possible**.\n- In this case, the car is **not** behind door 2.\n\n#### Case 2: Car is behind **door 2**\n\n- You picked door 1.\n- Host opens door 3 (randomly), and it's a goat.\n- This is **possible**.\n- In this case, the car **is** behind door 2.\n\n#### Case 3: Car is behind **door 3**\n\n- You picked door 1.\n- Host opens door 3 (randomly), and it's a goat.\n- This is **not possible**, because the host would have revealed the car.\n\nSo, **Case 3 is invalid**.\n\n---\n\n### Step 3: Count Valid Cases\n\n- Case 1: Valid → Car is behind door 1 → Not behind door 2.\n- Case 2: Valid → Car is behind door 2.\n- Case 3: Invalid → Car is behind door 3 → Host would have revealed the car.\n\nSo, only **2 valid cases** remain: Case 1 and Case 2.\n\nOut of these, **1 case** has the car behind door 2.\n\n---\n\n### Step 4: Compute the Probability\n\n- Total valid cases: 2\n- Favorable case: 1\n\nSo, the probability that the car is behind **door 2** is:\n\n$$\n\\frac{1}{2} = 0.5\n$$\n\n---\n\n### ✅ Final Answer:\n\n**ANSWER: 0.5**",
          "output_truncated": false
        },
        "b": {
          "score": 0.0,
          "verdict": "fail",
          "full_marks": false,
          "output_text": "We are given a variation of the **Monty Hall problem**, where:\n\n- There are **3 doors**: one has a **car**, the other two have **goats**.\n- You pick **door 1**.\n- The **host**, who **does not know** where the car is, opens **door 3** at random.\n- It turns out that **door 3** has a **goat**.\n- Now, we are to find the **probability that the car is behind door 2**.\n\n---\n\n### Step-by-step analysis:\n\nLet’s define the possible scenarios:\n\nThere are **3 doors**: 1, 2, 3.\n\nLet’s assume the car is behind one of the doors. Since the host opens a door **at random**, and it turns out to be a **goat**, we need to consider all possible placements of the car and how the host's action affects the probabilities.\n\n---\n\n### Step 1: List all possible placements of the car\n\nThere are 3 possible placements of the car:\n\n1. **Car is behind door 1**\n2. **Car is behind door 2**\n3. **Car is behind door 3**\n\n---\n\n### Step 2: Consider the host's action\n\nYou picked **door 1**. The host opens **door 3** at random. So, the host has a 50% chance of opening **door 2** or **door 3**.\n\nBut we are told that the host **opened door 3**, and it showed a **goat**.\n\nSo, we are only considering the cases where the host **opened door 3**.\n\nLet’s consider each case:\n\n---\n\n### Case 1: Car is behind **door 1**\n\n- You picked door 1.\n- Host opens door 3 at random.\n- Since the car is behind door 1, the host has a 50% chance of opening door 2 or door 3.\n- So, the host **could** open door 3 (and it would show a goat).\n- So, this case is **possible**.\n\n---\n\n### Case 2: Car is behind **door 2**\n\n- You picked door 1.\n- Host opens door 3 at random.\n- Since the car is behind door 2, the host has a 50% chance of opening door 2 or door 3.\n- So, the host **could** open door 3 (and it would show a goat).\n- So, this case is **possible**.\n\n---\n\n### Case 3: Car is behind **door 3**\n\n- You picked door 1.\n- Host opens door 3 at random.\n- But the car is behind door 3, so the host **cannot** open door 3 (because it has the car).\n- S",
          "output_truncated": true
        }
      }
    ]
  },
  "note": "Speed and memory were measured on one laptop with this server's settings; speeds compare only between models on the same backend, and memory can differ from what one user would need, in either direction. Answer text appears only for public items."
}
