Use perplexity from mlxbench-2 only (BOS in every window)

app/recommend.py

@@ -70,6 +70,9 @@ class CommunitySignal:
 
 
 EVAL_DATASET = "wikitext2-test-v1"
+# Benchmark versions whose perplexity is comparable. mlxbench-1 left BOS off most windows,
+# which inflated perplexity for models that have a BOS token (Gemma by 100x).
+PPL_BENCHMARK_VERSIONS = ("mlxbench-2",)
 EVAL_LABEL = "wikitext-2"
 # Perplexity is only comparable between quantizations of the same base model (same tokenizer).
 # Across bit widths the priority-based preference already applies, so measurements only move a

app/stats.py

@@ -14,7 +14,7 @@ from collections import Counter, defaultdict
 from typing import Callable, Iterable
 
 from .events import latest_sessions
-from .recommend import EVAL_DATASET, CommunitySignal
+from .recommend import EVAL_DATASET, PPL_BENCHMARK_VERSIONS, CommunitySignal
 
 K_MIN = 5
 BENCH_TYPES = ("mlx_benchmark", "mlx_benchmark_submission")
@@ -91,7 +91,8 @@ def community_signals(rows: Iterable[dict]) -> dict[str, CommunitySignal]:
         elif r.get("event_type") in BENCH_TYPES:
             if r.get("generation_tps"):
                 bench[m].append(float(r["generation_tps"]))
-            if r.get("perplexity") and r.get("eval_dataset") == EVAL_DATASET:
+            if (r.get("perplexity") and r.get("eval_dataset") == EVAL_DATASET
+                    and r.get("benchmark_version") in PPL_BENCHMARK_VERSIONS):
                 ppl[m].append(float(r["perplexity"]))
     out = {}
     for m in set(fb) | set(bench) | set(ppl):

bench/mlx_explorer_bench.py

@@ -29,7 +29,7 @@ import urllib.error
 import urllib.request
 from pathlib import Path
 
-BENCHMARK_VERSION = "mlxbench-1"
+BENCHMARK_VERSION = "mlxbench-2"  # 2: every perplexity window starts with BOS
 
 # Quality: perplexity on a fixed slice of the wikitext-2 test set (CC BY-SA 3.0).
 # Perplexity depends on the tokenizer, so results are only compared between
@@ -92,12 +92,19 @@ def perplexity(model, tokenizer, max_tokens: int, window: int) -> tuple[float, f
     import mlx.nn as nn
 
     ids = tokenizer.encode(load_eval_text())
-    n_windows = min(len(ids), max_tokens) // window
+    # Models with a BOS token (Gemma, LFM, MiniCPM, ...) expect it at the start of
+    # every sequence; without it Gemma's perplexity runs into the thousands.
+    # Some tokenizers add it on encode, some don't, so strip it and add it per window.
+    bos = getattr(tokenizer, "bos_token_id", None)
+    if bos is not None and ids and ids[0] == bos:
+        ids = ids[1:]
+    step = window - 1 if bos is not None else window
+    n_windows = min(len(ids), max_tokens) // step
     if n_windows < 1:
         raise SystemExit("evaluation text is shorter than one window")
     losses = []
     for w in range(n_windows):
-        chunk = mx.array(ids[w * window:(w + 1) * window])[None]
+        chunk = mx.array(([bos] if bos is not None else []) + ids[w * step:(w + 1) * step])[None]
         logits = model(chunk[:, :-1]).astype(mx.float32)
         loss = nn.losses.cross_entropy(logits, chunk[:, 1:], reduction="none")
         mx.eval(loss)

tests/test_events.py

@@ -155,3 +155,14 @@ def test_quality_submission_validation():
         ev(event_type="mlx_benchmark", perplexity=0.5)
     with pytest.raises(ValidationError):
         ev(event_type="mlx_benchmark", perplexity=9.8, eval_dataset="my-own-text")
+
+
+def test_perplexity_from_old_bench_version_is_ignored():
+    from app.stats import community_signals
+
+    row = dict(event_type="mlx_benchmark", selected_model="mlx-community/gemma-4-e2b-it-4bit", perplexity=5268.9,
+               eval_dataset="wikitext2-test-v1", suspicious_flags=[])
+    sig = community_signals([{**row, "benchmark_version": "mlxbench-1"},
+                             {**row, "benchmark_version": "mlxbench-2", "perplexity": 21.4}])
+    assert sig[row["selected_model"]].median_perplexity == 21.4
+    assert sig[row["selected_model"]].perplexity_count == 1