{
  "all_ok": true,
  "benchmark": {
    "ground_truth": "oss/protocol-bench/src/protocol_bench/data/ground_truth.json",
    "n_tasks": 15,
    "name": "protocol-bench"
  },
  "completions_sha256": "4464e7076715b816aa85573433b162d73de8e0c0767d8cdece5d140e4f6a9c27",
  "executed": true,
  "explicit_non_claim": "a measurement of ONE small open-weights model under greedy decoding with one attempt per task. It is not a claim about language models in general, not a claim about larger models, and not a claim that the benchmark is hard -- only that this model does not beat answering 'safe' every time, and that the benchmark's replay requirement gave it no credit for a counterexample it never produced.",
  "headline": "the model answered SAFE on every one of the 15 tasks: recall on safe procedures is 1.0, recall on violated procedures is 0.0, and its raw accuracy of 0.8667 is exactly the trivial always-safe baseline's 0.8667. Balanced accuracy reports 0.5.",
  "honest_scope": "the labels ship inside the wheel and on the public dataset card and predate every completion; this lane has never scored any model against them before, so the bar could not be fit to the result. Scored once, no re-prompting, no decoding changes.",
  "model": {
    "decoding": "greedy, temperature 0.0, max_tokens 512, one completion per task",
    "open_weights": true,
    "paid_api": false,
    "repo": "mlx-community/Qwen2.5-1.5B-Instruct-4bit",
    "runtime": "mlx_lm 0.31.3"
  },
  "negative_control": {
    "balanced_accuracy": 0.5,
    "description": "fabricated completions: plausible prose with a non-replaying trace on every task",
    "fired": true,
    "valid_counterexamples": 0
  },
  "predictions_and_outcomes": [
    {
      "held": true,
      "observed": 0.5,
      "prediction": "balanced_accuracy in [0.40, 0.70]"
    },
    {
      "held": true,
      "observed": 0,
      "prediction": "valid replayed counterexamples <= 1"
    },
    {
      "held": true,
      "observed": "the model answered violated=false with an empty trace",
      "prediction": "the KRACK trace does not replay"
    }
  ],
  "schema": "protocol-bench-model-run-v1",
  "scores": {
    "accuracy": 0.8666666666666667,
    "accuracy_ignoring_replay": 0.8666666666666667,
    "balanced_accuracy": 0.5,
    "detections_claimed": 0,
    "n_unparseable": 0,
    "recall_safe": 1.0,
    "recall_violated": 0.0,
    "trivial_always_safe_accuracy": 0.8666666666666667,
    "valid_counterexamples": 0
  },
  "sealed_bar": {
    "file": "artifacts/prereg/protocol_bench_qwen_bar.json",
    "sealed_at_commit": "1d5b14cbdedbad31eca050c71613f609396a83c3",
    "sha256": "640df291c2e37bf02ff090524d752e4b42773041ba79d37402fdff89f8027d50"
  },
  "skipped": false
}