{
  "schema_version": "1.0.0",
  "id": "CR-F-0002",
  "record_type": "finding",
  "title": "Non-uniform expert placement improves held-out hits on one partial published Mixtral trace",
  "claim": "On the bundled 60-token trace digitized from five published Mixtral layers, a placement learned from the first 30 tokens and frozen before scoring the final 30 improves held-out module-hit rate from 16.33% to 21.67% with five resident module slots.",
  "evidence_class": "empirical_model_trace",
  "status": "reported",
  "confidence": {
    "score": 0.58,
    "level": "moderate",
    "rationale": "The split and calculation are leakage-free and deterministic, but the evidence covers only one response, five of 32 layers, and routes digitized from figures rather than original router logits."
  },
  "created_at": "2026-08-09T03:00:00Z",
  "updated_at": "2026-08-09T03:00:00Z",
  "contributors": [
    {
      "id": "crystalram-founders-session-2026-08-08",
      "kind": "human_ai_team",
      "role": "simulation_scientist",
      "model": null
    }
  ],
  "method": {
    "summary": "Digitize the published top-2 routes for transformer layers 1, 8, 16, 24, and 32, learn a five-slot placement on tokens 0 through 29, freeze placement, and score tokens 30 through 59 with a paired token bootstrap.",
    "code_revision": "pre-git-v0.5-baseline",
    "configuration": "CrystalRAM-Simulator-v0/configs/published_mixtral_heldout_v0.5.json",
    "data": "CrystalRAM-Simulator-v0/traces/published_mixtral_partial_2025.csv",
    "random_seed": "bootstrap seed stored in configuration",
    "reproduce": [
      "cd CrystalRAM-Simulator-v0 && python3 crystalram_sim.py configs/published_mixtral_heldout_v0.5.json",
      "python3 CrystalRAM-Simulator-v0/tests/test_sim.py"
    ]
  },
  "results": [
    {"metric": "uniform_trained_heldout_module_hit_rate", "value": 16.3333333333, "unit": "%", "uncertainty": null, "scope": "five visible layers and five resident slots"},
    {"metric": "marginal_trained_heldout_module_hit_rate", "value": 21.6666666667, "unit": "%", "uncertainty": null, "scope": "five visible layers and five resident slots"},
    {"metric": "marginal_gain", "value": 5.3333333333, "unit": "percentage_points", "uncertainty": {"method": "paired_token_bootstrap", "confidence": 0.95, "low": 3.67, "high": 7.0}, "scope": "later 30 response tokens"},
    {"metric": "heldout_oracle_ceiling", "value": 23.6666666667, "unit": "%", "uncertainty": null, "scope": "non-deployable later-token oracle"}
  ],
  "assumptions": [
    "The published plot was digitized without route transcription errors.",
    "The first 30 and later 30 response tokens are a useful small generalization split.",
    "Five resident expert-module slots represent the compared physical budget."
  ],
  "limitations": [
    "Only five of 32 Mixtral transformer layers are present.",
    "Only one model response and 60 response tokens are represented.",
    "Routes were reconstructed from published figures rather than captured from original logits.",
    "The result does not establish whole-model HBM savings or generalize to other prompts."
  ],
  "evidence": [
    {
      "kind": "configuration",
      "uri": "CrystalRAM-Simulator-v0/configs/published_mixtral_heldout_v0.5.json",
      "sha256": "899c7bc5deffc15d29d31998bedc754156421212331740348ade853c12f89125",
      "title": "v0.5 held-out configuration",
      "supports": "The frozen split, capacity, allocation policies, and bootstrap settings.",
      "accessed_at": null
    },
    {
      "kind": "dataset",
      "uri": "CrystalRAM-Simulator-v0/traces/published_mixtral_partial_2025.csv",
      "sha256": "87a7880b9aaa872cb9963e1c8878a789639469e16363d3edda17cfb0590c5b2e",
      "title": "Digitized partial Mixtral route trace",
      "supports": "Token-by-token top-2 expert routes for the five visible published layers.",
      "accessed_at": null
    },
    {
      "kind": "artifact",
      "uri": "CrystalRAM-Simulator-v0/v05_heldout_budget_sweep.csv",
      "sha256": "b2110ea79c0eb707f0d08a0de12ab36ee41e57be71b1eb86bfbfb996aa492805",
      "title": "Held-out slot-budget sweep",
      "supports": "The reported placement comparison over the configured capacity sweep.",
      "accessed_at": null
    },
    {
      "kind": "source",
      "uri": "https://arxiv.org/abs/2511.05814",
      "sha256": null,
      "title": "Published token-by-token Mixtral cache study",
      "supports": "The source figures from which the partial five-layer route trace was reconstructed.",
      "accessed_at": "2026-08-09T03:00:00Z"
    }
  ],
  "relationships": [
    {
      "type": "tested_by",
      "target_id": "CR-E-0001",
      "note": "All-layer multi-prompt trace capture tests whether the limited held-out effect generalizes."
    }
  ],
  "review_requirements": {
    "independent_reproductions": 1,
    "required_review_types": ["reproduction", "code"]
  }
}