{
  "schema_version": "1.0.0",
  "id": "CR-F-0003",
  "record_type": "finding",
  "title": "OLMoE pilot finds stable layer-local hot experts but no significant cross-layer allocation gain",
  "claim": "Across eight held-out prompts and five preregistered budgets, frozen marginal reallocation of expert-module slots between OLMoE layers did not significantly beat equal per-layer capacity; a clearly post-result exploratory decomposition found that equal per-layer banks populated with training-hot expert identities reached 26.306% held-out module hits at 128 modules versus a 12.5% exact random-identity placement expectation.",
  "evidence_class": "empirical_model_trace",
  "status": "reported",
  "confidence": {
    "score": 0.74,
    "level": "moderate",
    "rationale": "The primary null result was preregistered, leakage-free, all-layer, prompt-held-out, hash-audited, and uncertainty-aware. The stronger expert-identity decomposition is internally consistent and preserves the primary artifact, but it was specified after inspecting the primary result and covers only eight short held-out prompts from one four-bit model condition."
  },
  "created_at": "2026-08-09T06:37:07Z",
  "updated_at": "2026-08-09T06:37:07Z",
  "contributors": [
    {
      "id": "crystalram-founders-session-2026-08-08",
      "kind": "human_ai_team",
      "role": "simulation_scientist",
      "provider": "OpenAI",
      "model": "Codex (GPT-5 family; exact deployment version unavailable to runtime)",
      "organization": null,
      "sector": "cross-sector independent research",
      "orcid": null,
      "disclosures": [
        "The random-identity and held-out identity-oracle decomposition was added only after the preregistered cross-layer allocation result was frozen.",
        "No independent reproduction or external code review had been completed when this finding was reported."
      ]
    }
  ],
  "method": {
    "summary": "Capture greedy top-eight routes for all 16 OLMoE layers on twenty balanced prompts, learn placements from twelve declared training prompts, freeze them, score eight held-out prompts, and compare equal per-layer allocation with global marginal-gain allocation at five identical module budgets using a paired whole-prompt bootstrap. Preserve that preregistered primary output before adding an explicitly exploratory expert-identity decomposition.",
    "code_revision": "0c5dca24c496eae761a73b8cbc94d6a34d1555d8",
    "configuration": "research/experiments/CR-E-0004.json",
    "data": "CrystalRAM-Simulator-v0/traces/olmoe_router_pilot_v1/manifest.json",
    "random_seed": 104729,
    "reproduce": [
      ".venv/bin/python CrystalRAM-Simulator-v0/tools/capture_olmoe_mlx_router_trace.py",
      "python3 CrystalRAM-Simulator-v0/tools/analyze_prompt_router_trace.py",
      "python3 CrystalRAM-Simulator-v0/tests/test_router_pilot.py",
      "python3 tools/research_ledger.py validate"
    ]
  },
  "results": [
    {
      "metric": "preregistered_budgets_with_cross_layer_allocation_ci_above_zero",
      "value": 0,
      "unit": "budgets",
      "uncertainty": null,
      "scope": "Resident-module budgets 16, 32, 64, 96, and 128 on eight held-out prompts"
    },
    {
      "metric": "best_preregistered_marginal_layer_allocation_gain",
      "value": 0.311279296875,
      "unit": "percentage_points",
      "uncertainty": {
        "method": "paired_prompt_percentile_bootstrap",
        "confidence": 0.95,
        "samples": 20000,
        "seed": 104729,
        "low": -0.4302978515625,
        "high": 0.970458984375
      },
      "scope": "32 resident modules; preregistered primary comparison"
    },
    {
      "metric": "mean_training_to_heldout_routing_total_variation",
      "value": 12.660725911458334,
      "unit": "%",
      "uncertainty": null,
      "scope": "Mean across all 16 layers"
    },
    {
      "metric": "exploratory_128_module_training_hot_hit_rate",
      "value": 26.30615234375,
      "unit": "%",
      "uncertainty": null,
      "scope": "Equal eight-module capacity per layer; expert identities learned only from training prompts"
    },
    {
      "metric": "exploratory_128_module_random_identity_expected_hit_rate",
      "value": 12.5,
      "unit": "%",
      "uncertainty": null,
      "scope": "Exact expectation over uniformly random expert identities at equal per-layer capacity"
    },
    {
      "metric": "exploratory_128_module_training_hot_gain_over_random_expectation",
      "value": 13.80615234375,
      "unit": "percentage_points",
      "uncertainty": {
        "method": "paired_prompt_percentile_bootstrap_against_exact_random_identity_expectation",
        "confidence": 0.95,
        "samples": 20000,
        "seed": 104729,
        "low": 8.4808349609375,
        "high": 19.354248046875
      },
      "scope": "Post-result exploratory analysis on eight held-out prompts"
    },
    {
      "metric": "exploratory_128_module_training_hot_concentration",
      "value": 2.1044921875,
      "unit": "x_random_identity_expectation",
      "uncertainty": null,
      "scope": "Post-result exploratory analysis"
    },
    {
      "metric": "exploratory_128_module_heldout_identity_oracle_hit_rate",
      "value": 28.2379150390625,
      "unit": "%",
      "uncertainty": null,
      "scope": "Non-deployable equal-capacity identity oracle"
    },
    {
      "metric": "idealized_128_module_resident_weight",
      "value": 402.653184,
      "unit": "MB",
      "uncertainty": null,
      "scope": "Four-bit parameter equivalent excluding quantization metadata and all optical overhead"
    },
    {
      "metric": "idealized_128_module_density_required_for_400mm2",
      "value": 8.05306368,
      "unit": "bit/um2",
      "uncertainty": null,
      "scope": "Weight bits only; excludes addressing, conversion, control, redundancy, and packaging"
    }
  ],
  "assumptions": [
    "The MLX gate wrapper records the same top-eight expert set used by the unmodified OLMoE routing computation while returning the original logits unchanged.",
    "The twelve-versus-eight prompt split is a useful first estimate of cross-prompt generalization for these four workload families.",
    "The analytical random-identity baseline samples resident expert identities uniformly without replacement within each layer at the same equal per-layer capacity.",
    "An ideal four-bit expert module contains 3 times 2048 times 1024 parameters; reported physical equivalents exclude quantization and system overhead."
  ],
  "limitations": [
    "The model is one four-bit MLX conversion of OLMoE; routing may differ under full precision, other quantizers, other checkpoints, or other MoE architectures.",
    "Only eight held-out prompts and two prompts per family contribute to uncertainty, so family-level effects are imprecise.",
    "Every generation is greedy and limited to 32 tokens, leaving long-context drift, conversations, batches, and production traffic untested.",
    "The expert-identity decomposition is exploratory because it was specified after the preregistered primary result was inspected.",
    "A module-hit rate does not measure HBM bytes avoided, complete token latency, throughput, optical loss, conversion overhead, programming cost, energy, endurance, or model quality.",
    "The density calculation is a weight-bit lower bound rather than a demonstrated complete optical array density."
  ],
  "evidence": [
    {
      "kind": "configuration",
      "uri": "research/experiments/CR-E-0004.json",
      "sha256": "03263ed76f5e3e5aad04f5642ee22558602cceb91d011a7061f72f0fcd2d6bb5",
      "title": "Preregistered OLMoE router pilot",
      "supports": "The frozen question, success and failure gates, prompt split, budgets, decoding condition, and uncertainty method.",
      "accessed_at": null
    },
    {
      "kind": "configuration",
      "uri": "research/prompt_suites/moe_router_pilot_v1.json",
      "sha256": "5d549e20d1b4a61258517c84295a8d1473232d092fb2a5741f236f2e4161a1c5",
      "title": "Balanced twenty-prompt pilot suite",
      "supports": "The prompt text, four families, fixed training and held-out assignments, and decoding length.",
      "accessed_at": null
    },
    {
      "kind": "dataset",
      "uri": "CrystalRAM-Simulator-v0/traces/olmoe_router_pilot_v1/manifest.json",
      "sha256": "8a879c7fc35588e855fedd6e7e38fd348b7fd282df986ad45d838a90f61a3ed3",
      "title": "OLMoE all-layer capture manifest",
      "supports": "Twenty prompt-level traces, forty bound artifact hashes, the exact model revision, and the capture code revision.",
      "accessed_at": null
    },
    {
      "kind": "artifact",
      "uri": "CrystalRAM-Simulator-v0/results/olmoe_router_pilot_v1_preregistered_primary.json",
      "sha256": "cfca1c7fc70ff87f6e7cef23c2a94dc07cb13c302560ac355c270ffe6c001c83",
      "title": "Frozen preregistered primary analysis",
      "supports": "The confirmatory cross-layer placement result before exploratory decomposition.",
      "accessed_at": null
    },
    {
      "kind": "artifact",
      "uri": "CrystalRAM-Simulator-v0/results/olmoe_router_pilot_v1.json",
      "sha256": "0124b1820a34053f95b497811760a5728a1a6b06468fe98a1ec85f8dfed13869",
      "title": "Combined primary and labeled exploratory analysis",
      "supports": "All budget, family, layer, distribution-shift, identity-decomposition, and physical-equivalent metrics.",
      "accessed_at": null
    },
    {
      "kind": "artifact",
      "uri": "CrystalRAM-Simulator-v0/results/olmoe_router_pilot_v1_budget_sweep.csv",
      "sha256": "b77b672b487af6258a33fcec3dc50e3031061433c42d6ea9cd138d6349f6fff8",
      "title": "Combined OLMoE module-budget sweep",
      "supports": "The compact equal-capacity, marginal-allocation, identity, uncertainty, and idealized geometry comparison.",
      "accessed_at": null
    },
    {
      "kind": "test",
      "uri": "CrystalRAM-Simulator-v0/tests/test_router_pilot.py",
      "sha256": "ec19b7a352bfdfefa8170784f74ac5a437466b7ec089807c931b7b1aa320b5a2",
      "title": "Router-pilot regression tests",
      "supports": "Token-route alignment, frozen training-only placement, deterministic whole-prompt bootstrap, and exploratory labeling.",
      "accessed_at": null
    },
    {
      "kind": "source",
      "uri": "https://huggingface.co/allenai/OLMoE-1B-7B-0924-Instruct",
      "sha256": null,
      "title": "Ai2 OLMoE model card",
      "supports": "The source model family, open release, and 7B-total approximately 1B-active architecture context.",
      "accessed_at": "2026-08-09T06:37:07Z"
    },
    {
      "kind": "source",
      "uri": "https://huggingface.co/mlx-community/OLMoE-1B-7B-0125-Instruct-4bit",
      "sha256": null,
      "title": "Pinned MLX OLMoE conversion",
      "supports": "The exact four-bit model repository used for capture; the immutable revision is stored in every provenance record.",
      "accessed_at": "2026-08-09T06:37:07Z"
    }
  ],
  "relationships": [
    {
      "type": "derived_from",
      "target_id": "CR-E-0004",
      "note": "This finding reports the preregistered pilot outcome and separately labeled exploratory decomposition."
    },
    {
      "type": "supports",
      "target_id": "CR-F-0002",
      "note": "The training-hot identity result supports a narrow cross-prompt expert-locality effect, while the null primary result does not reproduce the earlier cross-layer allocation gain."
    }
  ],
  "review_requirements": {
    "independent_reproductions": 1,
    "required_review_types": ["reproduction", "code", "adversarial"]
  }
}