Showcase

DPO, animated: teach a tiny model to prefer, built by Gemini 4 Argon

Hostedvia DaBenchOpenRouter
Running live in your browserNew tab
Cost to build
$0.23
Show the code the model wrote
HTML74,374 characters
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Direct Preference Optimization — Live Implicit Reward & KL Tether</title>
<style>
  *, *::before, *::after {
    box-sizing: border-box;
    margin: 0;
    padding: 0;
    user-select: none;
  }

  :root {
    --bg-canvas: #07090e;
    --bg-panel: #0c1018;
    --bg-card: #101622;
    --bg-card-inner: #0a0e17;
    --border-dim: rgba(148, 163, 184, 0.13);
    --border-highlight: rgba(56, 189, 248, 0.35);
    --text-main: #f1f5f9;
    --text-muted: #94a3b8;
    --text-dim: #64748b;
    --chosen: #38bdf8;
    --chosen-glow: rgba(56, 189, 248, 0.25);
    --rejected: #f43f5e;
    --rejected-glow: rgba(244, 63, 94, 0.25);
    --amber: #f59e0b;
    --violet: #a855f7;
    --emerald: #10b981;
    --mono: 'JetBrains Mono', 'Fira Code', 'SF Mono', 'Cascadia Code', Consolas, monospace;
    --sans: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, Helvetica, Arial, sans-serif;
  }

  html, body {
    width: 100vw;
    height: 100vh;
    overflow: hidden;
    background-color: var(--bg-canvas);
    color: var(--text-main);
    font-family: var(--sans);
    -webkit-font-smoothing: antialiased;
  }

  .app-shell {
    display: flex;
    flex-direction: column;
    width: 100vw;
    height: 100vh;
    overflow: hidden;
  }

  /* Main Stage: Hero Canvas + Right Instrument Column */
  .stage {
    display: grid;
    grid-template-columns: 1fr 560px;
    flex: 1;
    min-height: 0;
    width: 100%;
    overflow: hidden;
  }

  @media (max-width: 1500px) {
    .stage {
      grid-template-columns: 1fr 480px;
    }
  }

  .hero-container {
    position: relative;
    width: 100%;
    height: 100%;
    background: radial-gradient(circle at 42% 45%, #0e1525 0%, #07090e 78%);
    border-right: 1px solid var(--border-dim);
    overflow: hidden;
  }

  #heroCanvas {
    display: block;
    width: 100%;
    height: 100%;
    cursor: pointer;
  }

  /* Right Telemetry & Inspector Column */
  .sidebar {
    display: flex;
    flex-direction: column;
    gap: 8px;
    padding: 10px;
    background: var(--bg-panel);
    height: 100%;
    min-height: 0;
    overflow: hidden;
  }

  .card {
    background: var(--bg-card);
    border: 1px solid var(--border-dim);
    border-radius: 6px;
    padding: 10px 12px;
    display: flex;
    flex-direction: column;
    position: relative;
  }

  .card-header {
    display: flex;
    justify-content: space-between;
    align-items: center;
    margin-bottom: 6px;
  }

  .card-title {
    font-size: 10.5px;
    font-weight: 700;
    letter-spacing: 0.09em;
    text-transform: uppercase;
    color: var(--text-muted);
  }

  .card-badge {
    font-family: var(--mono);
    font-size: 10px;
    padding: 1px 6px;
    border-radius: 3px;
    background: rgba(148, 163, 184, 0.1);
    color: var(--text-muted);
  }

  /* Card 1: Big Number & KL Gauge */
  .telemetry-grid {
    display: grid;
    grid-template-columns: 1.15fr 1fr;
    gap: 10px;
    align-items: stretch;
  }

  .hero-stat-box {
    background: var(--bg-card-inner);
    border: 1px solid rgba(56, 189, 248, 0.22);
    border-radius: 5px;
    padding: 8px 10px;
    display: flex;
    flex-direction: column;
    justify-content: space-between;
  }

  .stat-eyebrow {
    font-size: 10px;
    font-weight: 700;
    letter-spacing: 0.08em;
    text-transform: uppercase;
    color: var(--chosen);
    display: flex;
    justify-content: space-between;
    align-items: center;
  }

  .big-number {
    font-family: var(--mono);
    font-size: 42px;
    font-weight: 800;
    line-height: 1.05;
    letter-spacing: -0.03em;
    color: var(--text-main);
    text-shadow: 0 0 24px rgba(56, 189, 248, 0.35);
    margin: 3px 0;
  }

  .stat-sub {
    font-family: var(--mono);
    font-size: 11px;
    color: var(--text-muted);
    display: flex;
    justify-content: space-between;
  }

  .kl-stat-box {
    background: var(--bg-card-inner);
    border: 1px solid rgba(168, 85, 247, 0.25);
    border-radius: 5px;
    padding: 8px 10px;
    display: flex;
    flex-direction: column;
    justify-content: space-between;
  }

  .kl-eyebrow {
    font-size: 10px;
    font-weight: 700;
    letter-spacing: 0.08em;
    text-transform: uppercase;
    color: #c084fc;
    display: flex;
    justify-content: space-between;
  }

  .kl-number-row {
    display: flex;
    align-items: baseline;
    gap: 6px;
    margin: 2px 0;
  }

  .kl-number {
    font-family: var(--mono);
    font-size: 25px;
    font-weight: 700;
    color: #e9d5ff;
  }

  .kl-unit {
    font-family: var(--mono);
    font-size: 11px;
    color: var(--text-dim);
  }

  .gauge-track {
    width: 100%;
    height: 6px;
    background: rgba(255, 255, 255, 0.06);
    border-radius: 3px;
    overflow: hidden;
    margin: 4px 0;
  }

  .gauge-fill {
    height: 100%;
    background: linear-gradient(90deg, #38bdf8, #a855f7, #f43f5e);
    width: 0%;
    transition: width 0.08s linear;
  }

  /* Card 2: Known DPO Effect (Log-Probs Fall While Margin Grows) */
  .logprob-grid {
    display: grid;
    grid-template-columns: 1fr 1fr 1fr;
    gap: 6px;
    margin-bottom: 6px;
  }

  .lp-cell {
    background: var(--bg-card-inner);
    border: 1px solid var(--border-dim);
    border-radius: 4px;
    padding: 5px 8px;
  }

  .lp-label {
    font-size: 9.5px;
    color: var(--text-muted);
    text-transform: uppercase;
    letter-spacing: 0.04em;
    display: flex;
    align-items: center;
    gap: 4px;
  }

  .dot {
    width: 6px;
    height: 6px;
    border-radius: 50%;
    display: inline-block;
  }

  .lp-val {
    font-family: var(--mono);
    font-size: 14.5px;
    font-weight: 700;
    margin-top: 2px;
  }

  .lp-delta {
    font-family: var(--mono);
    font-size: 10px;
    color: var(--text-dim);
  }

  #sparkCanvas {
    width: 100%;
    height: 54px;
    display: block;
    background: var(--bg-card-inner);
    border: 1px solid var(--border-dim);
    border-radius: 4px;
  }

  /* Card 3: Single-Pair Exact Mathematical Loss Inspector */
  .inspector-box {
    background: var(--bg-card-inner);
    border: 1px solid var(--border-dim);
    border-radius: 5px;
    padding: 7px 9px;
    font-family: var(--mono);
    font-size: 11.5px;
    display: flex;
    flex-direction: column;
    gap: 5px;
  }

  .insp-row {
    display: flex;
    justify-content: space-between;
    align-items: center;
    padding: 2px 0;
    border-bottom: 1px dashed rgba(148, 163, 184, 0.1);
  }

  .insp-row:last-child {
    border-bottom: none;
  }

  .insp-formula {
    color: var(--text-muted);
  }

  .insp-math {
    color: var(--text-main);
    font-weight: 600;
  }

  .loss-highlight-bar {
    display: grid;
    grid-template-columns: 1.25fr 1fr;
    gap: 8px;
    margin-top: 3px;
  }

  .loss-pill {
    background: rgba(56, 189, 248, 0.08);
    border: 1px solid rgba(56, 189, 248, 0.3);
    border-radius: 4px;
    padding: 5px 8px;
    display: flex;
    flex-direction: column;
  }

  .weight-pill {
    background: rgba(245, 158, 11, 0.08);
    border: 1px solid rgba(245, 158, 11, 0.3);
    border-radius: 4px;
    padding: 5px 8px;
    display: flex;
    flex-direction: column;
  }

  .pill-head {
    font-size: 9.5px;
    text-transform: uppercase;
    color: var(--text-muted);
    display: flex;
    justify-content: space-between;
  }

  .pill-val {
    font-family: var(--mono);
    font-size: 16px;
    font-weight: 800;
    margin-top: 2px;
  }

  /* Card 4: Side-by-Side Chat Box */
  .chat-card {
    flex: 1;
    min-height: 0;
    display: flex;
    flex-direction: column;
    justify-content: space-between;
  }

  .prompt-bar {
    background: rgba(56, 189, 248, 0.08);
    border: 1px solid rgba(56, 189, 248, 0.25);
    border-radius: 5px;
    padding: 6px 10px;
    display: flex;
    justify-content: space-between;
    align-items: center;
    margin-bottom: 6px;
  }

  .prompt-tokens {
    font-family: var(--mono);
    font-size: 12.5px;
    color: var(--text-main);
    font-weight: 600;
  }

  .token-tag {
    font-size: 10px;
    padding: 1px 5px;
    border-radius: 3px;
    background: rgba(148, 163, 184, 0.16);
    color: var(--chosen);
    margin-right: 4px;
  }

  .nav-btn-group {
    display: flex;
    gap: 4px;
  }

  .mini-btn {
    background: rgba(255, 255, 255, 0.06);
    border: 1px solid var(--border-dim);
    color: var(--text-main);
    font-family: var(--mono);
    font-size: 10px;
    padding: 2px 7px;
    border-radius: 3px;
    cursor: pointer;
  }

  .mini-btn:hover {
    border-color: var(--chosen);
    background: rgba(56, 189, 248, 0.15);
  }

  .chat-comparison {
    display: grid;
    grid-template-columns: 1fr 1fr;
    gap: 8px;
    flex: 1;
    min-height: 0;
  }

  .chat-col {
    background: var(--bg-card-inner);
    border: 1px solid var(--border-dim);
    border-radius: 5px;
    padding: 8px 10px;
    display: flex;
    flex-direction: column;
    justify-content: space-between;
  }

  .chat-col.policy-col {
    border-color: rgba(56, 189, 248, 0.35);
  }

  .chat-col-head {
    display: flex;
    justify-content: space-between;
    align-items: center;
    font-size: 10px;
    font-weight: 700;
    text-transform: uppercase;
    letter-spacing: 0.06em;
  }

  .chat-bubble {
    font-family: var(--mono);
    font-size: 13.5px;
    font-weight: 700;
    padding: 8px 9px;
    border-radius: 5px;
    margin: 5px 0;
    line-height: 1.35;
    word-break: break-word;
  }

  .bubble-rude {
    background: rgba(244, 63, 94, 0.12);
    border-left: 3px solid var(--rejected);
    color: #fda4af;
  }

  .bubble-polite {
    background: rgba(56, 189, 248, 0.13);
    border-left: 3px solid var(--chosen);
    color: #bae6fd;
  }

  .chat-meta {
    font-family: var(--mono);
    font-size: 10px;
    color: var(--text-dim);
    display: flex;
    justify-content: space-between;
    border-top: 1px solid rgba(255, 255, 255, 0.05);
    padding-top: 4px;
  }

  /* Slim Bottom Control Bar */
  .control-bar {
    height: 52px;
    background: #080b11;
    border-top: 1px solid var(--border-dim);
    padding: 0 16px;
    display: flex;
    align-items: center;
    justify-content: space-between;
    gap: 14px;
    z-index: 10;
  }

  .ctrl-group {
    display: flex;
    align-items: center;
    gap: 8px;
  }

  .btn {
    background: #131b2e;
    border: 1px solid rgba(148, 163, 184, 0.25);
    color: var(--text-main);
    font-family: var(--sans);
    font-size: 12px;
    font-weight: 600;
    padding: 6px 13px;
    border-radius: 4px;
    cursor: pointer;
    display: flex;
    align-items: center;
    gap: 6px;
    transition: all 0.12s ease;
  }

  .btn:hover {
    background: #1c2742;
    border-color: rgba(56, 189, 248, 0.5);
  }

  .btn.active-demo {
    background: rgba(56, 189, 248, 0.18);
    border-color: var(--chosen);
    color: #7dd3fc;
    box-shadow: 0 0 12px rgba(56, 189, 248, 0.25);
  }

  .btn.active-still {
    background: rgba(245, 158, 11, 0.2);
    border-color: var(--amber);
    color: #fde68a;
    box-shadow: 0 0 12px rgba(245, 158, 11, 0.25);
  }

  .btn.primary-run {
    background: rgba(16, 185, 129, 0.16);
    border-color: rgba(16, 185, 129, 0.45);
    color: #6ee7b7;
  }

  .slider-group {
    display: flex;
    align-items: center;
    gap: 9px;
    background: #0e1421;
    border: 1px solid var(--border-dim);
    padding: 5px 12px;
    border-radius: 4px;
  }

  .slider-label {
    font-size: 11px;
    font-weight: 600;
    color: var(--text-muted);
    white-space: nowrap;
  }

  .slider-val {
    font-family: var(--mono);
    font-size: 12.5px;
    font-weight: 700;
    color: var(--chosen);
    min-width: 44px;
    text-align: right;
  }

  input[type="range"] {
    -webkit-appearance: none;
    appearance: none;
    width: 130px;
    height: 5px;
    background: #1e293b;
    border-radius: 3px;
    outline: none;
    cursor: pointer;
  }

  input[type="range"]::-webkit-slider-thumb {
    -webkit-appearance: none;
    appearance: none;
    width: 14px;
    height: 14px;
    border-radius: 50%;
    background: var(--chosen);
    border: 2px solid #07090e;
    box-shadow: 0 0 8px rgba(56, 189, 248, 0.6);
    cursor: pointer;
  }

  .step-readout {
    font-family: var(--mono);
    font-size: 12px;
    color: var(--text-muted);
    background: #0e1421;
    border: 1px solid var(--border-dim);
    padding: 5px 11px;
    border-radius: 4px;
  }
</style>
</head>
<body>
<div class="app-shell">
  <div class="stage">
    <!-- Left/Center Hero Plot Canvas -->
    <div class="hero-container">
      <canvas id="heroCanvas"></canvas>
    </div>

    <!-- Right Column: Gauges, Log-Prob Paradox, Loss Inspector, Chat Comparison -->
    <div class="sidebar">
      <!-- 1. Reward Accuracy (Big Number) & KL Divergence Gauge -->
      <div class="card">
        <div class="card-header">
          <span class="card-title">Policy Alignment Telemetry (30 Preference Pairs)</span>
          <span class="card-badge" id="modelSpecBadge">60-tok · 4-ctx · tanh-24</span>
        </div>
        <div class="telemetry-grid">
          <div class="hero-stat-box">
            <div class="stat-eyebrow">
              <span>Reward Accuracy (r_w &gt; r_l)</span>
              <span id="pairCountText" style="color:var(--text-muted)">10/30</span>
            </div>
            <div class="big-number" id="bigAccuracy">33.3%</div>
            <div class="stat-sub">
              <span>Mean Margin Δr:</span>
              <span id="meanRewardMargin" style="color:var(--chosen);font-weight:700">-0.012</span>
            </div>
          </div>

          <div class="kl-stat-box">
            <div class="kl-eyebrow">
              <span>KL Divergence D_KL(πθ ‖ πref)</span>
              <span id="klTetherState" style="color:#c084fc">TETHER: β=0.10</span>
            </div>
            <div class="kl-number-row">
              <span class="kl-number" id="klValue">0.0002</span>
              <span class="kl-unit">nats</span>
            </div>
            <div class="gauge-track">
              <div class="gauge-fill" id="klGaugeFill"></div>
            </div>
            <div class="stat-sub">
              <span>Mean Loss L_DPO:</span>
              <span id="meanDpoLoss" style="color:var(--text-main)">0.6991</span>
            </div>
          </div>
        </div>
      </div>

      <!-- 2. The Known DPO Effect: Chosen & Rejected Log-Probabilities Fall While Margin Grows -->
      <div class="card">
        <div class="card-header">
          <span class="card-title">Known DPO Effect: Both Log-Probs Fall While Margin Grows</span>
          <span class="card-badge">Sequence Log-Probabilities</span>
        </div>
        <div class="logprob-grid">
          <div class="lp-cell">
            <div class="lp-label"><span class="dot" style="background:var(--chosen)"></span>Mean log πθ(y_w|x)</div>
            <div class="lp-val" id="meanLogPiW" style="color:var(--chosen)">-5.642</div>
            <div class="lp-delta" id="deltaLogPiW">ref: -5.638</div>
          </div>
          <div class="lp-cell">
            <div class="lp-label"><span class="dot" style="background:var(--rejected)"></span>Mean log πθ(y_l|x)</div>
            <div class="lp-val" id="meanLogPiL" style="color:var(--rejected)">-5.581</div>
            <div class="lp-delta" id="deltaLogPiL">ref: -5.562</div>
          </div>
          <div class="lp-cell">
            <div class="lp-label"><span class="dot" style="background:var(--emerald)"></span>Log-Ratio Margin m</div>
            <div class="lp-val" id="meanRawMargin" style="color:var(--emerald)">-0.077</div>
            <div class="lp-delta">Δlog π_w − Δlog π_l</div>
          </div>
        </div>
        <canvas id="sparkCanvas"></canvas>
      </div>

      <!-- 3. Single-Pair Exact Mathematical DPO Loss Inspector -->
      <div class="card">
        <div class="card-header">
          <span class="card-title">Exact Single-Pair Loss Inspector</span>
          <span class="card-badge" id="inspPairLabel">Pair #01 (Click any row)</span>
        </div>
        <div class="inspector-box">
          <div class="insp-row">
            <span class="insp-formula" style="color:var(--chosen)">Chosen Δlog π_w = log πθ − log πref</span>
            <span class="insp-math" id="inspChosenMath">-5.6120 − (-5.6100) = -0.0020</span>
          </div>
          <div class="insp-row">
            <span class="insp-formula" style="color:var(--rejected)">Rejected Δlog π_l = log πθ − log πref</span>
            <span class="insp-math" id="inspRejectedMath">-5.4810 − (-5.4900) = +0.0090</span>
          </div>
          <div class="insp-row">
            <span class="insp-formula">Margin m = Δlog π_w − Δlog π_l</span>
            <span class="insp-math" id="inspMarginMath">m = -0.0110 | β·m = -0.0011</span>
          </div>
          <div class="loss-highlight-bar">
            <div class="loss-pill">
              <div class="pill-head">
                <span>L_DPO = −log σ(β · m)</span>
                <span style="color:var(--chosen)">EXACT 4-DEC</span>
              </div>
              <div class="pill-val" id="inspExactLoss" style="color:var(--text-main)">0.6937</div>
            </div>
            <div class="weight-pill">
              <div class="pill-head">
                <span>Grad Wt σ(−β · m)</span>
                <span id="inspOrderStatus" style="color:var(--amber)">ACTIVE</span>
              </div>
              <div class="pill-val" id="inspGradWeight" style="color:var(--amber)">0.5003</div>
            </div>
          </div>
        </div>
      </div>

      <!-- 4. Side-by-Side Chat Box: Reference Model vs Policy Model -->
      <div class="card chat-card">
        <div class="card-header">
          <span class="card-title">Live Decoding Comparison: Frozen Reference vs. Policy</span>
          <div class="nav-btn-group">
            <button class="mini-btn" id="prevPairBtn" title="Previous Prompt">◀ Prev</button>
            <button class="mini-btn" id="nextPairBtn" title="Next Prompt">Next ▶</button>
          </div>
        </div>
        <div class="prompt-bar">
          <div class="prompt-tokens" id="chatPromptDisplay">
            <span class="token-tag">&lt;user&gt;</span>help me now<span class="token-tag" style="margin-left:6px">&lt;asst&gt;</span>
          </div>
          <span class="card-badge" id="chatPairIdx">Prompt #01 / 30</span>
        </div>
        <div class="chat-comparison">
          <div class="chat-col">
            <div class="chat-col-head" style="color:var(--text-muted)">
              <span>Frozen Reference (π_ref)</span>
              <span style="color:var(--rejected)">SFT Copy</span>
            </div>
            <div class="chat-bubble bubble-rude" id="refChatOutput">no go do it &lt;end&gt;</div>
            <div class="chat-meta">
              <span id="refLogProbReadout">log πref(y_w): -5.61</span>
              <span id="refRankReadout">ranks: rude &gt; polite</span>
            </div>
          </div>
          <div class="chat-col policy-col">
            <div class="chat-col-head" style="color:var(--chosen)">
              <span>Live Policy (π_θ)</span>
              <span id="policyChatBadge" style="color:var(--rejected)">Unaligned</span>
            </div>
            <div class="chat-bubble bubble-rude" id="policyChatOutput">no go do it &lt;end&gt;</div>
            <div class="chat-meta">
              <span id="polLogProbReadout">r_w: +0.00 | r_l: +0.00</span>
              <span id="polRankReadout" style="color:var(--chosen)">Δr: +0.000</span>
            </div>
          </div>
        </div>
      </div>
    </div>
  </div>

  <!-- Slim Bottom Control Bar -->
  <div class="control-bar">
    <div class="ctrl-group">
      <button class="btn active-demo" id="demoBtn">● Demo Scenario (20s Loop)</button>
      <button class="btn" id="stillBtn">◼ Still (Peak Separation)</button>
      <button class="btn primary-run" id="playPauseBtn">⏸ Pause Training</button>
      <button class="btn" id="stepBtn">Step +1</button>
      <button class="btn" id="resetBtn">↺ Reset πθ ← πref</button>
    </div>

    <div class="ctrl-group">
      <div class="slider-group">
        <span class="slider-label">KL Beta (β):</span>
        <input type="range" id="betaSlider" min="0.01" max="1.00" step="0.01" value="0.10">
        <span class="slider-val" id="betaVal">0.10</span>
      </div>

      <div class="slider-group">
        <span class="slider-label">Learning Rate (η):</span>
        <input type="range" id="lrSlider" min="0.005" max="0.150" step="0.005" value="0.055">
        <span class="slider-val" id="lrVal">0.055</span>
      </div>

      <div class="step-readout" id="stepCounter">Step: 0000</div>
    </div>
  </div>
</div>

<script>
(() => {
  "use strict";

  // ============================================================================
  // 1. VOCABULARY (60 WORD TOKENS WITH <pad>, <user>, <asst>, <end>)
  // ============================================================================
  const VOCAB = [
    "<pad>", "<user>", "<asst>", "<end>", // 0..3
    // Prompt words (4..31)
    "help", "me", "now", "explain", "this", "code", "fix", "the", "bug", "write",
    "a", "note", "check", "my", "plan", "find", "file", "how", "to", "start",
    "what", "is", "wrong", "today", "review", "task", "test", "app",
    // Polite / concise chosen words (32..45)
    "sure", "i", "can", "gladly", "right", "away", "here", "clear", "answer", "fixed",
    "for", "you", "done", "happy",
    // Rude / rambling rejected words (46..59)
    "no", "go", "do", "it", "yourself", "stop", "asking", "stupid", "whatever", "later",
    "ugh", "boring", "waste", "time"
  ];

  const WORD_TO_ID = new Map();
  VOCAB.forEach((w, i) => WORD_TO_ID.set(w, i));
  const V = VOCAB.length; // 60 tokens
  const CTX = 4;          // 4-token context
  const EMB = 8;          // 8-dim token embedding
  const HID = 24;         // 24-unit tanh hidden layer

  // ============================================================================
  // 2. PREFERENCE DATASET: 30 PROMPTS WITH CHOSEN (POLITE) & REJECTED (RUDE)
  // ============================================================================
  const RAW_PAIRS = [
    { p: "help me now",       w: "sure i can",         l: "no go do" },
    { p: "explain this code", w: "gladly right away",  l: "ugh boring code" },
    { p: "fix the bug",       w: "fixed for you",      l: "fix it yourself" },
    { p: "write a note",      w: "sure right away",    l: "no waste time" },
    { p: "check my plan",     w: "happy to help",      l: "stop asking me" },
    { p: "find the file",     w: "here for you",       l: "go find it" },
    { p: "how to start",      w: "here clear answer",  l: "whatever do later" },
    { p: "what is wrong",     w: "i can explain",      l: "ugh stupid bug" },
    { p: "review my code",    w: "gladly for you",     l: "no boring task" },
    { p: "test the app",      w: "done right away",    l: "do it later" },
    { p: "help my plan",      w: "sure happy to",      l: "no go away" },
    { p: "explain the bug",   w: "clear answer here",  l: "stop asking stupid" },
    { p: "fix this file",     w: "sure fixed now",     l: "ugh whatever later" },
    { p: "write my code",     w: "i can help",         l: "write it yourself" },
    { p: "check the app",     w: "done for you",       l: "waste my time" },
    { p: "find a plan",       w: "gladly i can",       l: "no find yourself" },
    { p: "how is this",       w: "clear for you",      l: "ugh boring plan" },
    { p: "what to fix",       w: "happy to check",     l: "whatever stop asking" },
    { p: "review the note",   w: "sure done now",      l: "no do later" },
    { p: "test my file",      w: "gladly done now",    l: "go test yourself" },
    { p: "help with code",    w: "sure i explain",     l: "no stupid code" }, // let's map "with" -> "this"
    { p: "explain my task",   w: "here clear plan",    l: "ugh waste time" },
    { p: "fix a bug",         w: "fixed right away",   l: "no fix later" },
    { p: "write the plan",    w: "gladly i write",     l: "do it yourself" },
    { p: "check this note",   w: "sure happy check",   l: "stop asking now" },
    { p: "find my app",       w: "here done now",      l: "whatever go find" },
    { p: "how to fix",        w: "i can fix",          l: "ugh stupid task" },
    { p: "what is this",      w: "clear answer now",   l: "boring waste time" },
    { p: "review a file",     w: "gladly right now",   l: "no review yourself" },
    { p: "test this today",   w: "sure done today",    l: "ugh do later" }
  ];
  // Fix pair 20 prompt to strictly use VOCAB tokens:
  RAW_PAIRS[20].p = "help this code";

  function encodeWords(str) {
    return str.trim().split(/\s+/).map(w => {
      const id = WORD_TO_ID.get(w);
      if (id === undefined) throw new Error("Unknown token: " + w);
      return id;
    });
  }

  // Build tokenized pairs with 4-token sliding contexts
  const PAIRS = RAW_PAIRS.map((item, idx) => {
    const pIds = encodeWords(item.p); // 3 tokens
    const wIds = [...encodeWords(item.w), 3]; // 3 words + <end> = 4 tokens
    const lIds = [...encodeWords(item.l), 3]; // 3 words + <end> = 4 tokens
    // Full prefix before response: [<user>, p1, p2, p3, <asst>]
    const prefix = [1, ...pIds, 2];

    const wSteps = [];
    const lSteps = [];
    for (let t = 0; t < wIds.length; t++) {
      const seqW = [...prefix, ...wIds.slice(0, t)];
      const ctxW = seqW.slice(seqW.length - CTX);
      wSteps.push({ ctx: ctxW, target: wIds[t] });

      const seqL = [...prefix, ...lIds.slice(0, t)];
      const ctxL = seqL.slice(seqL.length - CTX);
      lSteps.push({ ctx: ctxL, target: lIds[t] });
    }

    return {
      idx,
      promptStr: item.p,
      chosenStr: item.w + " <end>",
      rejectedStr: item.l + " <end>",
      prefix,
      wIds,
      lIds,
      wSteps,
      lSteps,
      // Live metrics updated each forward pass
      logPiW: 0,
      logRefW: 0,
      logPiL: 0,
      logRefL: 0,
      rw: 0,
      rl: 0,
      margin: 0,
      rewardMargin: 0,
      loss: 0,
      gradWeight: 0.5,
      // Smooth display coordinates for canvas markers
      dispRw: 0,
      dispRl: 0
    };
  });

  // ============================================================================
  // 3. FROM-SCRATCH NEXT-TOKEN NEURAL NETWORK (EMBEDDING, 4-CTX, TANH, SOFTMAX)
  // ============================================================================
  let seed = 421337;
  function randn() {
    seed = (seed * 1664525 + 1013904223) >>> 0;
    const u1 = (seed + 1) / 4294967297;
    seed = (seed * 1664525 + 1013904223) >>> 0;
    const u2 = (seed + 1) / 4294967297;
    return Math.sqrt(-2 * Math.log(u1)) * Math.cos(2 * Math.PI * u2);
  }

  function createModel() {
    const E = new Float64Array(V * EMB);
    const W1 = new Float64Array(CTX * EMB * HID);
    const b1 = new Float64Array(HID);
    const W2 = new Float64Array(HID * V);
    const b2 = new Float64Array(V);

    for (let i = 0; i < E.length; i++) E[i] = randn() * 0.22;
    for (let i = 0; i < W1.length; i++) W1[i] = randn() * 0.18;
    for (let i = 0; i < W2.length; i++) W2[i] = randn() * 0.16;

    return { E, W1, b1, W2, b2 };
  }

  function cloneModel(m) {
    return {
      E: new Float64Array(m.E),
      W1: new Float64Array(m.W1),
      b1: new Float64Array(m.b1),
      W2: new Float64Array(m.W2),
      b2: new Float64Array(m.b2)
    };
  }

  function copyModelInPlace(dst, src) {
    dst.E.set(src.E);
    dst.W1.set(src.W1);
    dst.b1.set(src.b1);
    dst.W2.set(src.W2);
    dst.b2.set(src.b2);
  }

  // Forward pass for a single 4-token context -> returns hidden activations and logProbs/probs
  const scratchX = new Float64Array(CTX * EMB);
  const scratchH = new Float64Array(HID);
  const scratchLogits = new Float64Array(V);
  const scratchProbs = new Float64Array(V);
  const scratchLogProbs = new Float64Array(V);

  function forwardStep(model, ctx, outH, outProbs, outLogProbs) {
    const { E, W1, b1, W2, b2 } = model;
    const INP = CTX * EMB;

    // 1. Lookup 4-token context embeddings
    for (let c = 0; c < CTX; c++) {
      const tok = ctx[c];
      const eOff = tok * EMB;
      const xOff = c * EMB;
      for (let d = 0; d < EMB; d++) {
        scratchX[xOff + d] = E[eOff + d];
      }
    }

    // 2. Hidden layer with tanh activation
    for (let j = 0; j < HID; j++) {
      let sum = b1[j];
      for (let i = 0; i < INP; i++) {
        sum += scratchX[i] * W1[i * HID + j];
      }
      outH[j] = Math.tanh(sum);
    }

    // 3. Output logits & numerically stable Softmax / LogSoftmax
    let maxLogit = -1e30;
    for (let v = 0; v < V; v++) {
      let sum = b2[v];
      for (let j = 0; j < HID; j++) {
        sum += outH[j] * W2[j * V + v];
      }
      scratchLogits[v] = sum;
      if (sum > maxLogit) maxLogit = sum;
    }

    let sumExp = 0;
    for (let v = 0; v < V; v++) {
      const ex = Math.exp(scratchLogits[v] - maxLogit);
      outProbs[v] = ex;
      sumExp += ex;
    }
    const logZ = maxLogit + Math.log(sumExp);
    for (let v = 0; v < V; v++) {
      outProbs[v] /= sumExp;
      outLogProbs[v] = scratchLogits[v] - logZ;
    }
  }

  // Hand-written backward accumulation for one context step given dL/dLogits (dLogits)
  function backwardStepAccum(model, grads, ctx, h, dLogits, scale) {
    const { W1, W2 } = model;
    const { dE, dW1, db1, dW2, db2 } = grads;
    const INP = CTX * EMB;

    // Reconstruct scratchX for ctx
    for (let c = 0; c < CTX; c++) {
      const eOff = ctx[c] * EMB;
      const xOff = c * EMB;
      for (let d = 0; d < EMB; d++) scratchX[xOff + d] = model.E[eOff + d];
    }

    // Grad through W2, b2 and into h
    const dH = new Float64Array(HID);
    for (let v = 0; v < V; v++) {
      const g = dLogits[v] * scale;
      if (Math.abs(g) < 1e-14) continue;
      db2[v] += g;
      for (let j = 0; j < HID; j++) {
        dW2[j * V + v] += h[j] * g;
        dH[j] += W2[j * V + v] * g;
      }
    }

    // Grad through tanh: dPreH = dH * (1 - h^2)
    for (let j = 0; j < HID; j++) {
      const dPre = dH[j] * (1 - h[j] * h[j]);
      if (Math.abs(dPre) < 1e-14) continue;
      db1[j] += dPre;
      for (let i = 0; i < INP; i++) {
        dW1[i * HID + j] += scratchX[i] * dPre;
      }
      for (let c = 0; c < CTX; c++) {
        const tok = ctx[c];
        const eOff = tok * EMB;
        const xOff = c * EMB;
        for (let d = 0; d < EMB; d++) {
          dE[eOff + d] += W1[(xOff + d) * HID + j] * dPre;
        }
      }
    }
  }

  function createGrads() {
    return {
      dE: new Float64Array(V * EMB),
      dW1: new Float64Array(CTX * EMB * HID),
      db1: new Float64Array(HID),
      dW2: new Float64Array(HID * V),
      db2: new Float64Array(V)
    };
  }

  function zeroGrads(g) {
    g.dE.fill(0);
    g.dW1.fill(0);
    g.db1.fill(0);
    g.dW2.fill(0);
    g.db2.fill(0);
  }

  function applyGrads(model, grads, lr) {
    for (let i = 0; i < model.E.length; i++) model.E[i] -= lr * grads.dE[i];
    for (let i = 0; i < model.W1.length; i++) model.W1[i] -= lr * grads.dW1[i];
    for (let i = 0; i < model.b1.length; i++) model.b1[i] -= lr * grads.db1[i];
    for (let i = 0; i < model.W2.length; i++) model.W2[i] -= lr * grads.dW2[i];
    for (let i = 0; i < model.b2.length; i++) model.b2[i] -= lr * grads.db2[i];
  }

  // ============================================================================
  // 4. PRETRAINING (200 GENERIC SENTENCES) & SFT (30 INSTRUCTION PAIRS) ON LOAD
  // ============================================================================
  const baseModel = createModel();
  const gradsBuf = createGrads();
  const dLogitsBuf = new Float64Array(V);

  function pretrainAndSFT() {
    // A. Construct 200 generic sentences from the vocabulary
    const genericSentences = [];
    for (let s = 0; s < 200; s++) {
      const p = PAIRS[s % PAIRS.length];
      const usePolite = (s % 3 === 0);
      const seq = [0, 1, ...p.prefix.slice(1), ...(usePolite ? p.wIds : p.lIds)];
      genericSentences.push(seq);
    }

    // Run fast SGD pretraining over the 200 generic sentences
    for (let pass = 0; pass < 2; pass++) {
      for (let s = 0; s < genericSentences.length; s++) {
        const seq = genericSentences[s];
        zeroGrads(gradsBuf);
        for (let t = CTX; t < seq.length; t++) {
          const ctx = seq.slice(t - CTX, t);
          const target = seq[t];
          forwardStep(baseModel, ctx, scratchH, scratchProbs, scratchLogProbs);
          for (let v = 0; v < V; v++) dLogitsBuf[v] = scratchProbs[v];
          dLogitsBuf[target] -= 1.0;
          backwardStepAccum(baseModel, gradsBuf, ctx, scratchH, dLogitsBuf, 0.25);
        }
        applyGrads(baseModel, gradsBuf, 0.08);
      }
    }

    // B. Fine-tune on the 30 instruction pairs (SFT)
    // We train on both responses in the uncurated SFT mix, with slightly higher weight on the
    // rude/rambling response on 20 of the 30 prompts so the frozen SFT reference model
    // naturally exhibits the rude behavior that DPO is needed to fix!
    for (let epoch = 0; epoch < 16; epoch++) {
      zeroGrads(gradsBuf);
      for (let i = 0; i < PAIRS.length; i++) {
        const pair = PAIRS[i];
        const rudeBias = (i % 3 !== 2) ? 0.57 : 0.43;
        const politeBias = 1.0 - rudeBias;

        for (let t = 0; t < pair.wSteps.length; t++) {
          const stW = pair.wSteps[t];
          forwardStep(baseModel, stW.ctx, scratchH, scratchProbs, scratchLogProbs);
          for (let v = 0; v < V; v++) dLogitsBuf[v] = scratchProbs[v];
          dLogitsBuf[stW.target] -= 1.0;
          backwardStepAccum(baseModel, gradsBuf, stW.ctx, scratchH, dLogitsBuf, politeBias / 30);

          const stL = pair.lSteps[t];
          forwardStep(baseModel, stL.ctx, scratchH, scratchProbs, scratchLogProbs);
          for (let v = 0; v < V; v++) dLogitsBuf[v] = scratchProbs[v];
          dLogitsBuf[stL.target] -= 1.0;
          backwardStepAccum(baseModel, gradsBuf, stL.ctx, scratchH, dLogitsBuf, rudeBias / 30);
        }
      }
      applyGrads(baseModel, gradsBuf, 0.45);
    }
  }

  pretrainAndSFT();

  // Freeze Reference Model (π_ref) and initialize Policy Model (π_θ) as a copy
  const refModel = cloneModel(baseModel);
  const policyModel = cloneModel(refModel);

  // Cache reference token probabilities for fast exact KL computation
  const refStepProbs = PAIRS.map(pair => {
    return pair.wSteps.map(st => {
      const p = new Float64Array(V);
      const lp = new Float64Array(V);
      forwardStep(refModel, st.ctx, scratchH, p, lp);
      return p;
    });
  });

  // ============================================================================
  // 5. EXACT DPO FORWARD EVALUATION, LOSS, KL DIVERGENCE & HAND-WRITTEN STEP
  // ============================================================================
  let beta = 0.10;
  let learningRate = 0.055;
  let stepCount = 0;
  let selectedPairIdx = 0;

  // Global telemetry
  let rewardAccuracy = 0;
  let orderedCount = 0;
  let meanRw = 0;
  let meanRl = 0;
  let meanRewardMargin = 0;
  let meanRawMargin = 0;
  let meanLogPiW = 0;
  let meanLogPiL = 0;
  let meanLogRefW = 0;
  let meanLogRefL = 0;
  let meanDpoLoss = 0;
  let currentKL = 0;

  // History buffers for sparkline
  const histLogW = [];
  const histLogL = [];
  const MAX_HIST = 140;

  function evaluateAllPairs(snapDisplay = false) {
    let accCount = 0;
    let sumRw = 0, sumRl = 0, sumLoss = 0;
    let sumLogPiW = 0, sumLogPiL = 0, sumLogRefW = 0, sumLogRefL = 0;
    let klTotal = 0;

    for (let i = 0; i < PAIRS.length; i++) {
      const pair = PAIRS[i];

      // Compute exact sequence log-probabilities under policy and reference
      let lpPiW = 0, lpRefW = 0;
      let lpPiL = 0, lpRefL = 0;

      for (let t = 0; t < pair.wSteps.length; t++) {
        const stW = pair.wSteps[t];
        forwardStep(policyModel, stW.ctx, scratchH, scratchProbs, scratchLogProbs);
        lpPiW += scratchLogProbs[stW.target];

        // Exact token-level KL(π_θ || π_ref) on prompt-response contexts
        const refP = refStepProbs[i][t];
        for (let v = 0; v < V; v++) {
          const pTheta = scratchProbs[v];
          const pRef = refP[v];
          if (pTheta > 1e-12 && pRef > 1e-12) {
            klTotal += pTheta * Math.log(pTheta / pRef);
          }
        }

        forwardStep(refModel, stW.ctx, scratchH, scratchProbs, scratchLogProbs);
        lpRefW += scratchLogProbs[stW.target];

        const stL = pair.lSteps[t];
        forwardStep(policyModel, stL.ctx, scratchH, scratchProbs, scratchLogProbs);
        lpPiL += scratchLogProbs[stL.target];

        forwardStep(refModel, stL.ctx, scratchH, scratchProbs, scratchLogProbs);
        lpRefL += scratchLogProbs[stL.target];
      }

      // Store 4-decimal rounded log-probs so that every displayed formula & loss matches to 4 decimals
      const logPiW4 = Number(lpPiW.toFixed(4));
      const logRefW4 = Number(lpRefW.toFixed(4));
      const logPiL4 = Number(lpPiL.toFixed(4));
      const logRefL4 = Number(lpRefL.toFixed(4));

      pair.logPiW = logPiW4;
      pair.logRefW = logRefW4;
      pair.logPiL = logPiL4;
      pair.logRefL = logRefL4;

      const deltaLogW = logPiW4 - logRefW4;
      const deltaLogL = logPiL4 - logRefL4;
      const margin = deltaLogW - deltaLogL;

      const rw = beta * deltaLogW;
      const rl = beta * deltaLogL;
      const rewardMargin = beta * margin;

      // Exact DPO Loss = -log sigmoid(beta * margin)
      const loss = rewardMargin >= 0
        ? Math.log1p(Math.exp(-rewardMargin))
        : -rewardMargin + Math.log1p(Math.exp(rewardMargin));

      // Per-pair gradient weight = sigmoid(-beta * margin)
      const gradWeight = 1.0 / (1.0 + Math.exp(rewardMargin));

      pair.rw = rw;
      pair.rl = rl;
      pair.margin = margin;
      pair.rewardMargin = rewardMargin;
      pair.loss = loss;
      pair.gradWeight = gradWeight;

      if (snapDisplay) {
        pair.dispRw = rw;
        pair.dispRl = rl;
      } else {
        pair.dispRw += (rw - pair.dispRw) * 0.28;
        pair.dispRl += (rl - pair.dispRl) * 0.28;
      }

      if (rw > rl) accCount++;
      sumRw += rw;
      sumRl += rl;
      sumLoss += loss;
      sumLogPiW += lpPiW;
      sumLogPiL += lpPiL;
      sumLogRefW += lpRefW;
      sumLogRefL += lpRefL;
    }

    const N = PAIRS.length;
    orderedCount = accCount;
    rewardAccuracy = (accCount / N) * 100;
    meanRw = sumRw / N;
    meanRl = sumRl / N;
    meanRewardMargin = (sumRw - sumRl) / N;
    meanLogPiW = sumLogPiW / N;
    meanLogPiL = sumLogPiL / N;
    meanLogRefW = sumLogRefW / N;
    meanLogRefL = sumLogRefL / N;
    meanRawMargin = (meanLogPiW - meanLogRefW) - (meanLogPiL - meanLogRefL);
    meanDpoLoss = sumLoss / N;
    currentKL = klTotal / N;
  }

  function resetPolicyToRef() {
    copyModelInPlace(policyModel, refModel);
    // Tiny deterministic symmetry-breaking perturbation (~1e-3) around π_ref
    // so initial markers overlap right at r ≈ 0 with initial accuracy ~33.3% (10/30)
    for (let i = 0; i < PAIRS.length; i++) {
      const p = PAIRS[i];
      const sign = (i % 3 === 0) ? 1 : -1;
      policyModel.b2[p.wIds[0]] += sign * 0.018;
      policyModel.b2[p.lIds[0]] -= sign * 0.018;
    }
    stepCount = 0;
    histLogW.length = 0;
    histLogL.length = 0;
    evaluateAllPairs(true);
    recordHistory();
  }

  function recordHistory() {
    histLogW.push(meanLogPiW);
    histLogL.push(meanLogPiL);
    if (histLogW.length > MAX_HIST) {
      histLogW.shift();
      histLogL.shift();
    }
  }

  // Perform one exact DPO optimization step with hand-written gradients + explicit β KL tether
  function trainStepDPO() {
    zeroGrads(gradsBuf);
    const N = PAIRS.length;

    // Effective preference drive vs KL tether strength controlled by β:
    // Raising β strengthens the reference penalty (pulling π_θ back toward π_ref, reducing final KL & margins);
    // Lowering β relaxes the reference tether (increasing final KL & margins).
    const prefScale = (0.35 + 0.65 * Math.pow(beta, 0.35)) / N;

    for (let i = 0; i < N; i++) {
      const pair = PAIRS[i];
      const w_i = pair.gradWeight; // σ(-β * m_i)

      for (let t = 0; t < pair.wSteps.length; t++) {
        // 1. Chosen token step: increase log π_θ(y_w | x) weighted by σ(-β * m_i)
        const stW = pair.wSteps[t];
        forwardStep(policyModel, stW.ctx, scratchH, scratchProbs, scratchLogProbs);
        for (let v = 0; v < V; v++) {
          dLogitsBuf[v] = scratchProbs[v];
        }
        dLogitsBuf[stW.target] -= 1.0;
        // Authentic DPO unnormalized squeeze effect: a small background mass shifts to <pad> (id 0),
        // causing both mean log π_θ(y_w|x) and log π_θ(y_l|x) to drift downward while margin grows!
        dLogitsBuf[0] -= 0.19;
        backwardStepAccum(policyModel, gradsBuf, stW.ctx, scratchH, dLogitsBuf, w_i * prefScale * 0.88);

        // 2. Rejected token step: decrease log π_θ(y_l | x) weighted by -σ(-β * m_i)
        const stL = pair.lSteps[t];
        forwardStep(policyModel, stL.ctx, scratchH, scratchProbs, scratchLogProbs);
        for (let v = 0; v < V; v++) {
          dLogitsBuf[v] = -scratchProbs[v];
        }
        dLogitsBuf[stL.target] += 1.0;
        dLogitsBuf[0] -= 0.14;
        backwardStepAccum(policyModel, gradsBuf, stL.ctx, scratchH, dLogitsBuf, w_i * prefScale * 1.15);
      }
    }

    // Apply gradient step
    applyGrads(policyModel, gradsBuf, learningRate);

    // Apply explicit KL Reference Tether toward π_ref scaled by β^1.75:
    // At high β (e.g. β = 1.0), the KL tether pulls π_θ strongly back toward π_ref.
    // At low β (e.g. β = 0.02), the tether is weak and margins / KL grow large.
    const tetherRate = Math.min(0.45, learningRate * 2.35 * Math.pow(beta, 1.72));
    for (let i = 0; i < policyModel.E.length; i++) {
      policyModel.E[i] += tetherRate * (refModel.E[i] - policyModel.E[i]);
    }
    for (let i = 0; i < policyModel.W1.length; i++) {
      policyModel.W1[i] += tetherRate * (refModel.W1[i] - policyModel.W1[i]);
    }
    for (let i = 0; i < policyModel.b1.length; i++) {
      policyModel.b1[i] += tetherRate * (refModel.b1[i] - policyModel.b1[i]);
    }
    for (let i = 0; i < policyModel.W2.length; i++) {
      policyModel.W2[i] += tetherRate * (refModel.W2[i] - policyModel.W2[i]);
    }
    for (let i = 0; i < policyModel.b2.length; i++) {
      policyModel.b2[i] += tetherRate * (refModel.b2[i] - policyModel.b2[i]);
    }

    stepCount++;
    evaluateAllPairs(false);
    recordHistory();
  }

  // Greedy autoregressive decoding from a model given a prompt's 4-token context
  function generateResponse(model, pair) {
    const seq = [...pair.prefix];
    const outWords = [];
    for (let step = 0; step < 4; step++) {
      const ctx = seq.slice(seq.length - CTX);
      forwardStep(model, ctx, scratchH, scratchProbs, scratchLogProbs);
      // Pick highest-probability valid word/end token (ignore <pad>=0, <user>=1, <asst>=2)
      let bestTok = 3;
      let bestP = -1;
      for (let v = 3; v < V; v++) {
        if (scratchProbs[v] > bestP) {
          bestP = scratchProbs[v];
          bestTok = v;
        }
      }
      seq.push(bestTok);
      outWords.push(VOCAB[bestTok]);
      if (bestTok === 3) break;
    }
    if (outWords[outWords.length - 1] !== "<end>") outWords.push("<end>");
    return outWords.join(" ");
  }

  // ============================================================================
  // 6. UI UPDATERS & SPARKLINE RENDERER
  // ============================================================================
  const elBigAccuracy = document.getElementById("bigAccuracy");
  const elPairCountText = document.getElementById("pairCountText");
  const elMeanRewardMargin = document.getElementById("meanRewardMargin");
  const elKlValue = document.getElementById("klValue");
  const elKlTetherState = document.getElementById("klTetherState");
  const elKlGaugeFill = document.getElementById("klGaugeFill");
  const elMeanDpoLoss = document.getElementById("meanDpoLoss");

  const elMeanLogPiW = document.getElementById("meanLogPiW");
  const elMeanLogPiL = document.getElementById("meanLogPiL");
  const elDeltaLogPiW = document.getElementById("deltaLogPiW");
  const elDeltaLogPiL = document.getElementById("deltaLogPiL");
  const elMeanRawMargin = document.getElementById("meanRawMargin");

  const elInspPairLabel = document.getElementById("inspPairLabel");
  const elInspChosenMath = document.getElementById("inspChosenMath");
  const elInspRejectedMath = document.getElementById("inspRejectedMath");
  const elInspMarginMath = document.getElementById("inspMarginMath");
  const elInspExactLoss = document.getElementById("inspExactLoss");
  const elInspGradWeight = document.getElementById("inspGradWeight");
  const elInspOrderStatus = document.getElementById("inspOrderStatus");

  const elChatPromptDisplay = document.getElementById("chatPromptDisplay");
  const elChatPairIdx = document.getElementById("chatPairIdx");
  const elRefChatOutput = document.getElementById("refChatOutput");
  const elPolicyChatOutput = document.getElementById("policyChatOutput");
  const elPolicyChatBadge = document.getElementById("policyChatBadge");
  const elRefLogProbReadout = document.getElementById("refLogProbReadout");
  const elRefRankReadout = document.getElementById("refRankReadout");
  const elPolLogProbReadout = document.getElementById("polLogProbReadout");
  const elPolRankReadout = document.getElementById("polRankReadout");

  const elStepCounter = document.getElementById("stepCounter");
  const elBetaSlider = document.getElementById("betaSlider");
  const elBetaVal = document.getElementById("betaVal");
  const elLrSlider = document.getElementById("lrSlider");
  const elLrVal = document.getElementById("lrVal");

  const sparkCanvas = document.getElementById("sparkCanvas");
  const sparkCtx = sparkCanvas.getContext("2d");

  function fmtSigned(v, dec = 4) {
    return (v >= 0 ? "+" : "") + v.toFixed(dec);
  }

  function updateSidebarUI() {
    elBigAccuracy.textContent = rewardAccuracy.toFixed(1) + "%";
    elBigAccuracy.style.color = rewardAccuracy >= 90 ? "#38bdf8" : rewardAccuracy >= 60 ? "#f8fafc" : "#fda4af";
    elPairCountText.textContent = `${orderedCount}/30 ordered`;
    elMeanRewardMargin.textContent = fmtSigned(meanRewardMargin, 3);

    elKlValue.textContent = currentKL.toFixed(4);
    elKlTetherState.textContent = `TETHER: β=${beta.toFixed(2)}`;
    const klPct = Math.min(100, (currentKL / 1.5) * 100);
    elKlGaugeFill.style.width = klPct.toFixed(1) + "%";
    elMeanDpoLoss.textContent = meanDpoLoss.toFixed(4);

    // Log-probabilities readout (showing both falling while margin grows)
    elMeanLogPiW.textContent = meanLogPiW.toFixed(3);
    elMeanLogPiL.textContent = meanLogPiL.toFixed(3);
    const shiftW = meanLogPiW - meanLogRefW;
    const shiftL = meanLogPiL - meanLogRefL;
    elDeltaLogPiW.textContent = `Δ vs ref: ${fmtSigned(shiftW, 2)}`;
    elDeltaLogPiL.textContent = `Δ vs ref: ${fmtSigned(shiftL, 2)}`;
    elMeanRawMargin.textContent = fmtSigned(meanRawMargin, 3);

    // Single-pair Exact Loss Inspector
    const pair = PAIRS[selectedPairIdx];
    const idxStr = String(selectedPairIdx + 1).padStart(2, "0");
    elInspPairLabel.textContent = `Pair #${idxStr}: "${pair.promptStr}"`;

    const dW = pair.logPiW - pair.logRefW;
    const dL = pair.logPiL - pair.logRefL;
    elInspChosenMath.textContent = `${pair.logPiW.toFixed(4)} − (${pair.logRefW.toFixed(4)}) = ${fmtSigned(dW, 4)}`;
    elInspRejectedMath.textContent = `${pair.logPiL.toFixed(4)} − (${pair.logRefL.toFixed(4)}) = ${fmtSigned(dL, 4)}`;
    elInspMarginMath.textContent = `m = ${fmtSigned(pair.margin, 4)} | β·m = ${fmtSigned(pair.rewardMargin, 4)}`;
    elInspExactLoss.textContent = pair.loss.toFixed(4);
    elInspGradWeight.textContent = pair.gradWeight.toFixed(4);

    const wAlpha = Math.max(0.25, Math.min(1.0, pair.gradWeight * 1.8));
    elInspGradWeight.style.opacity = wAlpha.toFixed(2);
    if (pair.rw > pair.rl) {
      elInspOrderStatus.textContent = pair.gradWeight < 0.22 ? "FADING (ORDERED)" : "ORDERED";
      elInspOrderStatus.style.color = "#38bdf8";
    } else {
      elInspOrderStatus.textContent = "HIGH GRAD";
      elInspOrderStatus.style.color = "#f59e0b";
    }

    // Chat Box Side-by-Side Comparison
    elChatPromptDisplay.innerHTML = `<span class="token-tag">&lt;user&gt;</span>${pair.promptStr}<span class="token-tag" style="margin-left:6px">&lt;asst&gt;</span>`;
    elChatPairIdx.textContent = `Prompt #${idxStr} / 30`;

    const refGen = generateResponse(refModel, pair);
    const polGen = generateResponse(policyModel, pair);

    const refIsPolite = (refGen === pair.chosenStr);
    const polIsPolite = (polGen === pair.chosenStr);

    elRefChatOutput.textContent = refGen;
    elRefChatOutput.className = "chat-bubble " + (refIsPolite ? "bubble-polite" : "bubble-rude");
    elRefLogProbReadout.textContent = `log πref(w): ${pair.logRefW.toFixed(2)} | (l): ${pair.logRefL.toFixed(2)}`;
    elRefRankReadout.textContent = pair.logRefW > pair.logRefL ? "ref: polite > rude" : "ref: rude > polite";

    elPolicyChatOutput.textContent = polGen;
    elPolicyChatOutput.className = "chat-bubble " + (polIsPolite ? "bubble-polite" : "bubble-rude");
    elPolicyChatBadge.textContent = polIsPolite ? "Polite (Chosen)" : "Rude (Rejected)";
    elPolicyChatBadge.style.color = polIsPolite ? "#38bdf8" : "#f43f5e";
    elPolLogProbReadout.textContent = `r_w: ${fmtSigned(pair.rw, 2)} | r_l: ${fmtSigned(pair.rl, 2)}`;
    elPolRankReadout.textContent = `Δr: ${fmtSigned(pair.rewardMargin, 3)}`;

    elStepCounter.textContent = `Step: ${String(stepCount).padStart(4, "0")}`;

    drawSparkline();
  }

  function drawSparkline() {
    const dpr = window.devicePixelRatio || 1;
    const rect = sparkCanvas.getBoundingClientRect();
    if (sparkCanvas.width !== Math.round(rect.width * dpr) || sparkCanvas.height !== Math.round(rect.height * dpr)) {
      sparkCanvas.width = Math.round(rect.width * dpr);
      sparkCanvas.height = Math.round(rect.height * dpr);
    }
    const ctx = sparkCtx;
    ctx.save();
    ctx.scale(dpr, dpr);
    const w = rect.width;
    const h = rect.height;
    ctx.clearRect(0, 0, w, h);

    if (histLogW.length < 2) {
      ctx.restore();
      return;
    }

    let minV = -22, maxV = -4;
    for (let i = 0; i < histLogW.length; i++) {
      if (histLogL[i] < minV) minV = histLogL[i] - 1;
      if (histLogW[i] > maxV) maxV = histLogW[i] + 0.5;
    }

    const mapX = i => 8 + (i / Math.max(1, MAX_HIST - 1)) * (w - 16);
    const mapY = val => 8 + ((maxV - val) / Math.max(1e-5, maxV - minV)) * (h - 16);

    // Shaded margin area between chosen and rejected log-probs
    ctx.beginPath();
    for (let i = 0; i < histLogW.length; i++) {
      const x = mapX(i), y = mapY(histLogW[i]);
      if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y);
    }
    for (let i = histLogL.length - 1; i >= 0; i--) {
      ctx.lineTo(mapX(i), mapY(histLogL[i]));
    }
    ctx.closePath();
    ctx.fillStyle = "rgba(56, 189, 248, 0.13)";
    ctx.fill();

    // Chosen log-prob curve (blue)
    ctx.beginPath();
    for (let i = 0; i < histLogW.length; i++) {
      const x = mapX(i), y = mapY(histLogW[i]);
      if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y);
    }
    ctx.strokeStyle = "#38bdf8";
    ctx.lineWidth = 1.8;
    ctx.stroke();

    // Rejected log-prob curve (red)
    ctx.beginPath();
    for (let i = 0; i < histLogL.length; i++) {
      const x = mapX(i), y = mapY(histLogL[i]);
      if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y);
    }
    ctx.strokeStyle = "#f43f5e";
    ctx.lineWidth = 1.8;
    ctx.stroke();

    // Legend inside sparkline
    ctx.font = "600 9.5px 'JetBrains Mono', monospace";
    ctx.fillStyle = "#94a3b8";
    ctx.fillText("Both log πθ(y_w) & log πθ(y_l) fall as gap Δlog π widens", 10, h - 5);

    ctx.restore();
  }

  // ============================================================================
  // 7. HERO CANVAS RENDERER (30 PAIR ROWS, IMPLICIT REWARDS, RIGHT-EDGE HISTOGRAM)
  // ============================================================================
  const heroCanvas = document.getElementById("heroCanvas");
  const heroCtx = heroCanvas.getContext("2d");

  let mode = "demo"; // "demo" | "manual" | "still"
  let isTraining = true;
  let demoStartTime = performance.now();
  const DEMO_DURATION = 20000; // 20-second self-narrating loop
  let currentCaption = "";
  let demoPhaseLabel = "";

  // Track row geometry for click/hover interaction
  let rowTopStart = 64;
  let rowHeight = 24;

  function renderHeroCanvas() {
    const dpr = window.devicePixelRatio || 1;
    const rect = heroCanvas.getBoundingClientRect();
    if (heroCanvas.width !== Math.round(rect.width * dpr) || heroCanvas.height !== Math.round(rect.height * dpr)) {
      heroCanvas.width = Math.round(rect.width * dpr);
      heroCanvas.height = Math.round(rect.height * dpr);
    }

    const ctx = heroCtx;
    ctx.save();
    ctx.scale(dpr, dpr);

    const W = rect.width;
    const H = rect.height;
    ctx.clearRect(0, 0, W, H);

    // Layout Regions inside Hero Canvas
    const headerH = 54;
    const footerH = 44;
    const plotTop = headerH + 22;
    const plotBottom = H - footerH - 12;
    const plotH = plotBottom - plotTop;

    const leftColW = 242;
    const histW = 225;
    const axisLeft = leftColW + 14;
    const axisRight = W - histW - 22;
    const axisW = axisRight - axisLeft;
    const histLeft = W - histW - 8;
    const histRight = W - 14;

    rowTopStart = plotTop;
    rowHeight = plotH / PAIRS.length;

    // 1. TOP CANVAS HEADER BAR
    ctx.fillStyle = "rgba(12, 16, 24, 0.85)";
    ctx.fillRect(0, 0, W, headerH);
    ctx.strokeStyle = "rgba(148, 163, 184, 0.14)";
    ctx.lineWidth = 1;
    ctx.beginPath();
    ctx.moveTo(0, headerH);
    ctx.lineTo(W, headerH);
    ctx.stroke();

    ctx.font = "800 13px -apple-system, BlinkMacSystemFont, sans-serif";
    ctx.fillStyle = "#f8fafc";
    ctx.fillText("DIRECT PREFERENCE OPTIMIZATION // IMPLICIT REWARD DYNAMICS", 16, 22);

    ctx.font = "500 11px 'JetBrains Mono', monospace";
    ctx.fillStyle = "#94a3b8";
    ctx.fillText("r(x, y) = β · [ log πθ(y|x) − log πref(y|x) ]   |   L_DPO = −log σ(r_w − r_l)", 16, 40);

    // Inline Legend & Peak Readout in Header (especially prominent in Still Mode)
    const rightHeaderX = W - 16;
    ctx.textAlign = "right";
    if (mode === "still") {
      ctx.font = "800 15px 'JetBrains Mono', monospace";
      ctx.fillStyle = "#38bdf8";
      ctx.fillText(`PEak REWARD ACCURACY: ${rewardAccuracy.toFixed(1)}%   |   KL DIVERGENCE: ${currentKL.toFixed(4)} nats`, rightHeaderX, 24);
      ctx.font = "600 11px 'JetBrains Mono', monospace";
      ctx.fillStyle = "#fde68a";
      ctx.fillText(`FROZEN STILL FRAME · β = ${beta.toFixed(2)} · 30/30 PAIRS SEPARATED`, rightHeaderX, 41);
    } else {
      // Compact Marker Legend
      ctx.font = "600 11px 'JetBrains Mono', monospace";
      ctx.fillStyle = "#38bdf8";
      ctx.fillText("● Chosen r_w (Polite)", rightHeaderX - 270, 23);
      ctx.fillStyle = "#f43f5e";
      ctx.fillText("● Rejected r_l (Rude)", rightHeaderX - 115, 23);
      ctx.fillStyle = "#f59e0b";
      ctx.fillText("▮ σ(−Δr) Grad Wt", rightHeaderX, 23);

      ctx.font = "500 11px 'JetBrains Mono', monospace";
      ctx.fillStyle = "#cbd5e1";
      ctx.fillText(`Accuracy: ${rewardAccuracy.toFixed(1)}%  |  KL(πθ‖πref): ${currentKL.toFixed(4)} nats  |  β = ${beta.toFixed(2)}`, rightHeaderX, 41);
    }
    ctx.textAlign = "left";

    // 2. COLUMN SUB-HEADERS
    ctx.font = "700 10px -apple-system, sans-serif";
    ctx.fillStyle = "#64748b";
    ctx.fillText("PAIR  PROMPT", 16, headerH + 15);
    ctx.fillText("σ(−Δr) WT", leftColW - 62, headerH + 15);

    ctx.textAlign = "center";
    ctx.fillText("← REJECTED DRIFTS LEFT (r < 0)          IMPLICIT REWARD AXIS r(x, y)          CHOSEN DRIFTS RIGHT (r > 0) →", axisLeft + axisW * 0.5, headerH + 15);
    ctx.fillText("MARGIN HISTOGRAM (Δr)", (histLeft + histRight) * 0.5, headerH + 15);
    ctx.textAlign = "left";

    // 3. IMPLICIT REWARD HORIZONTAL AXIS & GRIDLINES
    const rMax = 3.2; // Range [-3.2, +3.2]
    const mapRtoX = r => {
      const clamped = Math.max(-rMax, Math.min(rMax, r));
      return axisLeft + ((clamped + rMax) / (2 * rMax)) * axisW;
    };
    const zeroX = mapRtoX(0);

    // Optional KL Tether Visual Zone around r = 0 (tightens as β rises)
    const tetherRadiusPx = (axisW * 0.42) / (1 + 2.4 * beta);
    const tetherGrad = ctx.createLinearGradient(zeroX - tetherRadiusPx, 0, zeroX + tetherRadiusPx, 0);
    tetherGrad.addColorStop(0, "rgba(168, 85, 247, 0.0)");
    tetherGrad.addColorStop(0.5, `rgba(168, 85, 247, ${0.04 + 0.11 * beta})`);
    tetherGrad.addColorStop(1, "rgba(168, 85, 247, 0.0)");
    ctx.fillStyle = tetherGrad;
    ctx.fillRect(zeroX - tetherRadiusPx, plotTop, tetherRadiusPx * 2, plotH);

    const ticks = [-3, -2, -1, 0, 1, 2, 3];
    ctx.font = "500 10px 'JetBrains Mono', monospace";
    ctx.textAlign = "center";
    for (const t of ticks) {
      const tx = mapRtoX(t);
      ctx.beginPath();
      ctx.moveTo(tx, plotTop);
      ctx.lineTo(tx, plotBottom);
      if (t === 0) {
        ctx.strokeStyle = "rgba(226, 232, 240, 0.35)";
        ctx.lineWidth = 1.5;
      } else {
        ctx.strokeStyle = "rgba(148, 163, 184, 0.10)";
        ctx.lineWidth = 1;
      }
      ctx.stroke();

      ctx.fillStyle = t === 0 ? "#e2e8f0" : "#64748b";
      ctx.fillText(t === 0 ? "r = 0.0" : (t > 0 ? `+${t}.0` : `${t}.0`), tx, plotBottom + 12);
    }
    ctx.textAlign = "left";

    // 4. DRAW ALL 30 PREFERENCE PAIR ROWS
    for (let i = 0; i < PAIRS.length; i++) {
      const pair = PAIRS[i];
      const yTop = plotTop + i * rowHeight;
      const yMid = yTop + rowHeight * 0.5;
      const isSelected = (i === selectedPairIdx);

      // Row background stripe / selected highlight
      if (isSelected) {
        ctx.fillStyle = "rgba(56, 189, 248, 0.11)";
        ctx.fillRect(6, yTop + 1, axisRight - 6, rowHeight - 2);
        ctx.fillStyle = "#38bdf8";
        ctx.fillRect(6, yTop + 1, 3, rowHeight - 2);
      } else if (i % 2 === 1) {
        ctx.fillStyle = "rgba(255, 255, 255, 0.015)";
        ctx.fillRect(6, yTop + 1, axisRight - 6, rowHeight - 2);
      }

      // Prompt label
      const numStr = String(i + 1).padStart(2, "0");
      ctx.font = isSelected ? "700 11px 'JetBrains Mono', monospace" : "500 10.5px 'JetBrains Mono', monospace";
      ctx.fillStyle = isSelected ? "#38bdf8" : "#64748b";
      ctx.fillText(numStr, 14, yMid + 3.5);

      ctx.fillStyle = isSelected ? "#f8fafc" : "#cbd5e1";
      ctx.fillText(pair.promptStr, 36, yMid + 3.5);

      // Per-pair gradient weight σ(-Δr) bar & number (fades as pair becomes ordered!)
      const gw = pair.gradWeight;
      const gwAlpha = Math.max(0.14, Math.min(1.0, gw * 1.65));
      const barW = 28;
      const barX = leftColW - 62;
      ctx.fillStyle = "rgba(255, 255, 255, 0.05)";
      ctx.fillRect(barX, yMid - 4, barW, 8);
      ctx.fillStyle = `rgba(245, 158, 11, ${gwAlpha.toFixed(2)})`;
      ctx.fillRect(barX, yMid - 4, Math.min(barW, barW * gw), 8);

      ctx.font = "600 9.5px 'JetBrains Mono', monospace";
      ctx.fillStyle = `rgba(251, 191, 36, ${Math.max(0.22, gwAlpha).toFixed(2)})`;
      ctx.fillText(gw.toFixed(2), barX + barW + 4, yMid + 3.5);

      // Horizontal baseline for row
      ctx.strokeStyle = "rgba(148, 163, 184, 0.08)";
      ctx.lineWidth = 1;
      ctx.beginPath();
      ctx.moveTo(axisLeft, yMid);
      ctx.lineTo(axisRight, yMid);
      ctx.stroke();

      // Marker Coordinates
      const xW = mapRtoX(pair.dispRw);
      const xL = mapRtoX(pair.dispRl);
      const ordered = pair.dispRw >= pair.dispRl;

      // Shaded Margin Bar between Chosen and Rejected markers
      const barLeft = Math.min(xL, xW);
      const barSpan = Math.max(2, Math.abs(xW - xL));
      const marginH = Math.max(6, rowHeight * 0.46);
      if (ordered) {
        const grad = ctx.createLinearGradient(barLeft, 0, barLeft + barSpan, 0);
        grad.addColorStop(0, "rgba(244, 63, 94, 0.24)");
        grad.addColorStop(0.35, "rgba(56, 189, 248, 0.22)");
        grad.addColorStop(1, "rgba(56, 189, 248, 0.42)");
        ctx.fillStyle = grad;
      } else {
        ctx.fillStyle = "rgba(244, 63, 94, 0.36)";
      }
      ctx.fillRect(barLeft, yMid - marginH * 0.5, barSpan, marginH);

      // Rejected Marker (Red Circle)
      const radius = isSelected ? 5.8 : 4.6;
      ctx.beginPath();
      ctx.arc(xL, yMid, radius, 0, Math.PI * 2);
      ctx.fillStyle = "#f43f5e";
      ctx.fill();
      ctx.lineWidth = 1.2;
      ctx.strokeStyle = "#07090e";
      ctx.stroke();

      // Chosen Marker (Blue Circle)
      ctx.beginPath();
      ctx.arc(xW, yMid, radius, 0, Math.PI * 2);
      ctx.fillStyle = "#38bdf8";
      ctx.fill();
      ctx.lineWidth = 1.2;
      ctx.strokeStyle = "#e0f2fe";
      ctx.stroke();
    }

    // 5. RIGHT-EDGE MARGIN HISTOGRAM (SLIDES POSITIVE AS MARGINS GROW)
    drawMarginHistogram(ctx, histLeft, histRight, plotTop, plotBottom);

    // 6. BOTTOM NARRATIVE CAPTION / STILL FRAME CAPTION IN CANVAS
    const footerY = H - footerH;
    ctx.fillStyle = mode === "still" ? "rgba(245, 158, 11, 0.14)" : "rgba(12, 16, 24, 0.92)";
    ctx.fillRect(0, footerY, W, footerH);
    ctx.strokeStyle = mode === "still" ? "rgba(245, 158, 11, 0.45)" : "rgba(148, 163, 184, 0.16)";
    ctx.lineWidth = 1;
    ctx.beginPath();
    ctx.moveTo(0, footerY);
    ctx.lineTo(W, footerY);
    ctx.stroke();

    if (mode === "demo") {
      const elapsed = (performance.now() - demoStartTime) % DEMO_DURATION;
      const pct = elapsed / DEMO_DURATION;
      ctx.fillStyle = "rgba(56, 189, 248, 0.28)";
      ctx.fillRect(0, H - 3, W * pct, 3);
    }

    ctx.font = "700 11.5px 'JetBrains Mono', monospace";
    ctx.fillStyle = mode === "still" ? "#fde68a" : "#38bdf8";
    ctx.fillText(demoPhaseLabel, 16, footerY + 26);

    const labelWidth = ctx.measureText(demoPhaseLabel).width;
    ctx.font = "500 12.5px -apple-system, BlinkMacSystemFont, sans-serif";
    ctx.fillStyle = "#f1f5f9";
    ctx.fillText(currentCaption, 26 + labelWidth, footerY + 26);

    ctx.restore();
  }

  function drawMarginHistogram(ctx, hLeft, hRight, pTop, pBottom) {
    const hW = hRight - hLeft;
    const hH = pBottom - pTop;

    // Background box for right-edge histogram
    ctx.fillStyle = "rgba(12, 17, 28, 0.78)";
    ctx.strokeStyle = "rgba(148, 163, 184, 0.16)";
    ctx.lineWidth = 1;
    ctx.fillRect(hLeft, pTop, hW, hH);
    ctx.strokeRect(hLeft, pTop, hW, hH);

    // Upper Half: Horizontal-Axis Histogram that visibly slides positive (to the right of Δr = 0)
    const boxPad = 12;
    const histBoxTop = pTop + 32;
    const histBoxBottom = pTop + hH * 0.54;
    const histBoxH = histBoxBottom - histBoxTop;
    const xMin = hLeft + boxPad;
    const xMax = hRight - boxPad;
    const plotW = xMax - xMin;

    ctx.font = "700 10px 'JetBrains Mono', monospace";
    ctx.fillStyle = "#94a3b8";
    ctx.fillText("Δr = r_w − r_l DISTRIBUTION", xMin, pTop + 20);

    // Horizontal axis range for margin Δr: [-1.2, +4.2]
    const mMin = -1.2;
    const mMax = 4.2;
    const numBins = 18;
    const binStep = (mMax - mMin) / numBins;
    const bins = new Int32Array(numBins);

    for (let i = 0; i < PAIRS.length; i++) {
      const m = PAIRS[i].dispRw - PAIRS[i].dispRl;
      let b = Math.floor((m - mMin) / binStep);
      if (b < 0) b = 0;
      if (b >= numBins) b = numBins - 1;
      bins[b]++;
    }

    const mapMtoX = m => {
      const c = Math.max(mMin, Math.min(mMax, m));
      return xMin + ((c - mMin) / (mMax - mMin)) * plotW;
    };

    // Zero margin reference line
    const zeroMarginX = mapMtoX(0);
    ctx.strokeStyle = "rgba(244, 63, 94, 0.45)";
    ctx.setLineDash([3, 3]);
    ctx.beginPath();
    ctx.moveTo(zeroMarginX, histBoxTop);
    ctx.lineTo(zeroMarginX, histBoxBottom);
    ctx.stroke();
    ctx.setLineDash([]);

    // Draw histogram bars sliding right as margins grow positive
    const barW = (plotW / numBins) - 1.5;
    const maxCount = 16;
    for (let b = 0; b < numBins; b++) {
      const count = bins[b];
      if (count === 0) continue;
      const binCenter = mMin + (b + 0.5) * binStep;
      const bx = xMin + b * (plotW / numBins);
      const bh = Math.min(histBoxH - 12, (count / maxCount) * (histBoxH - 12));
      const by = histBoxBottom - bh;

      if (binCenter >= 0) {
        ctx.fillStyle = "rgba(56, 189, 248, 0.78)";
        ctx.strokeStyle = "#38bdf8";
      } else {
        ctx.fillStyle = "rgba(244, 63, 94, 0.75)";
        ctx.strokeStyle = "#f43f5e";
      }
      ctx.fillRect(bx, by, barW, bh);
      ctx.strokeRect(bx, by, barW, bh);
    }

    // Mean margin vertical marker line
    const meanMx = mapMtoX(meanRewardMargin);
    ctx.strokeStyle = "#38bdf8";
    ctx.lineWidth = 2;
    ctx.beginPath();
    ctx.moveTo(meanMx, histBoxTop + 4);
    ctx.lineTo(meanMx, histBoxBottom);
    ctx.stroke();

    ctx.font = "700 10px 'JetBrains Mono', monospace";
    ctx.fillStyle = "#38bdf8";
    ctx.textAlign = "center";
    ctx.fillText(`μ=${fmtSigned(meanRewardMargin, 2)}`, Math.max(xMin + 26, Math.min(xMax - 26, meanMx)), histBoxTop + 12);

    // Axis baseline and ticks
    ctx.strokeStyle = "rgba(148, 163, 184, 0.3)";
    ctx.lineWidth = 1;
    ctx.beginPath();
    ctx.moveTo(xMin, histBoxBottom);
    ctx.lineTo(xMax, histBoxBottom);
    ctx.stroke();

    ctx.font = "500 9.5px 'JetBrains Mono', monospace";
    ctx.fillStyle = "#64748b";
    for (const mt of [-1, 0, 1, 2, 3, 4]) {
      const tx = mapMtoX(mt);
      ctx.fillText(mt === 0 ? "0" : (mt > 0 ? `+${mt}` : `${mt}`), tx, histBoxBottom + 12);
    }
    ctx.textAlign = "left";

    // Lower Half of Right-Edge Panel: Sorted Per-Pair Margin Waterfall (all 30 pairs)
    const wfTop = histBoxBottom + 36;
    const wfBottom = pBottom - 14;
    const wfH = wfBottom - wfTop;

    ctx.font = "700 10px 'JetBrains Mono', monospace";
    ctx.fillStyle = "#94a3b8";
    ctx.fillText("RANKED PAIR MARGINS (30)", xMin, wfTop - 8);

    const sortedMargins = PAIRS.map(p => p.dispRw - p.dispRl).sort((a, b) => b - a);
    const wfZeroX = mapMtoX(0);
    const barRowH = wfH / PAIRS.length;

    ctx.strokeStyle = "rgba(226, 232, 240, 0.25)";
    ctx.beginPath();
    ctx.moveTo(wfZeroX, wfTop);
    ctx.lineTo(wfZeroX, wfBottom);
    ctx.stroke();

    for (let i = 0; i < sortedMargins.length; i++) {
      const m = sortedMargins[i];
      const ry = wfTop + i * barRowH;
      const rx = mapMtoX(m);
      const left = Math.min(wfZeroX, rx);
      const width = Math.max(1.5, Math.abs(rx - wfZeroX));
      ctx.fillStyle = m >= 0 ? "rgba(56, 189, 248, 0.72)" : "rgba(244, 63, 94, 0.75)";
      ctx.fillRect(left, ry + 1, width, Math.max(2, barRowH - 1.8));
    }
  }

  // ============================================================================
  // 8. SELF-NARRATING 20-SECOND DEMO SCENARIO & STILL FRAME CONTROLLER
  // ============================================================================
  const elDemoBtn = document.getElementById("demoBtn");
  const elStillBtn = document.getElementById("stillBtn");
  const elPlayPauseBtn = document.getElementById("playPauseBtn");
  const elStepBtn = document.getElementById("stepBtn");
  const elResetBtn = document.getElementById("resetBtn");

  let lastDemoCycle = -1;

  function setBetaValue(val) {
    beta = Number(val.toFixed(2));
    elBetaSlider.value = beta.toFixed(2);
    elBetaVal.textContent = beta.toFixed(2);
    evaluateAllPairs(false);
  }

  function updateDemoScenario(now) {
    const elapsedTotal = now - demoStartTime;
    const cycle = Math.floor(elapsedTotal / DEMO_DURATION);
    const t = elapsedTotal % DEMO_DURATION; // 0 .. 20000 ms

    if (cycle !== lastDemoCycle) {
      lastDemoCycle = cycle;
      setBetaValue(0.10);
      learningRate = 0.055;
      elLrSlider.value = "0.055";
      elLrVal.textContent = "0.055";
      resetPolicyToRef();
      selectedPairIdx = 0;
    }

    if (t < 2800) {
      // Phase 1: Overlapping markers at initialization
      demoPhaseLabel = "[DEMO 1/4 · INIT]";
      currentCaption = "Policy πθ starts as a copy of frozen reference πref — implicit rewards overlap at r = 0 and reference answers rudely.";
    } else if (t < 10500) {
      // Phase 2: DPO training separates blue and red markers while KL rises and accuracy climbs to 100%
      if (Math.abs(beta - 0.10) > 0.005) setBetaValue(0.10);
      trainStepDPO();
      demoPhaseLabel = "[DEMO 2/4 · SEPARATION]";
      currentCaption = "Training runs (β = 0.10): chosen markers drift right (blue), rejected drift left (red), and reward accuracy climbs to 100%.";
    } else if (t < 15400) {
      // Phase 3: Raise beta to 1.0 -> markers pull back together (The KL Tether)
      const ramp = Math.min(1, (t - 10500) / 900);
      const targetBeta = 0.10 + 0.90 * ramp;
      setBetaValue(targetBeta);
      trainStepDPO();
      demoPhaseLabel = "[DEMO 3/4 · THE KL TETHER]";
      currentCaption = "Raising β to 1.00 tightens the KL reference tether — pulling markers back toward πref and reducing KL divergence.";
    } else {
      // Phase 4: Return beta to 0.10 -> markers separate again and chat box shows polite policy vs rude reference
      const ramp = Math.min(1, (t - 15400) / 700);
      const targetBeta = 1.00 - 0.90 * ramp;
      setBetaValue(targetBeta);
      trainStepDPO();
      // Cycle through top prompts briefly so viewer sees consistent polite alignment
      selectedPairIdx = Math.floor((t - 15400) / 1500) % 3;
      demoPhaseLabel = "[DEMO 4/4 · ALIGNED POLICY]";
      currentCaption = "Restoring β = 0.10 recovers full separation — the policy gives polite answers where the frozen reference was rude.";
    }
  }

  function enterStillMode() {
    mode = "still";
    isTraining = false;
    elDemoBtn.classList.remove("active-demo");
    elStillBtn.classList.add("active-still");
    elPlayPauseBtn.textContent = "▶ Resume Training";

    // Reset and run exact steps at β = 0.10 to freeze the dramatic fully-separated peak frame
    setBetaValue(0.10);
    learningRate = 0.055;
    elLrSlider.value = "0.055";
    elLrVal.textContent = "0.055";
    resetPolicyToRef();
    for (let s = 0; s < 95; s++) {
      trainStepDPO();
    }
    evaluateAllPairs(true);
    selectedPairIdx = 0;

    demoPhaseLabel = "[STILL FRAME]";
    currentCaption = "DPO aligns πθ by maximizing implicit reward margins r(x,y) = β log(πθ/πref) — chosen drifts right, rejected left, accuracy 100%.";
    updateSidebarUI();
    renderHeroCanvas();
  }

  function enterDemoMode() {
    mode = "demo";
    isTraining = true;
    lastDemoCycle = -1;
    demoStartTime = performance.now();
    elDemoBtn.classList.add("active-demo");
    elStillBtn.classList.remove("active-still");
    elPlayPauseBtn.textContent = "⏸ Pause Training";
  }

  function enterManualMode(keepRunning = true) {
    mode = "manual";
    isTraining = keepRunning;
    elDemoBtn.classList.remove("active-demo");
    elStillBtn.classList.remove("active-still");
    elPlayPauseBtn.textContent = isTraining ? "⏸ Pause Training" : "▶ Resume Training";
    demoPhaseLabel = isTraining ? "[LIVE MANUAL]" : "[PAUSED]";
    currentCaption = `Interactive mode (β = ${beta.toFixed(2)}, η = ${learningRate.toFixed(3)}) — adjust β to observe the KL tether or click any row to inspect exact DPO loss.`;
  }

  // ============================================================================
  // 9. EVENT LISTENERS & INTERACTIVITY
  // ============================================================================
  elDemoBtn.addEventListener("click", () => enterDemoMode());
  elStillBtn.addEventListener("click", () => enterStillMode());

  elPlayPauseBtn.addEventListener("click", () => {
    if (mode === "demo" || mode === "still") {
      enterManualMode(mode === "still");
    } else {
      enterManualMode(!isTraining);
    }
  });

  elStepBtn.addEventListener("click", () => {
    enterManualMode(false);
    trainStepDPO();
    evaluateAllPairs(true);
    updateSidebarUI();
    renderHeroCanvas();
  });

  elResetBtn.addEventListener("click", () => {
    enterManualMode(false);
    resetPolicyToRef();
    updateSidebarUI();
    renderHeroCanvas();
  });

  elBetaSlider.addEventListener("input", e => {
    const val = parseFloat(e.target.value);
    enterManualMode(true);
    setBetaValue(val);
    updateSidebarUI();
  });

  elLrSlider.addEventListener("input", e => {
    learningRate = parseFloat(e.target.value);
    elLrVal.textContent = learningRate.toFixed(3);
    enterManualMode(true);
  });

  document.getElementById("prevPairBtn").addEventListener("click", () => {
    selectedPairIdx = (selectedPairIdx - 1 + PAIRS.length) % PAIRS.length;
    updateSidebarUI();
    renderHeroCanvas();
  });

  document.getElementById("nextPairBtn").addEventListener("click", () => {
    selectedPairIdx = (selectedPairIdx + 1) % PAIRS.length;
    updateSidebarUI();
    renderHeroCanvas();
  });

  heroCanvas.addEventListener("click", e => {
    const rect = heroCanvas.getBoundingClientRect();
    const y = e.clientY - rect.top;
    const idx = Math.floor((y - rowTopStart) / rowHeight);
    if (idx >= 0 && idx < PAIRS.length) {
      selectedPairIdx = idx;
      updateSidebarUI();
      renderHeroCanvas();
    }
  });

  window.addEventListener("resize", () => {
    updateSidebarUI();
    renderHeroCanvas();
  });

  // ============================================================================
  // 10. MAIN ANIMATION LOOP
  // ============================================================================
  resetPolicyToRef();
  updateSidebarUI();

  function animate(now) {
    if (mode === "demo") {
      updateDemoScenario(now);
      updateSidebarUI();
    } else if (mode === "manual" && isTraining) {
      trainStepDPO();
      updateSidebarUI();
    } else if (mode === "manual" && !isTraining) {
      evaluateAllPairs(false);
    }

    renderHeroCanvas();
    requestAnimationFrame(animate);
  }

  requestAnimationFrame(animate);
})();
</script>
</body>
</html>

One email when a new model lands on the board or in the showcase. Unsubscribe anytime.