参考：数值与机制核验

{
  "bpe": {
    "text": "ababab 你好 abab",
    "merges": [
      [
        [
          97,
          98
        ],
        256
      ],
      [
        [
          256,
          256
        ],
        257
      ],
      [
        [
          32,
          228
        ],
        258
      ],
      [
        [
          32,
          257
        ],
        259
      ]
    ],
    "encoded": [
      257,
      256,
      258,
      189,
      160,
      229,
      165,
      189,
      259
    ],
    "round_trip": true
  },
  "gqa_cache": {
    "max_full_vs_incremental_error": 6.661338147750939e-16,
    "cache_K_shape": [
      2,
      2,
      7,
      4
    ],
    "Q_heads": 4,
    "KV_heads": 2
  },
  "router": {
    "top1_then_softmax_gradient": [
      0.0,
      0.0,
      0.0
    ],
    "full_softmax_then_truncate_gradient": [
      0.47457920027138556,
      -0.3374018463508932,
      -0.13717735380947005
    ]
  },
  "lora": {
    "analytic_gradients_match_finite_difference": true
  },
  "distillation": {
    "soft_CE_KL_gradient_verified": true
  },
  "status": "All assertions passed; no production performance claims"
}

{
  "attention": {
    "Q": [
      [
        2.0,
        1.0
      ],
      [
        1.0,
        2.0
      ]
    ],
    "K": [
      [
        1.0,
        2.0
      ],
      [
        1.0,
        1.0
      ]
    ],
    "V": [
      [
        2.0,
        -1.0
      ],
      [
        1.0,
        0.0
      ]
    ],
    "S": [
      [
        2.82842712474619,
        2.1213203435596424
      ],
      [
        3.5355339059327373,
        2.1213203435596424
      ]
    ],
    "A": [
      [
        0.6697615493266569,
        0.3302384506733431
      ],
      [
        0.8044296825069569,
        0.19557031749304313
      ]
    ],
    "Z": [
      [
        1.6697615493266569,
        -0.6697615493266569
      ],
      [
        1.804429682506957,
        -0.8044296825069569
      ]
    ],
    "A_causal": [
      [
        1.0,
        0.0
      ],
      [
        0.8044296825069569,
        0.19557031749304313
      ]
    ],
    "Z_causal": [
      [
        2.0,
        -1.0
      ],
      [
        1.804429682506957,
        -0.8044296825069569
      ]
    ]
  },
  "rope": {
    "q_position_1": [
      -1.1426396637476532,
      1.922075596544176,
      2.9598506679133294,
      4.029799501669161
    ],
    "k_position_3": [
      -4.383330010581384,
      -2.4054974575618675,
      1.9691045672954794,
      1.0595410341539788
    ],
    "dot_1_3": 10.483012168789427,
    "dot_11_13": 10.483012168789427
  },
  "online_softmax": {
    "output": 2.833333333333333,
    "expected": 2.8333333333333335
  },
  "speculative": {
    "acceptance_mass": [
      0.6,
      0.2
    ],
    "rejection_probability": 0.19999999999999996,
    "final": [
      0.6,
      0.39999999999999997
    ]
  },
  "resources": {
    "gqa_kv_bytes": 1073741824,
    "gqa_kv_GiB": 1.0,
    "8B_bf16_GiB": 14.901161193847656,
    "27B_bf16_GiB": 50.29141902923584
  },
  "additional_exercises": {
    "ideal_cache_tokens_q35": 41769,
    "KL_q_p": 0.19274475702175753,
    "KL_p_q": 0.22314355131420976,
    "option1_MoE_layer_parameters": 638976,
    "option2_MoE_layer_parameters": 884736
  },
  "grpo": {
    "population_std_advantages": [
      1.0,
      1.0,
      -1.0,
      -1.0
    ],
    "clip_terms": [
      1.2,
      -0.8
    ]
  },
  "status": "All assertions passed"
}