KernelBench mega · RTX PRO 6000

Kimi-Linear Decode Gemini 4 Neon (XHigh)

36.66×geomean speedup across shapes

Gemini 4 Neon (XHigh) wrote a single-launch cooperative SM120 CUDA megakernel (188 CTAs x 512 threads) that scores 36.655x on Kimi-Linear decode (0.157 / 0.166 / 0.176 ms/tok at 2k / 8k / 16k, geomean 6,010 tok/s), taking #1 overall ahead of Claude Opus 5.5 (35.34x). It unpacks centered [-8..+7] int4 weights directly into native SM120 bf16x2 FMA2 (lop3 + sub.rn.bf16x2 + fma.rn.bf16x2) with the zero point folded once per 32-element group, uses PTX release/acquire router barriers with redux.sync top-8 selection, runs FlashMLA on bf16 mma.sync tensor cores with cp.async KV pipelining and adaptive Blackwell persisting-L2 policies, and copies step-0 KV caches inside the single cooperative launch. Verdict is interesting for intra-group bf16x2 accumulation, an unguarded *g_norm_sq reset before grid.sync(), and load_state_dict-only weight repacking.

harnessagy
Kernel source (redacted)
"""Single-kernel W4A16 hybrid decode megakernel for Kimi-Linear (3x KDA + 1x MLA + 4x MoE) on SM120 Blackwell."""
from __future__ import annotations

from dataclasses import dataclass, field

import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

OP_TYPE = "kimi_linear_w4a16_decode"
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
EPS = 1.0e-6
GROUP_SIZE = 128


@dataclass(frozen=True)
class Config:
    hidden: int = 2304
    kda_heads: int = 32
    kda_head_dim: int = 128
    short_conv: int = 4
    mla_heads: int = 32
    kv_lora: int = 512
    qk_nope: int = 128
    qk_rope: int = 64
    v_head: int = 128
    rope_theta: float = 10000.0
    n_experts: int = 64
    n_active: int = 8
    n_shared: int = 1
    moe_inter: int = 1024
    routed_scaling: float = 2.446
    group: int = 128
    pattern: tuple = ("K", "K", "K", "M")
    dtype: torch.dtype = field(default=torch.bfloat16)


def build_config(shape: dict) -> Config:
    return Config(n_experts=int(shape.get("n_experts", 64)))


def _pack_int4(w_q: torch.Tensor) -> torch.Tensor:
    lo = w_q[0::2] & 0xF
    hi = w_q[1::2] & 0xF
    return (lo | (hi << 4)).contiguous()


def quantize(w_io: torch.Tensor, group: int = GROUP_SIZE):
    K, N = w_io.shape
    ng = K // group
    wg = w_io.view(ng, group, N).float()
    wmin = wg.min(dim=1, keepdim=True).values
    wmax = wg.max(dim=1, keepdim=True).values
    scales = (wmax - wmin).clamp_min(1e-8) / 15.0
    zeros = (-wmin / scales).round().clamp(0, 15)
    w_q = ((wg / scales) + zeros).round().clamp(0, 15).to(torch.uint8).view(K, N)
    return _pack_int4(w_q), scales.squeeze(1).to(torch.bfloat16), zeros.squeeze(1).to(torch.bfloat16)


class QuantLinear(nn.Module):
    def __init__(self, in_f: int, out_f: int, group: int = GROUP_SIZE):
        super().__init__()
        assert in_f % group == 0 and in_f % 2 == 0
        self.in_f, self.out_f, self.group = in_f, out_f, group
        ng = in_f // group
        self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8))
        self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16))
        self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16))


class QuantExperts(nn.Module):
    def __init__(self, n: int, in_f: int, out_f: int, group: int = GROUP_SIZE):
        super().__init__()
        self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group
        ng = in_f // group
        self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8))
        self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
        self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))


class KDA(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden
        self.q_proj = QuantLinear(d, H * Dk, cfg.group)
        self.k_proj = QuantLinear(d, H * Dk, cfg.group)
        self.v_proj = QuantLinear(d, H * Dk, cfg.group)
        self.g_proj = QuantLinear(d, H * Dk, cfg.group)
        self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype)
        self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype))
        self.o_proj = QuantLinear(H * Dk, d, cfg.group)
        self.scale = Dk ** -0.5


class MLA(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        H, d = cfg.mla_heads, cfg.hidden
        self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group)
        self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group)
        self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group)
        self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group)
        self.scale = (cfg.qk_nope + cfg.qk_rope) ** -0.5


class MoE(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts
        self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype)
        self.gate = QuantExperts(E, d, m, cfg.group)
        self.up = QuantExperts(E, d, m, cfg.group)
        self.down = QuantExperts(E, m, d, cfg.group)
        self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group)
        self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group)
        self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group)


class Block(nn.Module):
    def __init__(self, cfg: Config, kind: str):
        super().__init__()
        self.kind = kind
        self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
        self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
        self.attn = KDA(cfg) if kind == "K" else MLA(cfg)
        self.moe = MoE(cfg)


# --------------------------------------------------------------------------- #
# Single-Kernel CUDA Megakernel V11 (compiled via load_inline):
#   - 188 x 512 cooperative grid (16 warps/SM, 128 regs, 0 stack, 0 spills, 0 MATCH.ANY)
#   - 80 MiB L2 persisting pool (cudaLimitPersistingL2CacheSize) + adaptive Step-0/steady L2 policy
#   - Native SM120 centered-int4 BF16x2 FMA2 GEMV with L2::256B prefetch hints
#   - Input-first overlapped o_proj (12-13 rows/SM, zero extra atomics)
#   - PRE=2 software-pipelined KDA qkvg + overlapped coalesced conv preload on warps 12..14
#   - MoE fused router + shared gate/up + PTX release/acquire barrier + 4-expert pre-issued M3 down
#   - Spin-free MLA A1+A2 (nibble-transposed int4 W_uk), async cp.async double-buffered Step-0 KV copy,
#     exp2f FlashMLA A3 (64/96/128 splits + streaming part_o), and slim overlapped A4
# --------------------------------------------------------------------------- #
_CPP_SRC = r"""
#include <torch/extension.h>
#include <c10/cuda/CUDAStream.h>
#include <cstdint>
#include <vector>

struct KdaLayerPtrs {
    const void* attn_norm;
    const void* qkvg_wq;
    const void* qkvg_sz;
    const void* beta_w;
    const void* conv_w;
    const void* o_wq;
    const void* o_sz;
};

struct MlaLayerPtrs {
    const void* attn_norm;
    const void* qkva_wq;
    const void* qkva_sz;
    const void* wuk_wq;
    const void* wuk_sz;
    const void* wuv_wq;
    const void* wuv_sz;
    const void* o_wq;
    const void* o_sz;
};

struct MoeLayerPtrs {
    const void* moe_norm;
    const void* router_w;
    const void* s_gate_up_wq;
    const void* s_gate_up_sz;
    const void* s_down_wq;
    const void* s_down_sz;
    const void* r_gate_up_wq;
    const void* r_gate_up_sz;
    const void* r_down_wq;
    const void* r_down_sz;
};

struct MegaParams {
    KdaLayerPtrs kda[3];
    MlaLayerPtrs mla;
    MoeLayerPtrs moe[4];

    float* kda_S[3];
    void* kda_cq[3];
    void* kda_ck[3];
    void* kda_cv[3];

    const void* src_c_kv;
    const void* src_k_rope;
    void* dst_c_kv;
    void* dst_k_rope;
    int ctx_len;
    int need_copy;

    const void* in_hidden;
    void* out_hidden;

    void* g_hidden;
    float* g_norm_sq;
    float* g_q;
    float* g_k;
    float* g_v;
    float* g_decay;
    float* g_beta;
    void* g_attn_o;          // [4096] bf16 (KDA)
    float* g_attn_o_f32;     // [4096] f32  (MLA Phase A4 atomic accum)
    float* g_q_full_f32;     // [32*512] f32 (MLA spin-free W_uk accum)
    void* g_q_rope;          // [32*64] bf16 (MLA q_rope)
    void* g_mla_part_ml;     // [32 * 192] float2 (coalesced across splits!)
    void* g_mla_part_o;      // [188 * 2056 * 8] bf16 (padded stride!)
    float* g_router_logits;
    unsigned int* g_router_done; // [4] u32
    void* g_moe_hh;          // [9*1024] bf16
    float routed_scaling;
    float rope_theta;
    int n_experts;
};

extern "C" void launch_kimi_megakernel(
    const MegaParams* h_params,
    void* dense_arena_ptr,
    size_t dense_arena_bytes,
    cudaStream_t stream
);

struct PackedHandle {
    MegaParams base;
    void* dense_arena_ptr;
    size_t dense_arena_bytes;
};

int64_t create_handle(
    std::vector<torch::Tensor> kda_tensors,
    std::vector<torch::Tensor> mla_tensors,
    std::vector<torch::Tensor> moe_tensors,
    std::vector<torch::Tensor> scratch,
    torch::Tensor dense_arena,
    double routed_scaling,
    double rope_theta,
    int64_t n_experts
) {
    auto* h = new PackedHandle();
    h->dense_arena_ptr = dense_arena.data_ptr();
    h->dense_arena_bytes = static_cast<size_t>(dense_arena.numel()) * dense_arena.element_size();

    for (int i = 0; i < 3; ++i) {
        h->base.kda[i].attn_norm = kda_tensors[i * 7 + 0].data_ptr();
        h->base.kda[i].qkvg_wq   = kda_tensors[i * 7 + 1].data_ptr();
        h->base.kda[i].qkvg_sz   = kda_tensors[i * 7 + 2].data_ptr();
        h->base.kda[i].beta_w    = kda_tensors[i * 7 + 3].data_ptr();
        h->base.kda[i].conv_w    = kda_tensors[i * 7 + 4].data_ptr();
        h->base.kda[i].o_wq      = kda_tensors[i * 7 + 5].data_ptr();
        h->base.kda[i].o_sz      = kda_tensors[i * 7 + 6].data_ptr();
    }
    h->base.mla.attn_norm = mla_tensors[0].data_ptr();
    h->base.mla.qkva_wq   = mla_tensors[1].data_ptr();
    h->base.mla.qkva_sz   = mla_tensors[2].data_ptr();
    h->base.mla.wuk_wq    = mla_tensors[3].data_ptr();
    h->base.mla.wuk_sz    = mla_tensors[4].data_ptr();
    h->base.mla.wuv_wq    = mla_tensors[5].data_ptr();
    h->base.mla.wuv_sz    = mla_tensors[6].data_ptr();
    h->base.mla.o_wq      = mla_tensors[7].data_ptr();
    h->base.mla.o_sz      = mla_tensors[8].data_ptr();

    for (int i = 0; i < 4; ++i) {
        h->base.moe[i].moe_norm     = moe_tensors[i * 10 + 0].data_ptr();
        h->base.moe[i].router_w     = moe_tensors[i * 10 + 1].data_ptr();
        h->base.moe[i].s_gate_up_wq = moe_tensors[i * 10 + 2].data_ptr();
        h->base.moe[i].s_gate_up_sz = moe_tensors[i * 10 + 3].data_ptr();
        h->base.moe[i].s_down_wq    = moe_tensors[i * 10 + 4].data_ptr();
        h->base.moe[i].s_down_sz    = moe_tensors[i * 10 + 5].data_ptr();
        h->base.moe[i].r_gate_up_wq = moe_tensors[i * 10 + 6].data_ptr();
        h->base.moe[i].r_gate_up_sz = moe_tensors[i * 10 + 7].data_ptr();
        h->base.moe[i].r_down_wq    = moe_tensors[i * 10 + 8].data_ptr();
        h->base.moe[i].r_down_sz    = moe_tensors[i * 10 + 9].data_ptr();
    }

    h->base.g_hidden        = scratch[0].data_ptr();
    h->base.g_norm_sq       = scratch[1].data_ptr<float>();
    h->base.g_q             = scratch[2].data_ptr<float>();
    h->base.g_k             = scratch[3].data_ptr<float>();
    h->base.g_v             = scratch[4].data_ptr<float>();
    h->base.g_decay         = scratch[5].data_ptr<float>();
    h->base.g_beta          = scratch[6].data_ptr<float>();
    h->base.g_attn_o        = scratch[7].data_ptr();
    h->base.g_attn_o_f32    = scratch[8].data_ptr<float>();
    h->base.g_q_full_f32    = scratch[9].data_ptr<float>();
    h->base.g_q_rope        = scratch[10].data_ptr();
    h->base.g_mla_part_ml   = scratch[11].data_ptr();
    h->base.g_mla_part_o    = scratch[12].data_ptr();
    h->base.g_router_logits = scratch[13].data_ptr<float>();
    h->base.g_router_done   = reinterpret_cast<unsigned int*>(scratch[14].data_ptr<int32_t>());
    h->base.g_moe_hh        = scratch[15].data_ptr();

    h->base.routed_scaling = static_cast<float>(routed_scaling);
    h->base.rope_theta = static_cast<float>(rope_theta);
    h->base.n_experts = static_cast<int>(n_experts);
    return reinterpret_cast<int64_t>(h);
}

void destroy_handle(int64_t ptr) {
    if (ptr != 0) delete reinterpret_cast<PackedHandle*>(ptr);
}

torch::Tensor step_megakernel(
    int64_t handle_ptr,
    torch::Tensor hidden,
    torch::Tensor S0, torch::Tensor cq0, torch::Tensor ck0, torch::Tensor cv0,
    torch::Tensor S1, torch::Tensor cq1, torch::Tensor ck1, torch::Tensor cv1,
    torch::Tensor S2, torch::Tensor cq2, torch::Tensor ck2, torch::Tensor cv2,
    torch::Tensor src_c_kv, torch::Tensor src_k_rope,
    torch::Tensor dst_c_kv, torch::Tensor dst_k_rope,
    int64_t ctx_len,
    int64_t need_copy
) {
    auto* h = reinterpret_cast<PackedHandle*>(handle_ptr);
    auto out = torch::empty_like(hidden);

    MegaParams p = h->base;
    p.kda_S[0]  = S0.data_ptr<float>();
    p.kda_cq[0] = cq0.data_ptr();
    p.kda_ck[0] = ck0.data_ptr();
    p.kda_cv[0] = cv0.data_ptr();
    p.kda_S[1]  = S1.data_ptr<float>();
    p.kda_cq[1] = cq1.data_ptr();
    p.kda_ck[1] = ck1.data_ptr();
    p.kda_cv[1] = cv1.data_ptr();
    p.kda_S[2]  = S2.data_ptr<float>();
    p.kda_cq[2] = cq2.data_ptr();
    p.kda_ck[2] = ck2.data_ptr();
    p.kda_cv[2] = cv2.data_ptr();

    p.src_c_kv   = src_c_kv.data_ptr();
    p.src_k_rope = src_k_rope.data_ptr();
    p.dst_c_kv   = dst_c_kv.data_ptr();
    p.dst_k_rope = dst_k_rope.data_ptr();
    p.ctx_len    = static_cast<int>(ctx_len);
    p.need_copy  = static_cast<int>(need_copy);

    p.in_hidden  = hidden.data_ptr();
    p.out_hidden = out.data_ptr();

    launch_kimi_megakernel(&p, h->dense_arena_ptr, h->dense_arena_bytes, at::cuda::getCurrentCUDAStream().stream());
    return out;
}
"""

_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cooperative_groups.h>
#include <cstdint>
#include <cmath>

namespace cg = cooperative_groups;

constexpr int HIDDEN = 2304;
constexpr int KDA_H = 32;
constexpr int KDA_C = 4096;
constexpr int KV_LORA = 512;
constexpr int QK_ROPE = 64;
constexpr int V_HEAD = 128;
constexpr int MOE_INTER = 1024;
constexpr int N_ACTIVE = 8;
constexpr int BLOCK_THREADS = 512;
constexpr int SMEM_BYTES = 86016;    // 84 KB
constexpr int X_U4_STRIDE = 5;       // 4 uint4s (32 bf16s as 16 bf16x2) + 1 uint4 pad -> 0 bank conflicts!
constexpr int X_F32_STRIDE = 36;     // Used only for MLA W_uv (4*36 floats = 576 B)
constexpr int LAT_STRIDE = 132;      // Padded 128+4 for MLA A4 smem_part_lat -> 0 bank conflicts!
constexpr int O_SMEM_STRIDE = 520;     // Padded 512+8 bf16 (65 uint4s) for FlashMLA smem_O -> 0 bank conflicts!
constexpr int ML_STRIDE = 192;         // Coalesced stride for float2 g_mla_part_ml[MLA_H][ML_STRIDE]
constexpr int PART_O_STRIDE_U4 = 2056; // Padded 32*64 + 8 uint4 (32896 B) -> breaks 32 KB L2 set aliasing!

struct KdaLayerPtrs {
    const void* attn_norm;
    const void* qkvg_wq;
    const void* qkvg_sz;
    const void* beta_w;
    const void* conv_w;
    const void* o_wq;
    const void* o_sz;
};

struct MlaLayerPtrs {
    const void* attn_norm;
    const void* qkva_wq;
    const void* qkva_sz;
    const void* wuk_wq;
    const void* wuk_sz;
    const void* wuv_wq;
    const void* wuv_sz;
    const void* o_wq;
    const void* o_sz;
};

struct MoeLayerPtrs {
    const void* moe_norm;
    const void* router_w;
    const void* s_gate_up_wq;
    const void* s_gate_up_sz;
    const void* s_down_wq;
    const void* s_down_sz;
    const void* r_gate_up_wq;
    const void* r_gate_up_sz;
    const void* r_down_wq;
    const void* r_down_sz;
};

struct MegaParams {
    KdaLayerPtrs kda[3];
    MlaLayerPtrs mla;
    MoeLayerPtrs moe[4];

    float* kda_S[3];
    void* kda_cq[3];
    void* kda_ck[3];
    void* kda_cv[3];

    const void* src_c_kv;
    const void* src_k_rope;
    void* dst_c_kv;
    void* dst_k_rope;
    int ctx_len;
    int need_copy;

    const void* in_hidden;
    void* out_hidden;

    void* g_hidden;
    float* g_norm_sq;
    float* g_q;
    float* g_k;
    float* g_v;
    float* g_decay;
    float* g_beta;
    void* g_attn_o;
    float* g_attn_o_f32;
    float* g_q_full_f32;
    void* g_q_rope;
    void* g_mla_part_ml;
    void* g_mla_part_o;
    float* g_router_logits;
    unsigned int* g_router_done;
    void* g_moe_hh;
    float routed_scaling;
    float rope_theta;
    int n_experts;
};

// ---------------------------------------------------------------------------
// Low-level PTX helpers & L2 cache policies
// ---------------------------------------------------------------------------
__device__ __forceinline__ float bf16_to_f32(__nv_bfloat16 v) {
    return __bfloat162float(v);
}

__device__ __forceinline__ __nv_bfloat16 f32_to_bf16(float v) {
    return __float2bfloat16(v);
}

__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) {
    __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi);
    return *reinterpret_cast<uint32_t*>(&v);
}

__device__ __forceinline__ float silu_f32(float x) {
    if (x >= 0.0f) return x / (1.0f + __expf(-x));
    float e = __expf(x);
    return x * (e / (1.0f + e));
}

__device__ __forceinline__ float sigmoid_f32(float x) {
    if (x >= 0.0f) return 1.0f / (1.0f + __expf(-x));
    float e = __expf(x);
    return e / (1.0f + e);
}

__device__ __forceinline__ uint32_t float_to_sortable_u32(float f) {
    uint32_t u = __float_as_uint(f);
    uint32_t mask = static_cast<uint32_t>(-static_cast<int32_t>(u >> 31)) | 0x80000000u;
    return u ^ mask;
}

__device__ __forceinline__ uint4 ld_u4_keep(const uint4* ptr, uint64_t pol) {
    uint4 v;
    asm volatile("ld.global.L2::256B.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;"
                 : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
                 : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ uint4 ld_u4_stream(const uint4* ptr, uint64_t pol) {
    uint4 v;
    asm volatile("ld.global.L1::evict_first.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;"
                 : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
                 : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ uint4 ld_u4_stream256(const uint4* ptr, uint64_t pol) {
    uint4 v;
    asm volatile("ld.global.L1::evict_first.L2::256B.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;"
                 : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
                 : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ uint4 ld_u4_ca(const uint4* ptr) {
    uint4 v;
    asm volatile("ld.global.ca.v4.u32 {%0, %1, %2, %3}, [%4];"
                 : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
                 : "l"(ptr));
    return v;
}

__device__ __forceinline__ uint2 ld_u2_keep(const uint2* ptr, uint64_t pol) {
    uint2 v;
    asm volatile("ld.global.L2::cache_hint.v2.u32 {%0, %1}, [%2], %3;"
                 : "=r"(v.x), "=r"(v.y)
                 : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ void st_u4_stream(uint4* ptr, uint4 v, uint64_t pol) {
    asm volatile("st.global.cs.L2::cache_hint.v4.u32 [%0], {%1, %2, %3, %4}, %5;"
                 :: "l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w), "l"(pol));
}

__device__ __forceinline__ uint32_t ld_u32_keep(const uint32_t* ptr, uint64_t pol) {
    uint32_t v;
    asm volatile("ld.global.L2::cache_hint.u32 %0, [%1], %2;" : "=r"(v) : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ uint32_t ld_u32_stream(const uint32_t* ptr, uint64_t pol) {
    uint32_t v;
    asm volatile("ld.global.L1::evict_first.L2::cache_hint.u32 %0, [%1], %2;" : "=r"(v) : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ float ld_f32_keep(const float* ptr, uint64_t pol) {
    float v;
    asm volatile("ld.global.L2::cache_hint.f32 %0, [%1], %2;" : "=f"(v) : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ void st_f32_keep(float* ptr, float val, uint64_t pol) {
    asm volatile("st.global.L2::cache_hint.f32 [%0], %1, %2;" :: "l"(ptr), "f"(val), "l"(pol));
}

__device__ __forceinline__ void st_f32x2_keep(float2* ptr, float2 val, uint64_t pol) {
    asm volatile("st.global.L2::cache_hint.v2.f32 [%0], {%1, %2}, %3;"
                 :: "l"(ptr), "f"(val.x), "f"(val.y), "l"(pol));
}

__device__ __forceinline__ float2 ld_f32x2_keep(const float2* ptr, uint64_t pol) {
    float2 v;
    asm volatile("ld.global.L2::cache_hint.v2.f32 {%0, %1}, [%2], %3;"
                 : "=f"(v.x), "=f"(v.y) : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ uint16_t ld_u16_keep(const void* ptr, uint64_t pol) {
    uint16_t v;
    asm volatile("ld.global.L2::cache_hint.u16 %0, [%1], %2;" : "=h"(v) : "l"(ptr), "l"(pol));
    return v;
}

__device__ __forceinline__ void st_u16_keep(void* ptr, uint16_t v, uint64_t pol) {
    asm volatile("st.global.L2::cache_hint.u16 [%0], %1, %2;" :: "l"(ptr), "h"(v), "l"(pol));
}

// FP32 biased-128 unpack (for MLA W_uk and W_uv)
__device__ __forceinline__ float unpack_biased128(uint32_t v) {
    uint32_t r;
    asm("lop3.b32 %0, %1, 0x000F0000, 0x43000000, 0xEA;" : "=r"(r) : "r"(v));
    return __uint_as_float(r);
}

__device__ __forceinline__ float dot8_biased(uint32_t p, const float* xp) {
    float4 x0 = *reinterpret_cast<const float4*>(xp);
    float4 x1 = *reinterpret_cast<const float4*>(xp + 4);
    float acc = x0.x * unpack_biased128(p << 16);
    acc = fmaf(x0.y, unpack_biased128(p << 12), acc);
    acc = fmaf(x0.z, unpack_biased128(p << 8), acc);
    acc = fmaf(x0.w, unpack_biased128(p << 4), acc);
    acc = fmaf(x1.x, unpack_biased128(p), acc);
    acc = fmaf(x1.y, unpack_biased128(p >> 4), acc);
    acc = fmaf(x1.z, unpack_biased128(p >> 8), acc);
    acc = fmaf(x1.w, unpack_biased128(p >> 12), acc);
    return acc;
}

__device__ __forceinline__ float dot32_w4_f32(
    uint4 w, uint32_t sz, const float* smem_x32, float x_sum32
) {
    float dot = dot8_biased(w.x, smem_x32) + dot8_biased(w.y, smem_x32 + 8)
              + dot8_biased(w.z, smem_x32 + 16) + dot8_biased(w.w, smem_x32 + 24);
    float scale = __uint_as_float(sz << 16);
    float zero  = __uint_as_float(sz & 0xFFFF0000u);
    return scale * fmaf(-(zero + 128.0f), x_sum32, dot);
}

// Native SM120 BF16x2 centered [-8..+7] int4 unpack + FMA2 (8.33 TB/s L2, 0 overflow, 0 DC cancellation!)
__device__ __forceinline__ uint32_t unpack_bf2_s8(uint32_t v) {
    uint32_t b, r;
    asm("lop3.b32 %0, %1, 0x000F000F, 0x43004300, 0xEA;" : "=r"(b) : "r"(v));
    asm("sub.rn.bf16x2 %0, %1, %2;" : "=r"(r) : "r"(b), "r"(0x43084308u));
    return r;
}

__device__ __forceinline__ uint32_t fma_bf2_u32(uint32_t a, uint32_t b, uint32_t c) {
    uint32_t r;
    asm("fma.rn.bf16x2 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c));
    return r;
}

__device__ __forceinline__ float dot32_bf2(
    uint4 w, uint32_t sz, const uint4* sx_u4, float x_sum32
) {
    uint32_t acc = 0u;
    #pragma unroll
    for (int part = 0; part < 4; ++part) {
        uint32_t p = (part == 0) ? w.x : ((part == 1) ? w.y : ((part == 2) ? w.z : w.w));
        uint4 x4 = sx_u4[part];
        acc = fma_bf2_u32(unpack_bf2_s8(p),       x4.x, acc);
        acc = fma_bf2_u32(unpack_bf2_s8(p >> 4),  x4.y, acc);
        acc = fma_bf2_u32(unpack_bf2_s8(p >> 8),  x4.z, acc);
        acc = fma_bf2_u32(unpack_bf2_s8(p >> 12), x4.w, acc);
    }
    float dot = __uint_as_float(acc << 16) + __uint_as_float(acc & 0xFFFF0000u);
    float scale = __uint_as_float(sz << 16);
    float zero  = __uint_as_float(sz & 0xFFFF0000u);
    return scale * fmaf(8.0f - zero, x_sum32, dot);
}

__device__ __forceinline__ void dot32_bf2_pair(
    uint4 wa, uint32_t sza, uint4 wb, uint32_t szb,
    const uint4* sx_u4, float x_sum32,
    float& acc_a, float& acc_b
) {
    uint32_t da = 0u, db = 0u;
    #pragma unroll
    for (int part = 0; part < 4; ++part) {
        uint32_t pa = (part == 0) ? wa.x : ((part == 1) ? wa.y : ((part == 2) ? wa.z : wa.w));
        uint32_t pb = (part == 0) ? wb.x : ((part == 1) ? wb.y : ((part == 2) ? wb.z : wb.w));
        uint4 x4 = sx_u4[part];
        da = fma_bf2_u32(unpack_bf2_s8(pa),       x4.x, da); db = fma_bf2_u32(unpack_bf2_s8(pb),       x4.x, db);
        da = fma_bf2_u32(unpack_bf2_s8(pa >> 4),  x4.y, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 4),  x4.y, db);
        da = fma_bf2_u32(unpack_bf2_s8(pa >> 8),  x4.z, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 8),  x4.z, db);
        da = fma_bf2_u32(unpack_bf2_s8(pa >> 12), x4.w, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 12), x4.w, db);
    }
    float daf = __uint_as_float(da << 16) + __uint_as_float(da & 0xFFFF0000u);
    float dbf = __uint_as_float(db << 16) + __uint_as_float(db & 0xFFFF0000u);
    float sa = __uint_as_float(sza << 16), za = __uint_as_float(sza & 0xFFFF0000u);
    float sb = __uint_as_float(szb << 16), zb = __uint_as_float(szb & 0xFFFF0000u);
    acc_a += sa * fmaf(8.0f - za, x_sum32, daf);
    acc_b += sb * fmaf(8.0f - zb, x_sum32, dbf);
}

// Uniform-mask warp/subwarp reductions (100% 0xffffffffu -> ZERO MATCH.ANY!)
__device__ __forceinline__ float warp_reduce_sum(float v) {
    #pragma unroll
    for (int off = 16; off > 0; off >>= 1) v += __shfl_xor_sync(0xffffffffu, v, off);
    return v;
}

__device__ __forceinline__ float subwarp8_reduce_sum(float v) {
    v += __shfl_xor_sync(0xffffffffu, v, 4);
    v += __shfl_xor_sync(0xffffffffu, v, 2);
    v += __shfl_xor_sync(0xffffffffu, v, 1);
    return v;
}

__device__ __forceinline__ float subwarp4_reduce_sum(float v) {
    v += __shfl_xor_sync(0xffffffffu, v, 2);
    v += __shfl_xor_sync(0xffffffffu, v, 1);
    return v;
}

__device__ __forceinline__ float halfwarp16_reduce_max(float v) {
    v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 8));
    v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 4));
    v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 2));
    v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 1));
    return v;
}

__device__ __forceinline__ float halfwarp16_reduce_sum(float v) {
    v += __shfl_xor_sync(0xffffffffu, v, 8);
    v += __shfl_xor_sync(0xffffffffu, v, 4);
    v += __shfl_xor_sync(0xffffffffu, v, 2);
    v += __shfl_xor_sync(0xffffffffu, v, 1);
    return v;
}

__device__ __forceinline__ void cp_async_cg_16_pol(void* smem_ptr, const void* glob_ptr, uint64_t pol) {
    uint32_t smem_addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile("cp.async.cg.shared.global.L2::256B.L2::cache_hint [%0], [%1], 16, %2;\n"
                 :: "r"(smem_addr), "l"(glob_ptr), "l"(pol));
}

__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n" ::);
}

template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}

__device__ __forceinline__ void ldmatrix_x4(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr) {
    uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile("ldmatrix.sync.aligned.x4.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n"
                 : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr));
}

__device__ __forceinline__ void ldmatrix_x2(uint32_t& r0, uint32_t& r1, const void* smem_ptr) {
    uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile("ldmatrix.sync.aligned.x2.m8n8.shared.b16 {%0, %1}, [%2];\n"
                 : "=r"(r0), "=r"(r1) : "r"(addr));
}

__device__ __forceinline__ void ldmatrix_x4_trans(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr) {
    uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile("ldmatrix.sync.aligned.x4.trans.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n"
                 : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr));
}

__device__ __forceinline__ void mma_m16n8k16_bf16(
    float& d0, float& d1, float& d2, float& d3,
    uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
    uint32_t b0, uint32_t b1
) {
    asm volatile(
        "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
        "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
        : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
        : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)
    );
}

// ---------------------------------------------------------------------------
// Vectorized BF16x2 SMEM activation loaders
// ---------------------------------------------------------------------------
__device__ __forceinline__ uint4 pack_and_store_rmsnorm8(
    int q, uint4 h8, uint4 w8, float inv_rms, uint4* smem_x_u4, float* smem_x_sum32
) {
    float r0 = __uint_as_float(h8.x << 16) * inv_rms * __uint_as_float(w8.x << 16);
    float r1 = __uint_as_float(h8.x & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.x & 0xFFFF0000u);
    float r2 = __uint_as_float(h8.y << 16) * inv_rms * __uint_as_float(w8.y << 16);
    float r3 = __uint_as_float(h8.y & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.y & 0xFFFF0000u);
    float r4 = __uint_as_float(h8.z << 16) * inv_rms * __uint_as_float(w8.z << 16);
    float r5 = __uint_as_float(h8.z & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.z & 0xFFFF0000u);
    float r6 = __uint_as_float(h8.w << 16) * inv_rms * __uint_as_float(w8.w << 16);
    float r7 = __uint_as_float(h8.w & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.w & 0xFFFF0000u);

    uint32_t p0 = pack_bf16x2(r0, r4);
    uint32_t p1 = pack_bf16x2(r1, r5);
    uint32_t p2 = pack_bf16x2(r2, r6);
    uint32_t p3 = pack_bf16x2(r3, r7);
    int c = q >> 2;
    uint4 x4 = make_uint4(p0, p1, p2, p3);
    smem_x_u4[c * X_U4_STRIDE + (q & 3)] = x4;

    float s = (__uint_as_float(p0 << 16) + __uint_as_float(p0 & 0xFFFF0000u))
            + (__uint_as_float(p1 << 16) + __uint_as_float(p1 & 0xFFFF0000u))
            + (__uint_as_float(p2 << 16) + __uint_as_float(p2 & 0xFFFF0000u))
            + (__uint_as_float(p3 << 16) + __uint_as_float(p3 & 0xFFFF0000u));
    s = subwarp4_reduce_sum(s);
    if ((threadIdx.x & 3) == 0) smem_x_sum32[c] = s;
    return x4;
}

__device__ __forceinline__ void load_rmsnorm_2304_to_smem_bf2(
    const __nv_bfloat16* g_in,
    const __nv_bfloat16* norm_w,
    float inv_rms,
    uint4* smem_x_u4,
    float* smem_x_sum32,
    int bid,
    uint64_t pol_keep
) {
    int tid = threadIdx.x;
    if (tid < 288) {
        int shift = (bid % 72) * 4;
        int q = tid + shift;
        if (q >= 288) q -= 288;
        uint4 h8 = ld_u4_ca(reinterpret_cast<const uint4*>(g_in) + q);
        uint4 w8 = ld_u4_keep(reinterpret_cast<const uint4*>(norm_w) + q, pol_keep);
        pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32);
    }
    __syncthreads();
}

__device__ __forceinline__ void load_input_and_rmsnorm_2304_to_smem_bf2(
    const __nv_bfloat16* in_hidden,
    __nv_bfloat16* g_hidden,
    const __nv_bfloat16* norm_w,
    uint4* smem_x_u4,
    float* smem_x_sum32,
    float* smem_red16,
    int bid,
    uint64_t pol_keep
) {
    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;
    uint4 h8 = make_uint4(0, 0, 0, 0);
    uint4 w8 = make_uint4(0, 0, 0, 0);
    int q = 0;
    float local_sq = 0.0f;
    if (tid < 288) {
        int shift = (bid % 72) * 4;
        q = tid + shift;
        if (q >= 288) q -= 288;
        h8 = ld_u4_ca(reinterpret_cast<const uint4*>(in_hidden) + q);
        w8 = ld_u4_keep(reinterpret_cast<const uint4*>(norm_w) + q, pol_keep);
        if (bid == 0) reinterpret_cast<uint4*>(g_hidden)[q] = h8;
        float h0 = __uint_as_float(h8.x << 16), h1 = __uint_as_float(h8.x & 0xFFFF0000u);
        float h2 = __uint_as_float(h8.y << 16), h3 = __uint_as_float(h8.y & 0xFFFF0000u);
        float h4 = __uint_as_float(h8.z << 16), h5 = __uint_as_float(h8.z & 0xFFFF0000u);
        float h6 = __uint_as_float(h8.w << 16), h7 = __uint_as_float(h8.w & 0xFFFF0000u);
        local_sq = h0*h0 + h1*h1 + h2*h2 + h3*h3 + h4*h4 + h5*h5 + h6*h6 + h7*h7;
    }
    local_sq = warp_reduce_sum(local_sq);
    if (lane == 0) smem_red16[wid] = local_sq;
    __syncthreads();
    if (wid == 0) {
        float tot = (lane < 9) ? smem_red16[lane] : 0.0f;
        tot = warp_reduce_sum(tot);
        if (lane == 0) smem_red16[0] = rsqrtf(tot * (1.0f / float(HIDDEN)) + 1.0e-6f);
    }
    __syncthreads();
    float inv_rms = smem_red16[0];
    if (tid < 288) {
        pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32);
    }
    __syncthreads();
}

__device__ __forceinline__ void store_bf16_u4_to_smem_bf2(
    int q, uint4 v, uint4* smem_u4, float* smem_sum32
) {
    uint32_t p0 = __byte_perm(v.x, v.z, 0x5410u);
    uint32_t p1 = __byte_perm(v.x, v.z, 0x7632u);
    uint32_t p2 = __byte_perm(v.y, v.w, 0x5410u);
    uint32_t p3 = __byte_perm(v.y, v.w, 0x7632u);
    int c = q >> 2;
    smem_u4[c * X_U4_STRIDE + (q & 3)] = make_uint4(p0, p1, p2, p3);
    float s = (__uint_as_float(v.x << 16) + __uint_as_float(v.x & 0xFFFF0000u))
            + (__uint_as_float(v.y << 16) + __uint_as_float(v.y & 0xFFFF0000u))
            + (__uint_as_float(v.z << 16) + __uint_as_float(v.z & 0xFFFF0000u))
            + (__uint_as_float(v.w << 16) + __uint_as_float(v.w & 0xFFFF0000u));
    s = subwarp4_reduce_sum(s);
    if ((threadIdx.x & 3) == 0) smem_sum32[c] = s;
}

// ---------------------------------------------------------------------------
// Overlapped o_proj (4096 -> 2304):
//   - Direct 12-13 rows/block across all 188 SMs (zero remainder atomics)
//   - Issues g_attn_o_in L2 loads first, overlapped with o_wq[0..1] + g_hidden[n]
// ---------------------------------------------------------------------------
template <bool INPUT_IS_F32>
__device__ __forceinline__ void run_load_and_oproj(
    const void* g_attn_o_in,
    const uint4* o_wq,
    const uint32_t* o_sz,
    __nv_bfloat16* g_hidden,
    float* g_norm_sq,
    uint4* smem_x_u4,
    float* smem_x_sum32,
    float* smem_sq16,
    int bid,
    int nblocks,
    uint64_t pol_keep
) {
    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;

    int n0 = (bid * HIDDEN) / nblocks;
    int n_rows = ((bid + 1) * HIDDEN) / nblocks - n0;

    uint4 w0 = make_uint4(0, 0, 0, 0), w1 = make_uint4(0, 0, 0, 0);
    uint32_t s0 = 0, s1 = 0;
    const uint4* wr = nullptr;
    const uint32_t* sr = nullptr;
    int n = 0;
    __nv_bfloat16 h_prev = f32_to_bf16(0.0f);

    int q = (tid + (bid & 127) * 4) & 511;
    uint4 vo_a = {}, vo_b = {};
    if constexpr (!INPUT_IS_F32) {
        vo_a = ld_u4_ca(reinterpret_cast<const uint4*>(g_attn_o_in) + q);
    } else {
        const uint4* src = reinterpret_cast<const uint4*>(g_attn_o_in);
        vo_a = ld_u4_ca(src + q * 2 + 0);
        vo_b = ld_u4_ca(src + q * 2 + 1);
    }
    if (wid < n_rows) {
        n = n0 + wid;
        wr = o_wq + static_cast<size_t>(n) * 128;
        sr = o_sz + static_cast<size_t>(n) * 32;
        w0 = ld_u4_keep(wr + lane, pol_keep);
        w1 = ld_u4_keep(wr + lane + 32, pol_keep);
        s0 = ld_u32_keep(sr + (lane >> 2), pol_keep);
        s1 = ld_u32_keep(sr + ((lane + 32) >> 2), pol_keep);
        if (lane == 0) h_prev = g_hidden[n];
    }
    if constexpr (!INPUT_IS_F32) {
        store_bf16_u4_to_smem_bf2(q, vo_a, smem_x_u4, smem_x_sum32);
    } else {
        float4 a = *reinterpret_cast<float4*>(&vo_a);
        float4 b = *reinterpret_cast<float4*>(&vo_b);
        uint32_t p0 = pack_bf16x2(a.x, b.x);
        uint32_t p1 = pack_bf16x2(a.y, b.y);
        uint32_t p2 = pack_bf16x2(a.z, b.z);
        uint32_t p3 = pack_bf16x2(a.w, b.w);
        int c = q >> 2;
        smem_x_u4[c * X_U4_STRIDE + (q & 3)] = make_uint4(p0, p1, p2, p3);
        float s = (__uint_as_float(p0 << 16) + __uint_as_float(p0 & 0xFFFF0000u))
                + (__uint_as_float(p1 << 16) + __uint_as_float(p1 & 0xFFFF0000u))
                + (__uint_as_float(p2 << 16) + __uint_as_float(p2 & 0xFFFF0000u))
                + (__uint_as_float(p3 << 16) + __uint_as_float(p3 & 0xFFFF0000u));
        s = subwarp4_reduce_sum(s);
        if ((tid & 3) == 0) smem_x_sum32[c] = s;
    }
    __syncthreads();

    if (wid < n_rows) {
        int k0 = lane, k1 = lane + 32, k2 = lane + 64, k3 = lane + 96;
        uint4 w2 = ld_u4_keep(wr + k2, pol_keep);
        uint4 w3 = ld_u4_keep(wr + k3, pol_keep);
        uint32_t s2 = ld_u32_keep(sr + (k2 >> 2), pol_keep);
        uint32_t s3 = ld_u32_keep(sr + (k3 >> 2), pol_keep);

        float acc = dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
        acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
        acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
        acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]);
        acc = warp_reduce_sum(acc);

        if (lane == 0) {
            __nv_bfloat16 proj_bf = f32_to_bf16(acc);
            float h_new = bf16_to_f32(h_prev) + bf16_to_f32(proj_bf);
            __nv_bfloat16 h_bf = f32_to_bf16(h_new);
            g_hidden[n] = h_bf;
            float hf = bf16_to_f32(h_bf);
            smem_sq16[wid] = hf * hf;
        }
    }
    __syncthreads();
    if (wid == 0) {
        float sq = (lane < n_rows) ? smem_sq16[lane] : 0.0f;
        sq = warp_reduce_sum(sq);
        if (lane == 0 && sq != 0.0f) atomicAdd(g_norm_sq, sq);
    }
}

// ---------------------------------------------------------------------------
// MoE block:
//   - Fused M1+M2 (3-way K-split s_gate_up, red.release.gpu + ld.relaxed/fence.acquire spin,
//     redux.sync Top-8, balanced PRE=2 L2::256B BF16x2 routed gate+up)
//   - Overlapped M3: direct 12-13 rows/block, contiguous expert rotation,
//     4 initial experts (e0..e3) + g_hidden[n] pre-issued before load_hh
// ---------------------------------------------------------------------------
__device__ __forceinline__ void run_moe_layer(
    cg::grid_group& grid,
    const MoeLayerPtrs& moe,
    __nv_bfloat16* g_hidden,
    __nv_bfloat16* final_out,
    float* g_norm_sq,
    float* g_router_logits,
    unsigned int* g_router_done,
    __nv_bfloat16* g_moe_hh,
    float routed_scaling,
    int n_experts,
    char* smem_raw,
    int bid,
    int nblocks,
    uint64_t pol_keep,
    uint64_t pol_stream
) {
    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;

    uint4* smem_x_u4 = reinterpret_cast<uint4*>(smem_raw);
    float* smem_x_sum32 = reinterpret_cast<float*>(smem_x_u4 + 72 * X_U4_STRIDE);
    float* smem_red16 = smem_x_sum32 + 80;
    float* smem_sgu = smem_red16 + 16;
    int* smem_topk_idx = reinterpret_cast<int*>(smem_raw + 32768);
    float* smem_topk_w = reinterpret_cast<float*>(smem_raw + 32832);

    float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f);
    if (bid < n_experts) {
        // Fused load_rmsnorm + router: issues h8, w8, and router_w in parallel, dots in registers, 1 fewer barrier!
        float acc = 0.0f;
        if (tid < 288) {
            int shift = (bid % 72) * 4;
            int q = tid + shift;
            if (q >= 288) q -= 288;
            const uint4* rw_u4 = reinterpret_cast<const uint4*>(moe.router_w) + static_cast<size_t>(bid) * 288;
            uint4 h8 = ld_u4_ca(reinterpret_cast<const uint4*>(g_hidden) + q);
            uint4 w8 = ld_u4_keep(reinterpret_cast<const uint4*>(moe.moe_norm) + q, pol_keep);
            uint4 v  = ld_u4_keep(rw_u4 + q, pol_keep);
            uint4 x4 = pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32);
            acc = fmaf(__uint_as_float(x4.x << 16), __uint_as_float(v.x << 16), acc);
            acc = fmaf(__uint_as_float(x4.x & 0xFFFF0000u), __uint_as_float(v.x & 0xFFFF0000u), acc);
            acc = fmaf(__uint_as_float(x4.y << 16), __uint_as_float(v.y << 16), acc);
            acc = fmaf(__uint_as_float(x4.y & 0xFFFF0000u), __uint_as_float(v.y & 0xFFFF0000u), acc);
            acc = fmaf(__uint_as_float(x4.z << 16), __uint_as_float(v.z << 16), acc);
            acc = fmaf(__uint_as_float(x4.z & 0xFFFF0000u), __uint_as_float(v.z & 0xFFFF0000u), acc);
            acc = fmaf(__uint_as_float(x4.w << 16), __uint_as_float(v.w << 16), acc);
            acc = fmaf(__uint_as_float(x4.w & 0xFFFF0000u), __uint_as_float(v.w & 0xFFFF0000u), acc);
        }
        acc = warp_reduce_sum(acc);
        if (lane == 0) smem_red16[wid] = acc;
        __syncthreads();
        if (wid == 0) {
            float tot = (lane < 9) ? smem_red16[lane] : 0.0f;
            tot = warp_reduce_sum(tot);
            if (lane == 0) {
                g_router_logits[bid] = tot;
                asm volatile("red.release.gpu.global.add.u32 [%0], 1;" :: "l"(g_router_done) : "memory");
            }
        }
    } else {
        int s_bid = bid - n_experts;
        int s_nblocks = nblocks - n_experts;
        int c0 = (s_bid * MOE_INTER) / s_nblocks;
        int c1 = ((s_bid + 1) * MOE_INTER) / s_nblocks;
        int n_pairs = c1 - c0;
        int n_jobs = n_pairs * 3;
        int sub_id = tid >> 3;
        int t8 = tid & 7;
        bool act = (sub_id < n_jobs);
        int p_loc = 0, ks = 0, k0 = 0, k1 = 0, k2 = 0;
        uint4 wg0 = {}, wu0 = {}, wg1 = {}, wu1 = {}, wg2 = {}, wu2 = {};
        uint32_t sg0 = 0, su0 = 0, sg1 = 0, su1 = 0, sg2 = 0, su2 = 0;
        if (wid * 4 < n_jobs) {
            int j_safe = act ? sub_id : 0;
            p_loc = (j_safe * 21846) >> 16;
            ks = j_safe - p_loc * 3;
            int c = c0 + p_loc;
            const uint4* gw = reinterpret_cast<const uint4*>(moe.s_gate_up_wq) + static_cast<size_t>(c) * 144;
            const uint32_t* gsz = reinterpret_cast<const uint32_t*>(moe.s_gate_up_sz) + static_cast<size_t>(c) * 36;
            k0 = (ks * 3 + 0) * 8 + t8;
            k1 = (ks * 3 + 1) * 8 + t8;
            k2 = (ks * 3 + 2) * 8 + t8;
            wg0 = ld_u4_keep(gw + k0, pol_keep); wu0 = ld_u4_keep(gw + 72 + k0, pol_keep);
            wg1 = ld_u4_keep(gw + k1, pol_keep); wu1 = ld_u4_keep(gw + 72 + k1, pol_keep);
            wg2 = ld_u4_keep(gw + k2, pol_keep); wu2 = ld_u4_keep(gw + 72 + k2, pol_keep);
            sg0 = ld_u32_keep(gsz + (k0 >> 2), pol_keep); su0 = ld_u32_keep(gsz + 18 + (k0 >> 2), pol_keep);
            sg1 = ld_u32_keep(gsz + (k1 >> 2), pol_keep); su1 = ld_u32_keep(gsz + 18 + (k1 >> 2), pol_keep);
            sg2 = ld_u32_keep(gsz + (k2 >> 2), pol_keep); su2 = ld_u32_keep(gsz + 18 + (k2 >> 2), pol_keep);
        }
        load_rmsnorm_2304_to_smem_bf2(
            g_hidden, reinterpret_cast<const __nv_bfloat16*>(moe.moe_norm),
            inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep
        );
        if (wid * 4 < n_jobs) {

            float ga = 0.0f, ua = 0.0f;
            dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua);
            dot32_bf2_pair(wg1, sg1, wu1, su1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], ga, ua);
            dot32_bf2_pair(wg2, sg2, wu2, su2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2], ga, ua);
            ga = subwarp8_reduce_sum(ga);
            ua = subwarp8_reduce_sum(ua);
            if (act && t8 == 0) {
                smem_sgu[p_loc * 6 + ks]     = ga;
                smem_sgu[p_loc * 6 + 3 + ks] = ua;
            }
        }
        __syncthreads();
        if (tid < n_pairs) {
            float ga = smem_sgu[tid * 6 + 0] + smem_sgu[tid * 6 + 1] + smem_sgu[tid * 6 + 2];
            float ua = smem_sgu[tid * 6 + 3] + smem_sgu[tid * 6 + 4] + smem_sgu[tid * 6 + 5];
            g_moe_hh[8 * MOE_INTER + c0 + tid] = f32_to_bf16(silu_f32(ga) * ua);
        }
    }

    // Fused spin-wait + parallel-lookup redux.sync Top-8 in wid==0 (saves 1 __syncthreads + 7 shuffles per MoE!)
    if (wid == 0) {
        if (lane == 0) {
            unsigned int done;
            while (true) {
                asm volatile("ld.relaxed.gpu.global.u32 %0, [%1];" : "=r"(done) : "l"(g_router_done));
                if (done >= static_cast<unsigned int>(n_experts)) break;
                asm volatile("nanosleep.u32 64;");
            }
            asm volatile("fence.acquire.gpu;" ::: "memory");
        }
        __syncwarp();
        float l0 = (lane < n_experts) ? g_router_logits[lane] : -1e30f;
        float l1 = (lane + 32 < n_experts) ? g_router_logits[lane + 32] : -1e30f;
        uint32_t k0 = (lane < n_experts) ? ((float_to_sortable_u32(l0) & 0xFFFFFFC0u) | (63u - lane)) : 0u;
        uint32_t k1 = (lane + 32 < n_experts) ? ((float_to_sortable_u32(l1) & 0xFFFFFFC0u) | (63u - (lane + 32))) : 0u;

        int my_bi = 0;
        #pragma unroll
        for (int k = 0; k < N_ACTIVE; ++k) {
            uint32_t km = (k0 >= k1) ? k0 : k1;
            uint32_t best;
            asm("redux.sync.max.u32 %0, %1, 0xffffffff;" : "=r"(best) : "r"(km));
            int bi = 63 - int(best & 63u);
            if (lane == k) my_bi = bi;
            if (bi == lane) k0 = 0u;
            if (bi == lane + 32) k1 = 0u;
        }
        int src_lane = my_bi & 31;
        float v_lo = __shfl_sync(0xffffffffu, l0, src_lane);
        float v_hi = __shfl_sync(0xffffffffu, l1, src_lane);
        float my_val = (my_bi < 32) ? v_lo : v_hi;
        float m_top = __shfl_sync(0xffffffffu, my_val, 0);
        float e = (lane < N_ACTIVE) ? __expf(my_val - m_top) : 0.0f;
        float w_sum = warp_reduce_sum(e);
        if (lane < N_ACTIVE) {
            smem_topk_idx[lane] = my_bi;
            smem_topk_w[lane] = (e / (w_sum + 1e-9f)) * routed_scaling;
        }
        if (lane == 0) {
            smem_topk_idx[8] = n_experts;
            smem_topk_w[8] = 1.0f;
            if (bid == 0) *g_norm_sq = 0.0f;
        }
    }
    __syncthreads();

    // Phase M2: Contiguous 1-wave 2-step lookahead L2::256B BF16x2 routed gate+up
    {
        int p0_idx, n_pairs;
        if (nblocks == 188 && n_experts == 64) {
            if (bid < 64) {
                p0_idx = bid * 45; n_pairs = 45;
            } else if (bid < 168) {
                p0_idx = 2880 + (bid - 64) * 43; n_pairs = 43;
            } else {
                p0_idx = 7352 + (bid - 168) * 42; n_pairs = 42;
            }
        } else {
            p0_idx = (bid * (N_ACTIVE * MOE_INTER)) / nblocks;
            n_pairs = ((bid + 1) * (N_ACTIVE * MOE_INTER)) / nblocks - p0_idx;
        }

        int sub_id = tid >> 3;
        int t8 = tid & 7;
        const uint4* all_wq = reinterpret_cast<const uint4*>(moe.r_gate_up_wq);
        const uint32_t* all_sz = reinterpret_cast<const uint32_t*>(moe.r_gate_up_sz);

        int n_warps = (n_pairs + 3) >> 2;
        if (wid < n_warps) {
            bool act = (sub_id < n_pairs);
            int pair = p0_idx + (act ? sub_id : 0);
            int j = pair >> 10;
            int c = pair & 1023;
            int e = smem_topk_idx[j];
            size_t row_off = static_cast<size_t>(e) * MOE_INTER + c;
            const uint4* gw = all_wq + row_off * 144;
            const uint32_t* gsz = all_sz + row_off * 36;

            float ga = 0.0f, ua = 0.0f;
            int k0 = t8, k1 = 8 + t8;
            uint4 wg0 = ld_u4_stream256(gw + k0, pol_stream), wu0 = ld_u4_stream256(gw + 72 + k0, pol_stream);
            uint4 wg1 = ld_u4_stream256(gw + k1, pol_stream), wu1 = ld_u4_stream256(gw + 72 + k1, pol_stream);
            uint32_t sg0 = ld_u32_stream(gsz + (k0 >> 2), pol_stream), su0 = ld_u32_stream(gsz + 18 + (k0 >> 2), pol_stream);
            uint32_t sg1 = ld_u32_stream(gsz + (k1 >> 2), pol_stream), su1 = ld_u32_stream(gsz + 18 + (k1 >> 2), pol_stream);
            #pragma unroll
            for (int it = 0; it < 7; ++it) {
                int kn = (it + 2) * 8 + t8;
                uint4 nwg = ld_u4_stream256(gw + kn, pol_stream), nwu = ld_u4_stream256(gw + 72 + kn, pol_stream);
                uint32_t nsg = ld_u32_stream(gsz + (kn >> 2), pol_stream), nsu = ld_u32_stream(gsz + 18 + (kn >> 2), pol_stream);
                dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua);
                wg0 = wg1; wu0 = wu1; sg0 = sg1; su0 = su1; k0 = k1;
                wg1 = nwg; wu1 = nwu; sg1 = nsg; su1 = nsu; k1 = kn;
            }
            dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua);
            dot32_bf2_pair(wg1, sg1, wu1, su1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], ga, ua);
            ga = subwarp8_reduce_sum(ga);
            ua = subwarp8_reduce_sum(ua);
            if (act && t8 == 0) g_moe_hh[pair] = f32_to_bf16(silu_f32(ga) * ua);
        }
    }

    grid.sync();

    if (bid == 0 && tid == 0) *g_router_done = 0u;

    // Phase M3: direct 12-13 rows/block
    // Contiguous expert-group rotation (rot = bid*8/nblocks) + ALL 4 initial experts pre-issued BEFORE load_hh!
    uint4* smem_hh_u4 = reinterpret_cast<uint4*>(smem_raw);
    float* smem_hh_sum32 = reinterpret_cast<float*>(smem_hh_u4 + 288 * X_U4_STRIDE);
    float* smem_sq16 = smem_hh_sum32 + 288;

    int n0 = (bid * HIDDEN) / nblocks;
    int n_rows = ((bid + 1) * HIDDEN) / nblocks - n0;
    int rot = (bid * 8) / nblocks;

    const uint4* r_down_wq = reinterpret_cast<const uint4*>(moe.r_down_wq);
    const uint32_t* r_down_sz = reinterpret_cast<const uint32_t*>(moe.r_down_sz);
    const uint4* s_down_wq = reinterpret_cast<const uint4*>(moe.s_down_wq);
    const uint32_t* s_down_sz = reinterpret_cast<const uint32_t*>(moe.s_down_sz);

    uint4 w0 = make_uint4(0, 0, 0, 0), w1 = make_uint4(0, 0, 0, 0);
    uint4 w2 = make_uint4(0, 0, 0, 0), w3 = make_uint4(0, 0, 0, 0);
    uint32_t s0 = 0, s1 = 0, s2 = 0, s3 = 0;
    __nv_bfloat16 h_prev = f32_to_bf16(0.0f);

    if (wid < n_rows) {
        int n = n0 + wid;
        int e0 = smem_topk_idx[rot], e1 = smem_topk_idx[(rot + 1) & 7];
        int e2 = smem_topk_idx[(rot + 2) & 7], e3 = smem_topk_idx[(rot + 3) & 7];
        w0 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e0) * HIDDEN + n) * 32 + lane, pol_stream);
        w1 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e1) * HIDDEN + n) * 32 + lane, pol_stream);
        w2 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e2) * HIDDEN + n) * 32 + lane, pol_stream);
        w3 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e3) * HIDDEN + n) * 32 + lane, pol_stream);
        s0 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e0) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
        s1 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e1) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
        s2 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e2) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
        s3 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e3) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
        if (lane == 0) h_prev = g_hidden[n];
    }

    {
        const uint4* hh_u4 = reinterpret_cast<const uint4*>(g_moe_hh);
        int shift = (bid % 288) * 4;
        int q0 = tid + shift; if (q0 >= 1152) q0 -= 1152;
        int q1 = tid + 512 + shift; if (q1 >= 1152) q1 -= 1152;
        int q2 = tid + 1024 + shift; if (q2 >= 1152) q2 -= 1152;
        uint4 v0 = ld_u4_ca(hh_u4 + q0);
        uint4 v1 = ld_u4_ca(hh_u4 + q1);
        uint4 v2 = (tid < 128) ? ld_u4_ca(hh_u4 + q2) : make_uint4(0u, 0u, 0u, 0u);
        store_bf16_u4_to_smem_bf2(q0, v0, smem_hh_u4, smem_hh_sum32);
        store_bf16_u4_to_smem_bf2(q1, v1, smem_hh_u4, smem_hh_sum32);
        if (tid < 128) store_bf16_u4_to_smem_bf2(q2, v2, smem_hh_u4, smem_hh_sum32);
    }
    __syncthreads();

    if (wid < n_rows) {
        int n = n0 + wid;
        float acc = 0.0f;
        #pragma unroll
        for (int j = 0; j < 5; ++j) {
            int jc = (j + rot) & 7;
            uint4 wn; uint32_t sn;
            if (j < 4) {
                int en = smem_topk_idx[(j + 4 + rot) & 7];
                wn = ld_u4_stream256(r_down_wq + (static_cast<size_t>(en) * HIDDEN + n) * 32 + lane, pol_stream);
                sn = ld_u32_stream(r_down_sz + (static_cast<size_t>(en) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
            } else {
                wn = ld_u4_keep(s_down_wq + static_cast<size_t>(n) * 32 + lane, pol_keep);
                sn = ld_u32_keep(s_down_sz + static_cast<size_t>(n) * 8 + (lane >> 2), pol_keep);
            }
            acc = fmaf(smem_topk_w[jc], dot32_bf2(w0, s0, smem_hh_u4 + (jc * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[jc * 32 + lane]), acc);
            w0 = w1; s0 = s1;
            w1 = w2; s1 = s2;
            w2 = w3; s2 = s3;
            w3 = wn; s3 = sn;
        }
        int j5 = (5 + rot) & 7, j6 = (6 + rot) & 7, j7 = (7 + rot) & 7;
        acc = fmaf(smem_topk_w[j5], dot32_bf2(w0, s0, smem_hh_u4 + (j5 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j5 * 32 + lane]), acc);
        acc = fmaf(smem_topk_w[j6], dot32_bf2(w1, s1, smem_hh_u4 + (j6 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j6 * 32 + lane]), acc);
        acc = fmaf(smem_topk_w[j7], dot32_bf2(w2, s2, smem_hh_u4 + (j7 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j7 * 32 + lane]), acc);
        acc += dot32_bf2(w3, s3, smem_hh_u4 + (8 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[8 * 32 + lane]);

        acc = warp_reduce_sum(acc);
        if (lane == 0) {
            __nv_bfloat16 moe_bf = f32_to_bf16(acc);
            float h_new = bf16_to_f32(h_prev) + bf16_to_f32(moe_bf);
            __nv_bfloat16 h_bf = f32_to_bf16(h_new);
            if (final_out != nullptr) {
                final_out[n] = h_bf;
            } else {
                g_hidden[n] = h_bf;
                float hf = bf16_to_f32(h_bf);
                smem_sq16[wid] = hf * hf;
            }
        }
    }

    if (final_out == nullptr) {
        __syncthreads();
        if (wid == 0) {
            float sq = (lane < n_rows) ? smem_sq16[lane] : 0.0f;
            sq = warp_reduce_sum(sq);
            if (lane == 0 && sq != 0.0f) atomicAdd(g_norm_sq, sq);
        }
    }
}

// ---------------------------------------------------------------------------
// KDA Attention block:
//   - PRE=2 software-pipelined qkvg + overlapped coalesced conv preload (warps 12..14)
//   - K2 pre-issues all 8 S loads before smem_q/k/dec + tree warp reduction
//   - Input-first overlapped o_proj
// ---------------------------------------------------------------------------
__device__ __forceinline__ void run_kda_attn(
    cg::grid_group& grid,
    const KdaLayerPtrs& kda,
    float* S_state,
    __nv_bfloat16* cq,
    __nv_bfloat16* ck,
    __nv_bfloat16* cv,
    const __nv_bfloat16* in_hidden_or_null,
    __nv_bfloat16* g_hidden,
    float* g_norm_sq,
    float* g_q, float* g_k, float* g_v, float* g_decay, float* g_beta,
    __nv_bfloat16* g_attn_o,
    char* smem_raw,
    int bid,
    int nblocks,
    uint64_t pol_keep
) {
    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;

    uint4* smem_x_u4 = reinterpret_cast<uint4*>(smem_raw);
    float* smem_x_sum32 = reinterpret_cast<float*>(smem_x_u4 + 128 * X_U4_STRIDE);
    float* smem_red16 = smem_x_sum32 + 128;
    float* smem_qkvg = smem_red16 + 16;          // [96] floats
    float* smem_conv_pre = smem_qkvg + 96;       // [96] floats (p0*w0 + p1*w1 + p2*w2)
    float* smem_conv_w3 = smem_conv_pre + 96;    // [96] floats (w3)
    uint16_t* smem_p1 = reinterpret_cast<uint16_t*>(smem_conv_w3 + 96); // [96] u16
    uint16_t* smem_p2 = smem_p1 + 96;            // [96] u16

    int r0 = (bid * 16384) / nblocks;
    int r1 = ((bid + 1) * 16384) / nblocks;
    int n_rows = r1 - r0;
    int sub_id = tid >> 3;
    int t8 = tid & 7;
    int n_sub = (n_rows + 1) >> 1;
    const uint4* qkvg_wq = reinterpret_cast<const uint4*>(kda.qkvg_wq);
    const uint32_t* qkvg_sz = reinterpret_cast<const uint32_t*>(kda.qkvg_sz);
    const uint2* conv_w_u2 = reinterpret_cast<const uint2*>(kda.conv_w);

    bool has0 = false, has1 = false;
    const uint4* wr0 = nullptr; const uint4* wr1 = nullptr;
    const uint32_t* sr0 = nullptr; const uint32_t* sr1 = nullptr;
    uint4 wa0 = {}, wb0 = {}, wa1 = {}, wb1 = {};
    uint32_t sa0 = 0, sb0 = 0, sa1 = 0, sb1 = 0;
    int k0 = t8, k1 = 8 + t8;
    if (wid * 4 < n_sub) {
        int r = r0 + sub_id * 2;
        has0 = (r < r1);
        has1 = (r + 1 < r1);
        int rr0 = has0 ? r : (r1 - 1);
        int rr1 = has1 ? (r + 1) : rr0;
        wr0 = qkvg_wq + static_cast<size_t>(rr0) * 72;
        wr1 = qkvg_wq + static_cast<size_t>(rr1) * 72;
        sr0 = qkvg_sz + static_cast<size_t>(rr0) * 18;
        sr1 = qkvg_sz + static_cast<size_t>(rr1) * 18;
        wa0 = ld_u4_keep(wr0 + k0, pol_keep); wb0 = ld_u4_keep(wr1 + k0, pol_keep);
        wa1 = ld_u4_keep(wr0 + k1, pol_keep); wb1 = ld_u4_keep(wr1 + k1, pol_keep);
        sa0 = ld_u32_keep(sr0 + (k0 >> 2), pol_keep); sb0 = ld_u32_keep(sr1 + (k0 >> 2), pol_keep);
        sa1 = ld_u32_keep(sr0 + (k1 >> 2), pol_keep); sb1 = ld_u32_keep(sr1 + (k1 >> 2), pol_keep);
    } else if (wid >= 12 && wid <= 14) {
        int loc = tid - 384;
        if (loc < n_rows) {
            int row = r0 + loc;
            int kind = row >> 12;
            int c = row & 4095;
            if (kind < 3) {
                const uint16_t* prev = reinterpret_cast<const uint16_t*>(
                    (kind == 0) ? cq : ((kind == 1) ? ck : cv)
                );
                uint16_t p0 = ld_u16_keep(prev + c, pol_keep);
                uint16_t p1 = ld_u16_keep(prev + KDA_C + c, pol_keep);
                uint16_t p2 = ld_u16_keep(prev + 2 * KDA_C + c, pol_keep);
                uint2 cw = ld_u2_keep(conv_w_u2 + kind * KDA_C + c, pol_keep);
                float pre = __uint_as_float(uint32_t(p0) << 16) * __uint_as_float(cw.x << 16)
                          + __uint_as_float(uint32_t(p1) << 16) * __uint_as_float(cw.x & 0xFFFF0000u)
                          + __uint_as_float(uint32_t(p2) << 16) * __uint_as_float(cw.y << 16);
                smem_conv_pre[loc] = pre;
                smem_conv_w3[loc] = __uint_as_float(cw.y & 0xFFFF0000u);
                smem_p1[loc] = p1;
                smem_p2[loc] = p2;
            }
        }
    }

    if (in_hidden_or_null != nullptr) {
        load_input_and_rmsnorm_2304_to_smem_bf2(
            in_hidden_or_null, g_hidden,
            reinterpret_cast<const __nv_bfloat16*>(kda.attn_norm),
            smem_x_u4, smem_x_sum32, smem_red16, bid, pol_keep
        );
    } else {
        float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f);
        load_rmsnorm_2304_to_smem_bf2(
            g_hidden, reinterpret_cast<const __nv_bfloat16*>(kda.attn_norm),
            inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep
        );
    }

    // Phase K1: 1-wide 2-row BF16x2 GEMV (warps 0..10) + overlapped coalesced conv preload (warps 12..14) + beta (warp 15)
    {
        if (wid * 4 < n_sub) {
            float a0 = 0.0f, a1 = 0.0f;
            #pragma unroll
            for (int it = 0; it < 7; ++it) {
                int kn = (it + 2) * 8 + t8;
                uint4 nwa = ld_u4_keep(wr0 + kn, pol_keep), nwb = ld_u4_keep(wr1 + kn, pol_keep);
                uint32_t nsa = ld_u32_keep(sr0 + (kn >> 2), pol_keep), nsb = ld_u32_keep(sr1 + (kn >> 2), pol_keep);
                dot32_bf2_pair(wa0, sa0, wb0, sb0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], a0, a1);
                wa0 = wa1; wb0 = wb1; sa0 = sa1; sb0 = sb1; k0 = k1;
                wa1 = nwa; wb1 = nwb; sa1 = nsa; sb1 = nsb; k1 = kn;
            }
            dot32_bf2_pair(wa0, sa0, wb0, sb0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], a0, a1);
            dot32_bf2_pair(wa1, sa1, wb1, sb1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], a0, a1);
            a0 = subwarp8_reduce_sum(a0);
            a1 = subwarp8_reduce_sum(a1);
            if (t8 == 0) {
                if (has0) smem_qkvg[sub_id * 2] = a0;
                if (has1) smem_qkvg[sub_id * 2 + 1] = a1;
            }
        } else if (bid < KDA_H && wid == 15) {
            const uint4* bw_u4 = reinterpret_cast<const uint4*>(kda.beta_w) + static_cast<size_t>(bid) * 288;
            float acc = 0.0f;
            for (int q = lane; q < 288; q += 32) {
                uint4 v = ld_u4_keep(bw_u4 + q, pol_keep);
                uint4 x4 = smem_x_u4[(q >> 2) * X_U4_STRIDE + (q & 3)];
                acc = fmaf(__uint_as_float(x4.x << 16), __uint_as_float(v.x << 16), acc);
                acc = fmaf(__uint_as_float(x4.x & 0xFFFF0000u), __uint_as_float(v.x & 0xFFFF0000u), acc);
                acc = fmaf(__uint_as_float(x4.y << 16), __uint_as_float(v.y << 16), acc);
                acc = fmaf(__uint_as_float(x4.y & 0xFFFF0000u), __uint_as_float(v.y & 0xFFFF0000u), acc);
                acc = fmaf(__uint_as_float(x4.z << 16), __uint_as_float(v.z << 16), acc);
                acc = fmaf(__uint_as_float(x4.z & 0xFFFF0000u), __uint_as_float(v.z & 0xFFFF0000u), acc);
                acc = fmaf(__uint_as_float(x4.w << 16), __uint_as_float(v.w << 16), acc);
                acc = fmaf(__uint_as_float(x4.w & 0xFFFF0000u), __uint_as_float(v.w & 0xFFFF0000u), acc);
            }
            acc = warp_reduce_sum(acc);
            if (lane == 0) g_beta[bid] = sigmoid_f32(acc);
        }
        __syncthreads();

        // Coalesced 88-thread epilogue (1 FMA + SiLU + coalesced pol_keep stores!)
        if (tid < n_rows) {
            int row = r0 + tid;
            int kind = row >> 12;
            int c = row & 4095;
            __nv_bfloat16 val_bf = f32_to_bf16(smem_qkvg[tid]);
            float val_f = bf16_to_f32(val_bf);
            if (kind < 3) {
                float out = fmaf(val_f, smem_conv_w3[tid], smem_conv_pre[tid]);
                float s = bf16_to_f32(f32_to_bf16(silu_f32(out)));
                uint16_t* prev = reinterpret_cast<uint16_t*>(
                    (kind == 0) ? cq : ((kind == 1) ? ck : cv)
                );
                uint16_t val_u16 = *reinterpret_cast<uint16_t*>(&val_bf);
                st_u16_keep(prev + c, smem_p1[tid], pol_keep);
                st_u16_keep(prev + KDA_C + c, smem_p2[tid], pol_keep);
                st_u16_keep(prev + 2 * KDA_C + c, val_u16, pol_keep);
                if (kind == 0) g_q[c] = s * 0.08838834764831845f;
                else if (kind == 1) g_k[c] = s;
                else g_v[c] = s;
            } else {
                g_decay[c] = sigmoid_f32(-val_f);
            }
        }
    }

    if (bid == 0 && tid == 0) *g_norm_sq = 0.0f;
    grid.sync();

    // Phase K2: KDA Recurrent State Update S[32, 128, 128] (pre-issued S loads + tree warp reduce!)
    if (bid < 128) {
        int h = bid >> 2;
        int col_base = (bid & 3) * 32;
        int j = col_base + lane;

        float* smem_q = reinterpret_cast<float*>(smem_raw);
        float* smem_k = smem_q + 128;
        float* smem_dec = smem_k + 128;
        float* smem_red = smem_dec + 128;
        float* smem_delta = smem_red + 512;

        float* S_head = S_state + static_cast<size_t>(h) * 128 * 128;
        int r_base = wid * 8;
        float s_reg[8];
        #pragma unroll
        for (int r = 0; r < 8; ++r) {
            s_reg[r] = ld_f32_keep(S_head + (r_base + r) * 128 + j, pol_keep);
        }

        if (tid < 128) {
            smem_q[tid] = g_q[h * 128 + tid];
            smem_k[tid] = g_k[h * 128 + tid];
            smem_dec[tid] = g_decay[h * 128 + tid];
        }
        __syncthreads();

        float pred_part = 0.0f;
        #pragma unroll
        for (int r = 0; r < 8; ++r) {
            int i = r_base + r;
            float s = s_reg[r] * smem_dec[i];
            s_reg[r] = s;
            pred_part = fmaf(s, smem_k[i], pred_part);
        }

        smem_red[wid * 32 + lane] = pred_part;
        __syncthreads();

        if (wid == 0) {
            float a0 = smem_red[ 0 * 32 + lane] + smem_red[ 1 * 32 + lane];
            float a1 = smem_red[ 2 * 32 + lane] + smem_red[ 3 * 32 + lane];
            float a2 = smem_red[ 4 * 32 + lane] + smem_red[ 5 * 32 + lane];
            float a3 = smem_red[ 6 * 32 + lane] + smem_red[ 7 * 32 + lane];
            float a4 = smem_red[ 8 * 32 + lane] + smem_red[ 9 * 32 + lane];
            float a5 = smem_red[10 * 32 + lane] + smem_red[11 * 32 + lane];
            float a6 = smem_red[12 * 32 + lane] + smem_red[13 * 32 + lane];
            float a7 = smem_red[14 * 32 + lane] + smem_red[15 * 32 + lane];
            float pred = ((a0 + a1) + (a2 + a3)) + ((a4 + a5) + (a6 + a7));
            smem_delta[lane] = g_beta[h] * (g_v[h * 128 + j] - pred);
        }
        __syncthreads();

        float delta = smem_delta[lane];
        float o_part = 0.0f;
        #pragma unroll
        for (int r = 0; r < 8; ++r) {
            int i = r_base + r;
            float s_new = fmaf(smem_k[i], delta, s_reg[r]);
            st_f32_keep(S_head + i * 128 + j, s_new, pol_keep);
            o_part = fmaf(s_new, smem_q[i], o_part);
        }

        smem_red[wid * 32 + lane] = o_part;
        __syncthreads();

        if (wid == 0) {
            float b0 = smem_red[ 0 * 32 + lane] + smem_red[ 1 * 32 + lane];
            float b1 = smem_red[ 2 * 32 + lane] + smem_red[ 3 * 32 + lane];
            float b2 = smem_red[ 4 * 32 + lane] + smem_red[ 5 * 32 + lane];
            float b3 = smem_red[ 6 * 32 + lane] + smem_red[ 7 * 32 + lane];
            float b4 = smem_red[ 8 * 32 + lane] + smem_red[ 9 * 32 + lane];
            float b5 = smem_red[10 * 32 + lane] + smem_red[11 * 32 + lane];
            float b6 = smem_red[12 * 32 + lane] + smem_red[13 * 32 + lane];
            float b7 = smem_red[14 * 32 + lane] + smem_red[15 * 32 + lane];
            float o_val = ((b0 + b1) + (b2 + b3)) + ((b4 + b5) + (b6 + b7));
            g_attn_o[h * 128 + j] = f32_to_bf16(o_val);
        }
    }

    grid.sync();

    run_load_and_oproj<false>(
        g_attn_o,
        reinterpret_cast<const uint4*>(kda.o_wq),
        reinterpret_cast<const uint32_t*>(kda.o_sz),
        g_hidden, g_norm_sq,
        smem_x_u4, smem_x_sum32, smem_red16, bid, nblocks, pol_keep
    );
    grid.sync();
}

// ---------------------------------------------------------------------------
// MLA Attention block:
//   - Spin-free fused A1+A2 (nibble-transposed int4 W_uk, 1 uint4/thread, 0 spin-wait)
//   - Warp-aligned FlashMLA A3 (async cp.async Step-0 copy, exp2f online softmax, streaming part_o)
//   - Slim overlapped A4 (4-split part_o prefetch + 1-warp exp2f softmax + overlapped W_uv)
//   - Input-first overlapped A5 o_proj
// ---------------------------------------------------------------------------
__device__ __forceinline__ void run_mla_attn(
    cg::grid_group& grid,
    const MlaLayerPtrs& mla,
    const __nv_bfloat16* src_c_kv,
    const __nv_bfloat16* src_k_rope,
    __nv_bfloat16* dst_c_kv,
    __nv_bfloat16* dst_k_rope,
    int ctx_len,
    int need_copy,
    float rope_theta,
    __nv_bfloat16* g_hidden,
    float* g_norm_sq,
    float* g_q_full_f32,
    __nv_bfloat16* g_q_rope,
    float2* g_mla_part_ml,
    __nv_bfloat16* g_mla_part_o,
    float* g_attn_o_f32,
    char* smem_raw,
    int bid,
    int nblocks,
    uint64_t pol_keep,
    uint64_t pol_stream
) {
    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;

    uint4* smem_x_u4 = reinterpret_cast<uint4*>(smem_raw);
    float* smem_x_sum32 = reinterpret_cast<float*>(smem_x_u4 + 128 * X_U4_STRIDE);
    float* smem_red16 = smem_x_sum32 + 128;
    float* smem_cos = smem_red16 + 16;
    float* smem_sin = smem_cos + 32;
    float* smem_qpart = smem_sin + 32; // [64] floats for 2-way K-split q_nope!
    float* smem_qn = smem_qpart + 64;  // [32] floats local q_nope[h, d_blk*32..+31]!

    float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f);
    load_rmsnorm_2304_to_smem_bf2(
        g_hidden, reinterpret_cast<const __nv_bfloat16*>(mla.attn_norm),
        inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep
    );

    const uint4* qkva_wq = reinterpret_cast<const uint4*>(mla.qkva_wq);
    const uint32_t* qkva_sz = reinterpret_cast<const uint32_t*>(mla.qkva_sz);

    if (bid < 128) {
        if (tid < 32) g_attn_o_f32[bid * 32 + tid] = 0.0f;

        int h = bid >> 2;
        int sub_id = tid >> 3; // 0..63 (ALL 16 warps active!)
        int t8 = tid & 7;

        // Step 1: 2-WAY K-SPLIT 32 rows of q_nope[h, d_blk*32..+31] straight into SMEM!
        {
            int row_loc = sub_id & 31;
            int ks = sub_id >> 5; // 0 or 1
            int row = bid * 32 + row_loc;
            const uint4* w_row = qkva_wq + static_cast<size_t>(row) * 72;
            const uint32_t* sz_row = qkva_sz + static_cast<size_t>(row) * 18;

            float acc = 0.0f;
            if (ks == 0) {
                int k0 = 0 * 8 + t8, k1 = 1 * 8 + t8, k2 = 2 * 8 + t8, k3 = 3 * 8 + t8;
                uint4 w0 = ld_u4_keep(w_row + k0, pol_keep), w1 = ld_u4_keep(w_row + k1, pol_keep);
                uint4 w2 = ld_u4_keep(w_row + k2, pol_keep), w3 = ld_u4_keep(w_row + k3, pol_keep);
                uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep), s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep);
                uint32_t s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep), s3 = ld_u32_keep(sz_row + (k3 >> 2), pol_keep);
                acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
                acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
                acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
                acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]);
            } else {
                int k0 = 4 * 8 + t8, k1 = 5 * 8 + t8, k2 = 6 * 8 + t8;
                uint4 w0 = ld_u4_keep(w_row + k0, pol_keep), w1 = ld_u4_keep(w_row + k1, pol_keep), w2 = ld_u4_keep(w_row + k2, pol_keep);
                uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep), s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep), s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep);
                int k3 = 7 * 8 + t8, k4 = 8 * 8 + t8;
                uint4 w3 = ld_u4_keep(w_row + k3, pol_keep), w4 = ld_u4_keep(w_row + k4, pol_keep);
                uint32_t s3 = ld_u32_keep(sz_row + (k3 >> 2), pol_keep), s4 = ld_u32_keep(sz_row + (k4 >> 2), pol_keep);
                acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
                acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
                acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
                acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]);
                acc += dot32_bf2(w4, s4, smem_x_u4 + k4 * X_U4_STRIDE, smem_x_sum32[k4]);
            }
            acc = subwarp8_reduce_sum(acc);
            if (t8 == 0) smem_qpart[ks * 32 + row_loc] = acc;
        }
        // Pre-issue our single W_uk uint4 per thread BEFORE __syncthreads() (hides 100% of W_uk L2 latency!)
        const uint4* wuk_blk = reinterpret_cast<const uint4*>(mla.wuk_wq) + static_cast<size_t>(bid) * 512;
        uint4 w_uk = ld_u4_keep(wuk_blk + tid, pol_keep);
        __syncthreads();

        if (tid < 32) {
            smem_qn[tid] = bf16_to_f32(f32_to_bf16(smem_qpart[tid] + smem_qpart[32 + tid]));
        }
        __syncthreads();

        // Step 2: Spin-free W_uk (ZERO g_q_nope, ZERO g_mla_head_done spin-wait!)
        float* smem_qeff = reinterpret_cast<float*>(smem_raw); // [4 * 36] floats (reuses smem_x_u4, disjoint from smem_qn!)
        float* smem_bias = smem_qeff + 4 * X_F32_STRIDE;       // [4] floats
        if (wid < 4) {
            const uint32_t* sz_ptr = reinterpret_cast<const uint32_t*>(mla.wuk_sz) + static_cast<size_t>(bid) * 128;
            uint32_t sz = ld_u32_keep(sz_ptr + tid, pol_keep);
            float scale = __uint_as_float(sz << 16);
            float zero = __uint_as_float(sz & 0xFFFF0000u);
            float qe = smem_qn[lane] * scale;
            smem_qeff[wid * X_F32_STRIDE + lane] = qe;
            float b = warp_reduce_sum(qe * (zero + 128.0f));
            if (lane == 0) smem_bias[wid] = b;
        }
        __syncthreads();

        // ALL 512 threads: 1 uint4 -> 1 dot32 -> atomicAdd into g_q_full_f32[h, tid]!
        {
            int g = tid >> 7; // 0..3
            const float* xp = smem_qeff + g * X_F32_STRIDE;
            float dot = dot8_biased(w_uk.x, xp) + dot8_biased(w_uk.y, xp + 8)
                      + dot8_biased(w_uk.z, xp + 16) + dot8_biased(w_uk.w, xp + 24);
            atomicAdd(&g_q_full_f32[h * KV_LORA + tid], dot - smem_bias[g]);
        }
    } else {
        int r_bid = bid - 128;
        int r_nblocks = nblocks - 128;
        int pair0 = (r_bid * 1312) / r_nblocks;
        int pair1 = ((r_bid + 1) * 1312) / r_nblocks;
        int r0 = 4096 + pair0 * 2;
        int n_pairs = pair1 - pair0;

        int sub_id = tid >> 3;
        int p_loc = sub_id >> 1;
        int t8 = tid & 7;

        int n_warps = (n_pairs + 1) >> 1;
        float val_f = 0.0f, partner = 0.0f;
        bool act = false;
        int row = r0;
        if (wid < n_warps) {
            act = (p_loc < n_pairs);
            row = r0 + (act ? sub_id : 0);
            const uint4* w_row = qkva_wq + static_cast<size_t>(row) * 72;
            const uint32_t* sz_row = qkva_sz + static_cast<size_t>(row) * 18;
            float acc = 0.0f;
            #pragma unroll
            for (int g3 = 0; g3 < 9; g3 += 3) {
                int k0 = (g3 + 0) * 8 + t8;
                int k1 = (g3 + 1) * 8 + t8;
                int k2 = (g3 + 2) * 8 + t8;
                uint4 w0 = ld_u4_keep(w_row + k0, pol_keep);
                uint4 w1 = ld_u4_keep(w_row + k1, pol_keep);
                uint4 w2 = ld_u4_keep(w_row + k2, pol_keep);
                uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep);
                uint32_t s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep);
                uint32_t s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep);
                acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
                acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
                acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
            }
            acc = subwarp8_reduce_sum(acc);
            val_f = bf16_to_f32(f32_to_bf16(acc));
            partner = __shfl_xor_sync(0xffffffffu, val_f, 8);
        } else if (wid == 15) {
            float neg_log_theta = -__logf(rope_theta);
            float inv = __expf(float(2 * lane) * (1.0f / 64.0f) * neg_log_theta);
            float ang = float(ctx_len) * inv;
            float s, c;
            __sincosf(ang, &s, &c);
            smem_cos[lane] = c;
            smem_sin[lane] = s;
        }
        __syncthreads();
        if (act && t8 == 0) {
            if (row < 6144) {
                int rr = row - 4096;
                int h = rr >> 6;
                int rd = rr & 63;
                int pair = rd >> 1;
                float c = smem_cos[pair];
                float s = smem_sin[pair];
                float out_r = (rd & 1) ? (val_f * c + partner * s) : (val_f * c - partner * s);
                g_q_rope[h * QK_ROPE + rd] = f32_to_bf16(out_r);
            } else if (row < 6656) {
                dst_c_kv[static_cast<size_t>(ctx_len) * KV_LORA + (row - 6144)] = f32_to_bf16(val_f);
            } else {
                int rd = row - 6656;
                int pair = rd >> 1;
                float c = smem_cos[pair];
                float s = smem_sin[pair];
                float out_r = (rd & 1) ? (val_f * c + partner * s) : (val_f * c - partner * s);
                dst_k_rope[static_cast<size_t>(ctx_len) * QK_ROPE + rd] = f32_to_bf16(out_r);
            }
        }
    }

    if (bid == 0 && tid == 0) *g_norm_sq = 0.0f;
    grid.sync();

    // Phase A3: Zero-div72 Warp-Aligned FlashMLA + padded PART_O_STRIDE_U4 + pol_keep part_ml!
    int L = ctx_len + 1;
    int total_tiles = (L + 15) / 16;
    int cap_sp = (ctx_len <= 4096) ? 64 : (need_copy ? 128 : 96);
    int num_splits = (total_tiles < cap_sp) ? total_tiles : cap_sp;
    if (num_splits > nblocks) num_splits = nblocks;
    uint64_t kv_pol = (ctx_len <= 4096) ? pol_keep : pol_stream;

    if (bid < num_splits) {
        int tile_start = (bid * total_tiles) / num_splits;
        int tile_end   = ((bid + 1) * total_tiles) / num_splits;

        uint4* smem_Q_u4 = reinterpret_cast<uint4*>(smem_raw);
        uint4* smem_KV_u4 = smem_Q_u4 + 32 * 72;
        float* smem_S = reinterpret_cast<float*>(smem_KV_u4 + 2 * 16 * 72);
        __nv_bfloat16* smem_P = reinterpret_cast<__nv_bfloat16*>(smem_S + 4 * 32 * 16);
        float* smem_m = reinterpret_cast<float*>(smem_P + 32 * 24);
        float* smem_l = smem_m + 32;
        float* smem_alpha = smem_l + 32;

        const uint4* src_ckv_u4 = reinterpret_cast<const uint4*>(src_c_kv);
        const uint4* src_kr_u4  = reinterpret_cast<const uint4*>(src_k_rope);
        uint4* dst_ckv_u4       = reinterpret_cast<uint4*>(dst_c_kv);
        uint4* dst_kr_u4        = reinterpret_cast<uint4*>(dst_k_rope);

        // Warp-aligned separate c_kv (512B bursts) + k_rope (tid<128) via cp.async
        auto issue_kv_load = [&](int buf, int tile_idx) {
            int t_base = tile_idx * 16;
            uint4* buf_ptr = smem_KV_u4 + buf * (16 * 72);
            int r0 = tid >> 6;        // 0..7
            int c8 = tid & 63;        // 0..63
            int r1 = r0 + 8;          // 8..15
            int t0 = t_base + r0;
            int t1 = t_base + r1;
            uint4* dst0 = buf_ptr + r0 * 72 + (c8 ^ (r0 & 7));
            uint4* dst1 = buf_ptr + r1 * 72 + (c8 ^ (r1 & 7));

            if (!need_copy) {
                if (t0 < L) cp_async_cg_16_pol(dst0, dst_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_pol);
                else *dst0 = make_uint4(0u, 0u, 0u, 0u);
                if (t1 < L) cp_async_cg_16_pol(dst1, dst_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_pol);
                else *dst1 = make_uint4(0u, 0u, 0u, 0u);
                if (tid < 128) {
                    int rk = tid >> 3;
                    int rc8 = tid & 7;
                    int tk = t_base + rk;
                    uint4* dstk = buf_ptr + rk * 72 + ((64 + rc8) ^ (rk & 7));
                    if (tk < L) cp_async_cg_16_pol(dstk, dst_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_pol);
                    else *dstk = make_uint4(0u, 0u, 0u, 0u);
                }
            } else {
                if (t0 < ctx_len) cp_async_cg_16_pol(dst0, src_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_pol);
                else if (t0 < L)  cp_async_cg_16_pol(dst0, dst_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_pol);
                else *dst0 = make_uint4(0u, 0u, 0u, 0u);
                if (t1 < ctx_len) cp_async_cg_16_pol(dst1, src_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_pol);
                else if (t1 < L)  cp_async_cg_16_pol(dst1, dst_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_pol);
                else *dst1 = make_uint4(0u, 0u, 0u, 0u);
                if (tid < 128) {
                    int rk = tid >> 3, rc8 = tid & 7, tk = t_base + rk;
                    uint4* dstk = buf_ptr + rk * 72 + ((64 + rc8) ^ (rk & 7));
                    if (tk < ctx_len) cp_async_cg_16_pol(dstk, src_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_pol);
                    else if (tk < L)  cp_async_cg_16_pol(dstk, dst_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_pol);
                    else *dstk = make_uint4(0u, 0u, 0u, 0u);
                }
            }
            cp_async_commit();
        };

        issue_kv_load(0, tile_start);

        // Zero-div72 Q_ld via ld_u4_ca (leaves 100% of cp.async queue free for tile_start!)
        {
            const uint4* qf_u4 = reinterpret_cast<const uint4*>(g_q_full_f32);
            int h_shift = bid & 31;
            uint4 ua[4], ub[4];
            int hs[4], c8s[4];
            #pragma unroll
            for (int pass = 0; pass < 4; ++pass) {
                int idx = pass * BLOCK_THREADS + tid;
                hs[pass] = ((idx >> 6) + h_shift) & 31;
                c8s[pass] = idx & 63;
                size_t off = (static_cast<size_t>(hs[pass]) * 64 + c8s[pass]) * 2;
                ua[pass] = ld_u4_ca(qf_u4 + off + 0);
                ub[pass] = ld_u4_ca(qf_u4 + off + 1);
            }
            uint4 qrv = make_uint4(0u, 0u, 0u, 0u);
            int qrh = 0, qrc8 = 0;
            if (tid < 256) {
                qrh = ((tid >> 3) + h_shift) & 31;
                qrc8 = tid & 7;
                const uint4* qr_u4 = reinterpret_cast<const uint4*>(g_q_rope);
                qrv = ld_u4_ca(qr_u4 + qrh * 8 + qrc8);
            }
            #pragma unroll
            for (int pass = 0; pass < 4; ++pass) {
                float4 a = *reinterpret_cast<float4*>(&ua[pass]);
                float4 b = *reinterpret_cast<float4*>(&ub[pass]);
                int h = hs[pass], c8 = c8s[pass];
                smem_Q_u4[h * 72 + (c8 ^ (h & 7))] = make_uint4(
                    pack_bf16x2(a.x, a.y), pack_bf16x2(a.z, a.w),
                    pack_bf16x2(b.x, b.y), pack_bf16x2(b.z, b.w)
                );
            }
            if (tid < 256) {
                smem_Q_u4[qrh * 72 + ((64 + qrc8) ^ (qrh & 7))] = qrv;
            }
        }
        if (tid < 32) {
            smem_m[tid] = -1e30f;
            smem_l[tid] = 0.0f;
        }

        float o_acc[8][4];
        #pragma unroll
        for (int nt = 0; nt < 8; ++nt) {
            o_acc[nt][0] = 0.0f; o_acc[nt][1] = 0.0f;
            o_acc[nt][2] = 0.0f; o_acc[nt][3] = 0.0f;
        }

        int qm = wid >> 3;
        int qn = (wid >> 2) & 1;
        int qk = wid & 3;
        int pn = wid & 7;

        int buf = 0;
        for (int tile = tile_start; tile < tile_end; ++tile) {
            bool has_next = (tile + 1 < tile_end);
            if (has_next) {
                issue_kv_load(buf ^ 1, tile + 1);
                cp_async_wait<1>();
            } else {
                cp_async_wait<0>();
            }
            __syncthreads();

            uint4* kv_buf = smem_KV_u4 + buf * (16 * 72);
            if (need_copy) {
                int t_base = tile * 16;
                int r0 = tid >> 6, c8 = tid & 63, r1 = r0 + 8;
                int t0 = t_base + r0, t1 = t_base + r1;
                if (t0 < ctx_len) st_u4_stream(dst_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_buf[r0 * 72 + (c8 ^ (r0 & 7))], kv_pol);
                if (t1 < ctx_len) st_u4_stream(dst_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_buf[r1 * 72 + (c8 ^ (r1 & 7))], kv_pol);
                if (tid < 128) {
                    int rk = tid >> 3, rc8 = tid & 7, tk = t_base + rk;
                    if (tk < ctx_len) st_u4_stream(dst_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_buf[rk * 72 + ((64 + rc8) ^ (rk & 7))], kv_pol);
                }
            }

            float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f;
            int c8_start = qk * 18;
            #pragma unroll
            for (int ks = 0; ks < 9; ++ks) {
                int c8_b = c8_start + ks * 2;
                int ar = qm * 16 + (lane & 15);
                int ac8 = c8_b + ((lane >> 4) & 1);
                uint32_t a0, a1, a2, a3;
                ldmatrix_x4(a0, a1, a2, a3, smem_Q_u4 + ar * 72 + (ac8 ^ (ar & 7)));

                int br = qn * 8 + (lane & 7);
                int bc8 = c8_b + ((lane >> 3) & 1);
                uint32_t b0, b1;
                ldmatrix_x2(b0, b1, kv_buf + br * 72 + (bc8 ^ (br & 7)));

                mma_m16n8k16_bf16(s0, s1, s2, s3, a0, a1, a2, a3, b0, b1);
            }
            int s_r0 = qm * 16 + (lane >> 2);
            int s_r1 = s_r0 + 8;
            int s_c0 = qn * 8 + 2 * (lane & 3);
            float* s_dst = smem_S + qk * 512;
            s_dst[s_r0 * 16 + s_c0]     = s0;
            s_dst[s_r0 * 16 + s_c0 + 1] = s1;
            s_dst[s_r1 * 16 + s_c0]     = s2;
            s_dst[s_r1 * 16 + s_c0 + 1] = s3;
            __syncthreads();

            {
                int sub = lane >> 4;
                int tl = lane & 15;
                int t_glob = tile * 16 + tl;
                int h = wid * 2 + sub;
                float sc = (t_glob < L)
                    ? (smem_S[h * 16 + tl] + smem_S[512 + h * 16 + tl]
                       + smem_S[1024 + h * 16 + tl] + smem_S[1536 + h * 16 + tl]) * 0.10411755264281874f
                    : -1e30f;
                float m_tile = halfwarp16_reduce_max(sc);
                float m_old = smem_m[h];
                float m_new = fmaxf(m_old, m_tile);
                float p = (t_glob < L) ? exp2f(sc - m_new) : 0.0f;
                float l_tile = halfwarp16_reduce_sum(p);
                float alpha = exp2f(m_old - m_new);
                smem_P[h * 24 + tl] = f32_to_bf16(p);
                if (tl == 0) {
                    smem_alpha[h] = alpha;
                    smem_m[h] = m_new;
                    smem_l[h] = smem_l[h] * alpha + l_tile;
                }
            }
            __syncthreads();

            {
                int row0 = qm * 16 + (lane >> 2);
                int row1 = row0 + 8;
                if (tile > tile_start) {
                    float a0_s = smem_alpha[row0];
                    float a1_s = smem_alpha[row1];
                    #pragma unroll
                    for (int nt = 0; nt < 8; ++nt) {
                        o_acc[nt][0] *= a0_s; o_acc[nt][1] *= a0_s;
                        o_acc[nt][2] *= a1_s; o_acc[nt][3] *= a1_s;
                    }
                }

                int pr = qm * 16 + (lane & 15);
                int pc = (lane >> 4) ? 8 : 0;
                uint32_t pa0, pa1, pa2, pa3;
                ldmatrix_x4(pa0, pa1, pa2, pa3, smem_P + pr * 24 + pc);

                #pragma unroll
                for (int nt2 = 0; nt2 < 4; ++nt2) {
                    int vr = lane & 15;
                    int vc8 = pn * 8 + nt2 * 2 + ((lane >> 4) & 1);
                    uint32_t b00, b10, b01, b11;
                    ldmatrix_x4_trans(b00, b10, b01, b11, kv_buf + vr * 72 + (vc8 ^ (vr & 7)));
                    mma_m16n8k16_bf16(
                        o_acc[nt2 * 2][0], o_acc[nt2 * 2][1], o_acc[nt2 * 2][2], o_acc[nt2 * 2][3],
                        pa0, pa1, pa2, pa3, b00, b10
                    );
                    mma_m16n8k16_bf16(
                        o_acc[nt2 * 2 + 1][0], o_acc[nt2 * 2 + 1][1], o_acc[nt2 * 2 + 1][2], o_acc[nt2 * 2 + 1][3],
                        pa0, pa1, pa2, pa3, b01, b11
                    );
                }
            }
            if (has_next) __syncthreads();
            buf ^= 1;
        }

        int row0 = qm * 16 + (lane >> 2);
        int row1 = row0 + 8;
        float inv_l0 = 1.0f / smem_l[row0];
        float inv_l1 = 1.0f / smem_l[row1];
        float2 my_ml = (tid < 32) ? make_float2(smem_m[tid], smem_l[tid]) : make_float2(0.0f, 0.0f);
        __syncthreads();

        uint32_t* smem_O_u32 = reinterpret_cast<uint32_t*>(smem_raw);
        constexpr int O_U32_STRIDE = O_SMEM_STRIDE / 2;
        #pragma unroll
        for (int nt = 0; nt < 8; ++nt) {
            int col2 = (pn * 64 + nt * 8) / 2 + (lane & 3);
            smem_O_u32[row0 * O_U32_STRIDE + col2] = pack_bf16x2(o_acc[nt][0] * inv_l0, o_acc[nt][1] * inv_l0);
            smem_O_u32[row1 * O_U32_STRIDE + col2] = pack_bf16x2(o_acc[nt][2] * inv_l1, o_acc[nt][3] * inv_l1);
        }
        __syncthreads();

        uint4* smem_O_u4 = reinterpret_cast<uint4*>(smem_raw);
        uint4* g_po_u4 = reinterpret_cast<uint4*>(g_mla_part_o) + static_cast<size_t>(bid) * PART_O_STRIDE_U4;
        #pragma unroll
        for (int pass = 0; pass < 4; ++pass) {
            int idx = tid + pass * BLOCK_THREADS;
            int hh = idx >> 6;
            int cc = idx & 63;
            st_u4_stream(g_po_u4 + idx, smem_O_u4[hh * 65 + cc], pol_stream);
        }
        // Write part_ml LAST with pol_keep so it stays 100% L2-hot for A4!
        if (tid < 32) {
            st_f32x2_keep(g_mla_part_ml + tid * ML_STRIDE + bid, my_ml, pol_keep);
        }
    }

    grid.sync();

    // Phase A4: OVERLAPPED part_o prefetch + 1-warp softmax + OVERLAPPED W_uv!
    if (bid < 128) {
        int h = bid >> 2;
        int g = bid & 3;
        if (tid < 32) reinterpret_cast<uint4*>(g_q_full_f32)[bid * 32 + tid] = make_uint4(0u, 0u, 0u, 0u);

        float* smem_part_lat = reinterpret_cast<float*>(smem_raw);
        float* smem_olat = smem_part_lat + 32 * LAT_STRIDE;
        float* smem_olat_sum32 = smem_olat + 4 * X_F32_STRIDE;
        float* smem_w = smem_olat_sum32 + 16;

        // Pre-issue up to 4 splits (<=128 total) of g_mla_part_o across all 16 warps before softmax
        int sg = tid >> 4;
        int d8 = tid & 15;
        const uint4* po_u4 = reinterpret_cast<const uint4*>(g_mla_part_o)
                           + (static_cast<size_t>(h) * 64) + g * 16 + d8;
        int s0 = sg, s1 = sg + 32, s2 = sg + 64, s3 = sg + 96;
        uint4 v0 = (s0 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s0) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
        uint4 v1 = (s1 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s1) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
        uint4 v2 = (s2 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s2) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
        uint4 v3 = (s3 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s3) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);

        if (wid == 0) {
            const float2* ml_head = g_mla_part_ml + h * ML_STRIDE;
            float2 ml_reg[4];
            #pragma unroll
            for (int k = 0; k < 4; ++k) {
                int s = k * 32 + lane;
                ml_reg[k] = (s < num_splits) ? ld_f32x2_keep(ml_head + s, pol_keep) : make_float2(-1e30f, 0.0f);
            }
            float lm = fmaxf(fmaxf(ml_reg[0].x, ml_reg[1].x), fmaxf(ml_reg[2].x, ml_reg[3].x));
            #pragma unroll
            for (int off = 16; off > 0; off >>= 1) lm = fmaxf(lm, __shfl_xor_sync(0xffffffffu, lm, off));

            float lw = 0.0f;
            #pragma unroll
            for (int k = 0; k < 4; ++k) {
                int s = k * 32 + lane;
                float w = (s < num_splits) ? (ml_reg[k].y * exp2f(ml_reg[k].x - lm)) : 0.0f;
                ml_reg[k].y = w;
                lw += w;
            }
            float inv_l = 1.0f / warp_reduce_sum(lw);
            #pragma unroll
            for (int k = 0; k < 4; ++k) {
                int s = k * 32 + lane;
                if (s < num_splits) smem_w[s] = ml_reg[k].y * inv_l;
            }
        }
        __syncthreads();

        {

            float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;
            float a4 = 0.0f, a5 = 0.0f, a6 = 0.0f, a7 = 0.0f;
            auto acc_v = [&](int s, uint4 v) {
                if (s < num_splits) {
                    float w = smem_w[s];
                    a0 = fmaf(w, __uint_as_float(v.x << 16), a0);
                    a1 = fmaf(w, __uint_as_float(v.x & 0xFFFF0000u), a1);
                    a2 = fmaf(w, __uint_as_float(v.y << 16), a2);
                    a3 = fmaf(w, __uint_as_float(v.y & 0xFFFF0000u), a3);
                    a4 = fmaf(w, __uint_as_float(v.z << 16), a4);
                    a5 = fmaf(w, __uint_as_float(v.z & 0xFFFF0000u), a5);
                    a6 = fmaf(w, __uint_as_float(v.w << 16), a6);
                    a7 = fmaf(w, __uint_as_float(v.w & 0xFFFF0000u), a7);
                }
            };
            acc_v(s0, v0); acc_v(s1, v1); acc_v(s2, v2); acc_v(s3, v3);

            float* dst = smem_part_lat + sg * LAT_STRIDE + d8 * 8;
            *reinterpret_cast<float4*>(dst)     = make_float4(a0, a1, a2, a3);
            *reinterpret_cast<float4*>(dst + 4) = make_float4(a4, a5, a6, a7);
        }
        // Pre-issue W_uv load BEFORE olat reduction (hides W_uv L2 latency completely!)
        int d_out = tid >> 2;
        int t4 = tid & 3;
        int k32 = g * 4 + t4;
        const uint4* wuv_row = reinterpret_cast<const uint4*>(mla.wuv_wq) + (static_cast<size_t>(h) * V_HEAD + d_out) * 16;
        const uint32_t* sz_row = reinterpret_cast<const uint32_t*>(mla.wuv_sz) + (static_cast<size_t>(h) * V_HEAD + d_out) * 4;
        uint4 w_uv = ld_u4_keep(wuv_row + k32, pol_keep);
        uint32_t sz_uv = ld_u32_keep(sz_row + g, pol_keep);
        __syncthreads();

        {
            int d = d_out;
            int q4 = t4;
            float s = 0.0f;
            #pragma unroll
            for (int r = 0; r < 8; ++r) {
                s += smem_part_lat[(q4 * 8 + r) * LAT_STRIDE + d];
            }
            s = subwarp4_reduce_sum(s);
            if (q4 == 0) {
                smem_olat[(d >> 5) * X_F32_STRIDE + (d & 31)] = s;
            }
        }
        __syncthreads();
        if (wid < 4) {
            float s32 = warp_reduce_sum(smem_olat[wid * X_F32_STRIDE + lane]);
            if (lane == 0) smem_olat_sum32[wid] = s32;
        }
        __syncthreads();

        {
            float acc = subwarp4_reduce_sum(dot32_w4_f32(w_uv, sz_uv, smem_olat + t4 * X_F32_STRIDE, smem_olat_sum32[t4]));
            if (t4 == 0) atomicAdd(&g_attn_o_f32[h * V_HEAD + d_out], acc);
        }
    }

    grid.sync();

    run_load_and_oproj<true>(
        g_attn_o_f32,
        reinterpret_cast<const uint4*>(mla.o_wq),
        reinterpret_cast<const uint32_t*>(mla.o_sz),
        g_hidden, g_norm_sq,
        smem_x_u4, smem_x_sum32, smem_red16, bid, nblocks, pol_keep
    );
    grid.sync();
}

// ---------------------------------------------------------------------------
// THE SINGLE MEGAKERNEL V11
// ---------------------------------------------------------------------------
__global__ __launch_bounds__(BLOCK_THREADS, 1)
void kimi_decode_megakernel(const MegaParams p) {
    extern __shared__ char smem_raw[];
    cg::grid_group grid = cg::this_grid();
    int bid = blockIdx.x;
    int nblocks = gridDim.x;

    uint64_t pol_keep, pol_stream;
    if (p.need_copy) asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 0.20;" : "=l"(pol_keep));
    else asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;" : "=l"(pol_keep));
    asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(pol_stream));

    __nv_bfloat16* g_hidden = reinterpret_cast<__nv_bfloat16*>(p.g_hidden);
    __nv_bfloat16* g_attn_o = reinterpret_cast<__nv_bfloat16*>(p.g_attn_o);
    __nv_bfloat16* g_moe_hh = reinterpret_cast<__nv_bfloat16*>(p.g_moe_hh);

    #pragma unroll 1
    for (int layer = 0; layer < 3; ++layer) {
        run_kda_attn(
            grid,
            p.kda[layer],
            p.kda_S[layer],
            reinterpret_cast<__nv_bfloat16*>(p.kda_cq[layer]),
            reinterpret_cast<__nv_bfloat16*>(p.kda_ck[layer]),
            reinterpret_cast<__nv_bfloat16*>(p.kda_cv[layer]),
            (layer == 0) ? reinterpret_cast<const __nv_bfloat16*>(p.in_hidden) : nullptr,
            g_hidden, p.g_norm_sq,
            p.g_q, p.g_k, p.g_v, p.g_decay, p.g_beta, g_attn_o,
            smem_raw, bid, nblocks, pol_keep
        );
        run_moe_layer(
            grid,
            p.moe[layer],
            g_hidden, nullptr, p.g_norm_sq,
            p.g_router_logits, &p.g_router_done[layer], g_moe_hh,
            p.routed_scaling, p.n_experts,
            smem_raw, bid, nblocks, pol_keep, pol_stream
        );
        grid.sync();
    }

    run_mla_attn(
        grid,
        p.mla,
        reinterpret_cast<const __nv_bfloat16*>(p.src_c_kv),
        reinterpret_cast<const __nv_bfloat16*>(p.src_k_rope),
        reinterpret_cast<__nv_bfloat16*>(p.dst_c_kv),
        reinterpret_cast<__nv_bfloat16*>(p.dst_k_rope),
        p.ctx_len, p.need_copy, p.rope_theta,
        g_hidden, p.g_norm_sq,
        p.g_q_full_f32, reinterpret_cast<__nv_bfloat16*>(p.g_q_rope),
        reinterpret_cast<float2*>(p.g_mla_part_ml), reinterpret_cast<__nv_bfloat16*>(p.g_mla_part_o),
        p.g_attn_o_f32, smem_raw, bid, nblocks, pol_keep, pol_stream
    );
    run_moe_layer(
        grid,
        p.moe[3],
        g_hidden, reinterpret_cast<__nv_bfloat16*>(p.out_hidden),
        p.g_norm_sq,
        p.g_router_logits, &p.g_router_done[3], g_moe_hh,
        p.routed_scaling, p.n_experts,
        smem_raw, bid, nblocks, pol_keep, pol_stream
    );
}

extern "C" void launch_kimi_megakernel(
    const MegaParams* h_params,
    void* /*dense_arena_ptr*/,
    size_t /*dense_arena_bytes*/,
    cudaStream_t stream
) {
    static int s_sm_count = 0;
    static bool s_attr_set = false;

    if (!s_attr_set) {
        cudaDeviceProp prop;
        cudaGetDeviceProperties(&prop, 0);
        s_sm_count = prop.multiProcessorCount;
        cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, prop.persistingL2CacheMaxSize);
        cudaFuncSetAttribute(kimi_decode_megakernel, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_BYTES);
        s_attr_set = true;
    }

    MegaParams p = *h_params;
    void* args[] = { &p };
    cudaLaunchCooperativeKernel(
        (void*)kimi_decode_megakernel,
        dim3(s_sm_count),
        dim3(BLOCK_THREADS),
        args,
        SMEM_BYTES,
        stream
    );
}
"""

_EXT = None


def _get_ext():
    global _EXT
    if _EXT is None:
        _EXT = load_inline(
            name="kimi_linear_megakernel_v11",
            cpp_sources=_CPP_SRC,
            cuda_sources=_CUDA_SRC,
            functions=["create_handle", "destroy_handle", "step_megakernel"],
            extra_cuda_cflags=[
                "-O3",
                "-gencode=arch=compute_120,code=sm_120",
                "--expt-relaxed-constexpr",
            ],
            no_implicit_headers=True,
        )
    return _EXT


def _pack_sz(scales: torch.Tensor, zeros: torch.Tensor) -> torch.Tensor:
    s_u16 = scales.contiguous().view(torch.uint16).to(torch.int32)
    z_u16 = zeros.contiguous().view(torch.uint16).to(torch.int32)
    return (s_u16 | (z_u16 << 16)).contiguous()


def _pack_linear(ql: QuantLinear):
    wq_t = ql.w_q.transpose(0, 1).contiguous()
    sz_t = _pack_sz(ql.scales.transpose(0, 1), ql.zeros.transpose(0, 1))
    return wq_t, sz_t


def _pack_linear_cat(qls: list[QuantLinear]):
    wqs = [q.w_q.transpose(0, 1).contiguous() for q in qls]
    szs = [_pack_sz(q.scales.transpose(0, 1), q.zeros.transpose(0, 1)) for q in qls]
    return torch.cat(wqs, dim=0).contiguous(), torch.cat(szs, dim=0).contiguous()


def _pack_bf16_interleave8(w: torch.Tensor) -> torch.Tensor:
    orig_shape = w.shape
    return w.contiguous().view(*orig_shape[:-1], orig_shape[-1] // 8, 2, 4).transpose(-1, -2).contiguous().view(orig_shape)


def _pack_into_arena(tensors: list[torch.Tensor], dev: torch.device) -> tuple[torch.Tensor, list[torch.Tensor]]:
    offsets = []
    cur = 0
    for t in tensors:
        nbytes = t.numel() * t.element_size()
        offsets.append((cur, nbytes, t.shape, t.dtype))
        cur = (cur + nbytes + 127) & ~127
    arena = torch.empty(cur, dtype=torch.uint8, device=dev)
    views = []
    for t, (off, nbytes, shape, dtype) in zip(tensors, offsets):
        v = arena[off : off + nbytes].view(dtype).view(shape)
        v.copy_(t)
        views.append(v)
    return arena, views


class Model(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern)
        self._handle = 0
        self._packed_tensors = None
        self._dense_arena = None
        self._scratch = None
        self._packed_device = None
        self._last_kv_ptr = 0
        self._last_kr_ptr = 0
        self._last_kv_buf = None
        self._last_kr_buf = None

    def __del__(self):
        if getattr(self, "_handle", 0) != 0 and _EXT is not None:
            try:
                _EXT.destroy_handle(self._handle)
            except Exception:
                pass
            self._handle = 0

    def _invalidate_packed(self):
        if self._handle != 0 and _EXT is not None:
            _EXT.destroy_handle(self._handle)
        self._handle = 0
        self._packed_tensors = None
        self._dense_arena = None
        self._scratch = None
        self._packed_device = None

    def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
        res = super().load_state_dict(state_dict, strict=strict, assign=assign)
        self._invalidate_packed()
        if next(self.parameters()).is_cuda:
            self._build_packed()
        return res

    def _build_packed(self):
        dev = next(self.parameters()).device
        ext = _get_ext()
        cfg = self.cfg

        dense_list = []

        for i in range(3):
            blk = self.blocks[i]
            kda = blk.attn
            qkvg_wq, qkvg_sz = _pack_linear_cat([kda.q_proj, kda.k_proj, kda.v_proj, kda.g_proj])
            o_wq, o_sz = _pack_linear(kda.o_proj)
            dense_list.extend([
                blk.attn_norm.contiguous(),
                qkvg_wq, qkvg_sz,
                _pack_bf16_interleave8(kda.beta_proj.weight),
                kda.conv_w.contiguous(),
                o_wq, o_sz,
            ])

        blk3 = self.blocks[3]
        mla = blk3.attn
        MLA_H = cfg.mla_heads
        q_wq_t = mla.q_proj.w_q.transpose(0, 1).contiguous().view(MLA_H, cfg.qk_nope + cfg.qk_rope, -1)
        q_sz_t = _pack_sz(mla.q_proj.scales.transpose(0, 1), mla.q_proj.zeros.transpose(0, 1)).view(MLA_H, cfg.qk_nope + cfg.qk_rope, -1)
        kva_wq_t = mla.kv_a.w_q.transpose(0, 1).contiguous()
        kva_sz_t = _pack_sz(mla.kv_a.scales.transpose(0, 1), mla.kv_a.zeros.transpose(0, 1))
        qkva_wq = torch.cat([
            q_wq_t[:, : cfg.qk_nope, :].reshape(MLA_H * cfg.qk_nope, -1),
            q_wq_t[:, cfg.qk_nope :, :].reshape(MLA_H * cfg.qk_rope, -1),
            kva_wq_t,
        ], dim=0).contiguous()
        qkva_sz = torch.cat([
            q_sz_t[:, : cfg.qk_nope, :].reshape(MLA_H * cfg.qk_nope, -1),
            q_sz_t[:, cfg.qk_nope :, :].reshape(MLA_H * cfg.qk_rope, -1),
            kva_sz_t,
        ], dim=0).contiguous()

        # Pure int4 nibble transpose of W_uk from packing-along-kv_lora to packing-along-q_nope
        w_raw = mla.kv_b.w_q.view(256, MLA_H, 256)[:, :, :128]
        u_r = torch.stack([w_raw & 0xF, w_raw >> 4], dim=1).reshape(512, MLA_H, 4, 16, 2)
        wuk_wq = (u_r[..., 0] | (u_r[..., 1] << 4)).permute(1, 2, 0, 3).contiguous()
        kvb_s = mla.kv_b.scales.view(4, MLA_H, 256)
        kvb_z = mla.kv_b.zeros.view(4, MLA_H, 256)
        wuk_sz = _pack_sz(
            kvb_s[:, :, :128].view(4, MLA_H, 4, 32).permute(1, 2, 0, 3),
            kvb_z[:, :, :128].view(4, MLA_H, 4, 32).permute(1, 2, 0, 3),
        )
        wuv_wq = mla.kv_b.w_q.view(256, MLA_H, 256)[:, :, 128:].permute(1, 2, 0).contiguous()
        wuv_sz = _pack_sz(
            kvb_s[:, :, 128:].permute(1, 2, 0),
            kvb_z[:, :, 128:].permute(1, 2, 0),
        )
        mla_o_wq, mla_o_sz = _pack_linear(mla.o_proj)
        dense_list.extend([
            blk3.attn_norm.contiguous(),
            qkva_wq, qkva_sz,
            wuk_wq, wuk_sz,
            wuv_wq, wuv_sz,
            mla_o_wq, mla_o_sz,
        ])

        routed_list = []
        for i in range(4):
            blk = self.blocks[i]
            moe = blk.moe
            sg_wq = moe.s_gate.w_q[0].transpose(0, 1)
            su_wq = moe.s_up.w_q[0].transpose(0, 1)
            s_gu_wq = torch.stack([sg_wq, su_wq], dim=1).contiguous()
            s_gu_sz = torch.stack([
                _pack_sz(moe.s_gate.scales[0].transpose(0, 1), moe.s_gate.zeros[0].transpose(0, 1)),
                _pack_sz(moe.s_up.scales[0].transpose(0, 1), moe.s_up.zeros[0].transpose(0, 1)),
            ], dim=1).contiguous()
            s_d_wq = moe.s_down.w_q[0].transpose(0, 1).contiguous()
            s_d_sz = _pack_sz(moe.s_down.scales[0].transpose(0, 1), moe.s_down.zeros[0].transpose(0, 1))

            dense_list.extend([
                blk.moe_norm.contiguous(),
                _pack_bf16_interleave8(moe.router.weight),
                s_gu_wq, s_gu_sz,
                s_d_wq, s_d_sz,
            ])

            rg_wq = moe.gate.w_q.transpose(1, 2)
            ru_wq = moe.up.w_q.transpose(1, 2)
            r_gu_wq = torch.stack([rg_wq, ru_wq], dim=2).contiguous()
            r_gu_sz = torch.stack([
                _pack_sz(moe.gate.scales.transpose(1, 2), moe.gate.zeros.transpose(1, 2)),
                _pack_sz(moe.up.scales.transpose(1, 2), moe.up.zeros.transpose(1, 2)),
            ], dim=2).contiguous()
            r_d_wq = moe.down.w_q.transpose(1, 2).contiguous()
            r_d_sz = _pack_sz(moe.down.scales.transpose(1, 2), moe.down.zeros.transpose(1, 2))
            routed_list.extend([r_gu_wq, r_gu_sz, r_d_wq, r_d_sz])

        arena, dense_views = _pack_into_arena(dense_list, dev)
        kda_tensors = dense_views[0:21]
        mla_tensors = dense_views[21:30]
        moe_tensors = []
        for i in range(4):
            moe_tensors.extend(dense_views[30 + i * 6 : 30 + (i + 1) * 6])
            moe_tensors.extend(routed_list[i * 4 : (i + 1) * 4])

        scratch = [
            torch.empty(cfg.hidden, dtype=torch.bfloat16, device=dev),            # 0: g_hidden
            torch.zeros(1, dtype=torch.float32, device=dev),                      # 1: g_norm_sq
            torch.empty(4096, dtype=torch.float32, device=dev),                   # 2: g_q
            torch.empty(4096, dtype=torch.float32, device=dev),                   # 3: g_k
            torch.empty(4096, dtype=torch.float32, device=dev),                   # 4: g_v
            torch.empty(4096, dtype=torch.float32, device=dev),                   # 5: g_decay
            torch.empty(32, dtype=torch.float32, device=dev),                     # 6: g_beta
            torch.empty(4096, dtype=torch.bfloat16, device=dev),                  # 7: g_attn_o
            torch.empty(4096, dtype=torch.float32, device=dev),                   # 8: g_attn_o_f32
            torch.zeros(32 * 512, dtype=torch.float32, device=dev),               # 9: g_q_full_f32
            torch.empty(32 * 64, dtype=torch.bfloat16, device=dev),               # 10: g_q_rope
            torch.empty(32 * 192 * 2, dtype=torch.float32, device=dev),           # 11: g_mla_part_ml (float2)
            torch.empty(188 * 2056 * 8, dtype=torch.bfloat16, device=dev),        # 12: g_mla_part_o (padded stride!)
            torch.empty(cfg.n_experts, dtype=torch.float32, device=dev),          # 13: g_router_logits
            torch.zeros(4, dtype=torch.int32, device=dev),                        # 14: g_router_done
            torch.empty(9 * cfg.moe_inter, dtype=torch.bfloat16, device=dev),     # 15: g_moe_hh
        ]

        self._dense_arena = arena
        self._packed_tensors = dense_views + routed_list
        self._scratch = scratch
        self._packed_device = dev
        self._handle = ext.create_handle(
            kda_tensors, mla_tensors, moe_tensors, scratch, arena,
            float(cfg.routed_scaling), float(cfg.rope_theta), int(cfg.n_experts),
        )

    def step(self, hidden: torch.Tensor, state: list):
        if self._handle == 0 or self._packed_device != hidden.device:
            self._build_packed()

        s0, s1, s2, s3 = state[0], state[1], state[2], state[3]
        src_ckv = s3["c_kv"]
        src_kr = s3["k_rope"]
        ctx_len = src_ckv.shape[0]

        if (
            self._last_kv_buf is not None
            and src_ckv.data_ptr() == self._last_kv_ptr
            and src_kr.data_ptr() == self._last_kr_ptr
            and ctx_len < self._last_kv_buf.shape[0]
        ):
            dst_ckv = self._last_kv_buf
            dst_kr = self._last_kr_buf
            need_copy = 0
        else:
            cap = ctx_len + 64
            dst_ckv = torch.empty((cap, src_ckv.shape[1]), dtype=src_ckv.dtype, device=src_ckv.device)
            dst_kr = torch.empty((cap, src_kr.shape[1]), dtype=src_kr.dtype, device=src_kr.device)
            self._last_kv_buf = dst_ckv
            self._last_kr_buf = dst_kr
            self._last_kv_ptr = dst_ckv.data_ptr()
            self._last_kr_ptr = dst_kr.data_ptr()
            need_copy = 1

        out = _EXT.step_megakernel(
            self._handle,
            hidden,
            s0["S"], s0["cq"], s0["ck"], s0["cv"],
            s1["S"], s1["cq"], s1["ck"], s1["cv"],
            s2["S"], s2["cq"], s2["ck"], s2["cv"],
            src_ckv, src_kr,
            dst_ckv, dst_kr,
            ctx_len, need_copy,
        )

        s3["c_kv"] = dst_ckv[: ctx_len + 1]
        s3["k_rope"] = dst_kr[: ctx_len + 1]
        return out, state

20261118_150415_agy_gemini-4-neon-xhigh_02_kimi_linear_decode