KernelBench mega · RTX PRO 6000

Kimi-Linear Decode Gemini 4 Carbon (High)

24.68×geomean speedup across shapes

manually audited: clean

Re-graded 2026-11-16 under grader v3 on tetra: 24.68x (23.76x before; see regrade_v3). Original audit: Gemini 4 Carbon wrote one persistent CUDA kernel, 376 CTAs (two per SM) held together by a single-word release/acquire grid barrier, that runs the whole decode token: three KDA layers, the absorbed MLA layer and a 64-expert top-8 plus shared-expert MoE after each. The int4 GEMVs build each weight as a bf16 "128 + nibble" with one OR (0x43004300) and fold the zero point in once per group against precomputed 32-element activation sums; activations stay fp32 from the bf16 RMSNorm output, so it is a true W4A16 GEMV. MLA attention runs in the 512-dim latent space on bf16 mma.sync tensor cores with a two-stage cp.async pipeline over up to 128 KV splits, and dense weights are pinned in L2 with evict_last hints while routed experts stream evict_first.

harnessagy
Kernel source (redacted)
"""Single-launch CUDA megakernel for Kimi-Linear W4A16 hybrid decode."""
from __future__ import annotations

from dataclasses import dataclass, field
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

OP_TYPE = "kimi_linear_w4a16_decode"
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
EPS = 1.0e-6
GROUP_SIZE = 128


@dataclass(frozen=True)
class Config:
    hidden: int = 2304
    kda_heads: int = 32
    kda_head_dim: int = 128
    short_conv: int = 4
    mla_heads: int = 32
    kv_lora: int = 512
    qk_nope: int = 128
    qk_rope: int = 64
    v_head: int = 128
    rope_theta: float = 10000.0
    n_experts: int = 64
    n_active: int = 8
    n_shared: int = 1
    moe_inter: int = 1024
    routed_scaling: float = 2.446
    group: int = 128
    pattern: tuple = ("K", "K", "K", "M")
    dtype: torch.dtype = field(default=torch.bfloat16)


def build_config(shape: dict) -> Config:
    return Config(n_experts=int(shape.get("n_experts", 64)))


def _pack_int4(w_q: torch.Tensor) -> torch.Tensor:
    lo = w_q[0::2] & 0xF
    hi = w_q[1::2] & 0xF
    return (lo | (hi << 4)).contiguous()


def quantize(w_io: torch.Tensor, group: int = GROUP_SIZE):
    K, N = w_io.shape
    ng = K // group
    wg = w_io.view(ng, group, N).float()
    wmin = wg.min(dim=1, keepdim=True).values
    wmax = wg.max(dim=1, keepdim=True).values
    scales = (wmax - wmin).clamp_min(1e-8) / 15.0
    zeros = (-wmin / scales).round().clamp(0, 15)
    w_q = ((wg / scales) + zeros).round().clamp(0, 15).to(torch.uint8).view(K, N)
    return (
        _pack_int4(w_q),
        scales.squeeze(1).to(torch.bfloat16),
        zeros.squeeze(1).to(torch.bfloat16),
    )


class QuantLinear(nn.Module):
    def __init__(self, in_f: int, out_f: int, group: int = GROUP_SIZE):
        super().__init__()
        assert in_f % group == 0 and in_f % 2 == 0
        self.in_f, self.out_f, self.group = in_f, out_f, group
        ng = in_f // group
        self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8))
        self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16))
        self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16))

    def init_random(self, gen: torch.Generator, std: float = 0.02) -> None:
        w = torch.randn(self.in_f, self.out_f, generator=gen) * std
        wq, s, z = quantize(w, self.group)
        self.w_q.copy_(wq)
        self.scales.copy_(s)
        self.zeros.copy_(z)


class QuantExperts(nn.Module):
    def __init__(self, n: int, in_f: int, out_f: int, group: int = GROUP_SIZE):
        super().__init__()
        self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group
        ng = in_f // group
        self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8))
        self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
        self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))

    def init_random(self, gen: torch.Generator, std: float = 0.02) -> None:
        for e in range(self.n):
            w = torch.randn(self.in_f, self.out_f, generator=gen) * std
            wq, s, z = quantize(w, self.group)
            self.w_q[e].copy_(wq)
            self.scales[e].copy_(s)
            self.zeros[e].copy_(z)


class KDA(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden
        self.q_proj = QuantLinear(d, H * Dk, cfg.group)
        self.k_proj = QuantLinear(d, H * Dk, cfg.group)
        self.v_proj = QuantLinear(d, H * Dk, cfg.group)
        self.g_proj = QuantLinear(d, H * Dk, cfg.group)
        self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype)
        self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype))
        self.o_proj = QuantLinear(H * Dk, d, cfg.group)


class MLA(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        H, d = cfg.mla_heads, cfg.hidden
        self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group)
        self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group)
        self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group)
        self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group)


class MoE(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts
        self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype)
        self.gate = QuantExperts(E, d, m, cfg.group)
        self.up = QuantExperts(E, d, m, cfg.group)
        self.down = QuantExperts(E, m, d, cfg.group)
        self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group)
        self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group)
        self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group)


class Block(nn.Module):
    def __init__(self, cfg: Config, kind: str):
        super().__init__()
        self.kind = kind
        self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
        self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
        self.attn = KDA(cfg) if kind == "K" else MLA(cfg)
        self.moe = MoE(cfg)


_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
#include <string.h>
#include <math.h>

constexpr int HIDDEN = 2304;
constexpr int KDA_DK = 128;
constexpr int KDA_C = 4096;
constexpr int KV_LORA = 512;
constexpr int QK_NOPE = 128;
constexpr int QK_ROPE = 64;
constexpr int V_HEAD = 128;
constexpr int MOE_INTER = 1024;
constexpr int GROUP = 128;
constexpr int QGROUP = 32;
constexpr int BLOCK_THREADS = 256;
constexpr int MAX_KV_SPLITS = 128;
constexpr int KV_SPLIT_STRIDE = 128;

// Sense-flipping single-word GPU release-acquire barrier across all resident CTAs
struct FastGridBarrier {
    uint32_t* arrived;
    __device__ __forceinline__ void sync() {
        __barrier_sync(0);
        if (threadIdx.x == 0) {
            uint32_t nb = (blockIdx.x == 0) ? (0x80000000u - (gridDim.x - 1u)) : 1u;
            uint32_t old_arrive;
            asm volatile("atom.add.release.gpu.u32 %0, [%1], %2;\n"
                         : "=r"(old_arrive)
                         : "l"(arrived), "r"(nb)
                         : "memory");
            uint32_t cur_arrive;
            do {
                asm volatile("ld.acquire.gpu.u32 %0, [%1];\n"
                             : "=r"(cur_arrive)
                             : "l"(arrived)
                             : "memory");
            } while (((old_arrive ^ cur_arrive) & 0x80000000u) == 0u);
        }
        __barrier_sync(0);
    }
};

struct QuantLinearPtrs {
    const uint8_t* w_q;
    const __nv_bfloat16* scales;
    const __nv_bfloat16* zeros;
};

struct KDAPtrs {
    QuantLinearPtrs q_proj;
    QuantLinearPtrs k_proj;
    QuantLinearPtrs v_proj;
    QuantLinearPtrs g_proj;
    const __nv_bfloat16* beta_w; // [32, 2304]
    const __nv_bfloat16* conv_w; // [3, 4096, 4]
    QuantLinearPtrs o_proj;      // [4096, 2304]
};

struct MLAPtrs {
    QuantLinearPtrs q_proj;      // [2304, 6144]
    QuantLinearPtrs kv_a;        // [2304, 576]
    QuantLinearPtrs kv_b;        // [512, 8192]
    QuantLinearPtrs o_proj;      // [4096, 2304]
};

struct MoEPtrs {
    const __nv_bfloat16* router_w; // [64, 2304]
    QuantLinearPtrs gate;          // [64, 1152, 1024]
    QuantLinearPtrs up;            // [64, 1152, 1024]
    QuantLinearPtrs down;          // [64, 512, 2304]
    QuantLinearPtrs s_gate;        // [1, 1152, 1024]
    QuantLinearPtrs s_up;          // [1, 1152, 1024]
    QuantLinearPtrs s_down;        // [1, 512, 2304]
};

struct KDAStatePtrs {
    float* S;            // [32, 128, 128]
    __nv_bfloat16* cq;   // [3, 4096]
    __nv_bfloat16* ck;   // [3, 4096]
    __nv_bfloat16* cv;   // [3, 4096]
};

struct MegakernelParams {
    const __nv_bfloat16* attn_norm[4];
    const __nv_bfloat16* moe_norm[4];
    KDAPtrs kda[3];
    MLAPtrs mla;
    MoEPtrs moe[4];

    KDAStatePtrs kda_st[3];
    const __nv_bfloat16* c_kv_in;
    __nv_bfloat16* c_kv_out;
    const __nv_bfloat16* k_rope_in;
    __nv_bfloat16* k_rope_out;
    int pos;
    float routed_scaling;

    const __nv_bfloat16* hidden_in;
    __nv_bfloat16* hidden_out;
    void* workspace;
};

struct alignas(16) Workspace {
    // Dedicated per-layer split atomicAdd buffers & arrival counters (zeroed once at start of step!)
    alignas(16) float kda_qkvg[3][2][16384];         // 384 KB (2-way split!)
    alignas(16) float mla_qkv[4][6720];              // 105 KB (4-way split!)
    alignas(16) float mla_o_raw[4][4096];            // 64 KB  (4-way split!)
    alignas(16) float attn_o_accum[4][4][2304];      // 144 KB (4-way split!)
    alignas(16) float moe_gu_accum[4][2][9 * 2048];  // 576 KB (2-way split!)
    alignas(16) float moe_down_accum[4][4][2304];    // 144 KB (4-way split!)
    alignas(16) uint32_t router_done[4];             // 16 B
    alignas(16) uint32_t moe_down_done[4];           // 16 B
    // End of zeroed section (~1.4 MB -> ~1.4 us across 376 CTAs!)

    alignas(16) __nv_bfloat16 h[2304];
    alignas(16) __nv_bfloat16 h_after_attn[2304];
    alignas(16) __nv_bfloat16 x_norm[2304];
    alignas(16) float x_qgsum[80];
    alignas(16) float beta[32];
    alignas(16) __nv_bfloat16 attn_o_bf16[4096];

    alignas(16) float router_logits[64];
    alignas(16) int topk_idx[8];
    alignas(16) float topk_w[12];

    alignas(16) __nv_bfloat16 q_latent_bf16[32 * 512];
    alignas(16) __nv_bfloat16 q_rope_bf16[32 * 64];
    alignas(16) float o_latent[32 * 512];
    alignas(16) float attn_m[32 * KV_SPLIT_STRIDE];
    alignas(16) float attn_l[32 * KV_SPLIT_STRIDE];
    alignas(16) __nv_bfloat16 attn_o_splits[MAX_KV_SPLITS * 32 * 512];
    alignas(16) uint32_t grid_bar[4];
};

__device__ __forceinline__ float silu_f(float x) {
    return x / (1.0f + __expf(-x));
}

__device__ __forceinline__ float warp_sum(float v) {
    #pragma unroll
    for (int offset = 16; offset > 0; offset >>= 1) {
        v += __shfl_xor_sync(0xffffffff, v, offset);
    }
    return v;
}

__device__ __forceinline__ uint64_t make_evict_first_policy() {
    uint64_t policy;
    asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;\n" : "=l"(policy));
    return policy;
}

__device__ __forceinline__ uint64_t make_evict_last_policy() {
    uint64_t policy;
    asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;\n" : "=l"(policy));
    return policy;
}

__device__ __forceinline__ uint32_t load_u32_policy(const void* ptr, uint64_t policy) {
    uint32_t ret;
    asm volatile("ld.global.L1::no_allocate.L2::cache_hint.u32 %0, [%1], %2;\n"
                 : "=r"(ret)
                 : "l"(ptr), "l"(policy));
    return ret;
}

__device__ __forceinline__ uint2 load_u2_policy(const void* ptr, uint64_t policy) {
    uint2 ret;
    asm volatile("ld.global.L1::no_allocate.L2::cache_hint.v2.u32 {%0, %1}, [%2], %3;\n"
                 : "=r"(ret.x), "=r"(ret.y)
                 : "l"(ptr), "l"(policy));
    return ret;
}

__device__ __forceinline__ uint4 load_u4_policy(const void* ptr, uint64_t policy) {
    uint4 ret;
    asm volatile("ld.global.L1::no_allocate.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;\n"
                 : "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w)
                 : "l"(ptr), "l"(policy));
    return ret;
}

__device__ __forceinline__ void cp_async_16_evict_first(void* smem_ptr, const void* glob_ptr, bool pred, uint64_t policy) {
    uint32_t smem_addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    int src_bytes = pred ? 16 : 0;
    asm volatile(
        "cp.async.cg.shared.global.L2::cache_hint [%0], [%1], 16, %2, %3;\n"
        :: "r"(smem_addr), "l"(glob_ptr), "r"(src_bytes), "l"(policy)
    );
}

__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n" ::: "memory");
}

template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N) : "memory");
}

__device__ __forceinline__ uint32_t load_u32_streaming(const void* ptr) {
    uint32_t ret;
    asm volatile("ld.global.cs.u32 %0, [%1];\n" : "=r"(ret) : "l"(ptr));
    return ret;
}

__device__ __forceinline__ void store_u32_streaming(void* ptr, uint32_t v) {
    asm volatile("st.global.cs.u32 [%0], %1;\n" :: "l"(ptr), "r"(v) : "memory");
}

__device__ __forceinline__ uint2 load_u2_streaming(const void* ptr) {
    uint2 ret;
    asm volatile("ld.global.cs.v2.u32 {%0, %1}, [%2];\n"
                 : "=r"(ret.x), "=r"(ret.y)
                 : "l"(ptr));
    return ret;
}

__device__ __forceinline__ uint4 load_u4_streaming(const void* ptr) {
    uint4 ret;
    asm volatile("ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n"
                 : "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w)
                 : "l"(ptr));
    return ret;
}

__device__ __forceinline__ void store_u4_streaming(void* ptr, uint4 v) {
    asm volatile("st.global.cs.v4.u32 [%0], {%1, %2, %3, %4};\n"
                 :
                 : "l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w)
                 : "memory");
}

__device__ __forceinline__ void mma_m16n8k16_bf16(
    float& d0, float& d1, float& d2, float& d3,
    uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
    uint32_t b0, uint32_t b1
) {
    asm volatile(
        "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
        "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
        : "=f"(d0), "=f"(d1), "=f"(d2), "=f"(d3)
        : "r"(a0), "r"(a1), "r"(a2), "r"(a3),
          "r"(b0), "r"(b1),
          "f"(d0), "f"(d1), "f"(d2), "f"(d3)
    );
}

__device__ __forceinline__ void ldmatrix_x2_trans(uint32_t& b0, uint32_t& b1, const void* smem_ptr) {
    uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile(
        "ldmatrix.sync.aligned.m8n8.x2.trans.shared.b16 {%0, %1}, [%2];\n"
        : "=r"(b0), "=r"(b1)
        : "r"(addr)
    );
}

__device__ __forceinline__ void load_x_norm_to_smem(
    const Workspace* ws,
    float* __restrict__ sx,
    float* __restrict__ sx_qgsum
) {
    #pragma unroll
    for (int pass = 0; pass < 2; ++pass) {
        int vec_idx = pass * 256 + threadIdx.x;
        if (vec_idx < 288) {
            int base = vec_idx * 8;
            uint4 xv = *reinterpret_cast<const uint4*>(&ws->x_norm[base]);
            const __nv_bfloat16* xb = reinterpret_cast<const __nv_bfloat16*>(&xv);
            #pragma unroll
            for (int m = 0; m < 8; ++m) {
                sx[base + m] = __bfloat162float(xb[m]);
            }
        }
    }
    if (threadIdx.x < (HIDDEN / QGROUP)) {
        sx_qgsum[threadIdx.x] = ws->x_qgsum[threadIdx.x];
    }
    __syncthreads();
}

__device__ __forceinline__ void cta_rmsnorm_from_h_2304(
    const __nv_bfloat16* __restrict__ h_in,
    const __nv_bfloat16* __restrict__ norm_w,
    float* __restrict__ sx,
    float* __restrict__ sx_qgsum, // nullptr if qgsum not needed
    float* __restrict__ s_red
) {
    float reg_h[16];
    float sq_sum = 0.0f;
    #pragma unroll
    for (int pass = 0; pass < 2; ++pass) {
        int vec_idx = pass * 256 + threadIdx.x;
        if (vec_idx < 288) {
            int base = vec_idx * 8;
            uint4 hv = *reinterpret_cast<const uint4*>(&h_in[base]);
            const __nv_bfloat16* hb = reinterpret_cast<const __nv_bfloat16*>(&hv);
            #pragma unroll
            for (int m = 0; m < 8; ++m) {
                float vf = __bfloat162float(hb[m]);
                reg_h[pass * 8 + m] = vf;
                sq_sum += vf * vf;
            }
        }
    }
    int wid = threadIdx.x >> 5;
    int lane = threadIdx.x & 31;
    sq_sum = warp_sum(sq_sum);
    if (lane == 0) s_red[wid] = sq_sum;
    __syncthreads();
    if (wid == 0) {
        float v = (lane < 8) ? s_red[lane] : 0.0f;
        v = warp_sum(v);
        if (lane == 0) s_red[0] = rsqrtf(v * (1.0f / float(HIDDEN)) + 1.0e-6f);
    }
    __syncthreads();

    float inv_rms = s_red[0];
    #pragma unroll
    for (int pass = 0; pass < 2; ++pass) {
        int vec_idx = pass * 256 + threadIdx.x;
        if (vec_idx < 288) {
            int base = vec_idx * 8;
            uint4 wv = *reinterpret_cast<const uint4*>(&norm_w[base]);
            const __nv_bfloat16* wb = reinterpret_cast<const __nv_bfloat16*>(&wv);
            #pragma unroll
            for (int m = 0; m < 8; ++m) {
                __nv_bfloat16 b = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m]));
                sx[base + m] = __bfloat162float(b);
            }
        }
    }
    __syncthreads();

    if (sx_qgsum != nullptr) {
        for (int qg = wid; qg < (HIDDEN / QGROUP); qg += 8) {
            float s = warp_sum(sx[qg * 32 + lane]);
            if (lane == 0) sx_qgsum[qg] = s;
        }
        __syncthreads();
    }
}

__device__ __forceinline__ void reduce4_only_2304(
    const __nv_bfloat16* __restrict__ res_in,
    const float (&accum)[4][HIDDEN],
    __nv_bfloat16* __restrict__ res_out
) {
    #pragma unroll
    for (int pass = 0; pass < 2; ++pass) {
        int vec_idx = pass * 256 + threadIdx.x;
        if (vec_idx < 288) {
            int base = vec_idx * 8;
            uint4 hv = *reinterpret_cast<const uint4*>(&res_in[base]);
            const __nv_bfloat16* hb = reinterpret_cast<const __nv_bfloat16*>(&hv);
            float4 d0a = *reinterpret_cast<const float4*>(&accum[0][base]);
            float4 d0b = *reinterpret_cast<const float4*>(&accum[0][base + 4]);
            float4 d1a = *reinterpret_cast<const float4*>(&accum[1][base]);
            float4 d1b = *reinterpret_cast<const float4*>(&accum[1][base + 4]);
            float4 d2a = *reinterpret_cast<const float4*>(&accum[2][base]);
            float4 d2b = *reinterpret_cast<const float4*>(&accum[2][base + 4]);
            float4 d3a = *reinterpret_cast<const float4*>(&accum[3][base]);
            float4 d3b = *reinterpret_cast<const float4*>(&accum[3][base + 4]);
            float ds[8] = {
                d0a.x + d1a.x + d2a.x + d3a.x,
                d0a.y + d1a.y + d2a.y + d3a.y,
                d0a.z + d1a.z + d2a.z + d3a.z,
                d0a.w + d1a.w + d2a.w + d3a.w,
                d0b.x + d1b.x + d2b.x + d3b.x,
                d0b.y + d1b.y + d2b.y + d3b.y,
                d0b.z + d1b.z + d2b.z + d3b.z,
                d0b.w + d1b.w + d2b.w + d3b.w
            };
            uint4 out_h;
            __nv_bfloat16* ob = reinterpret_cast<__nv_bfloat16*>(&out_h);
            #pragma unroll
            for (int m = 0; m < 8; ++m) {
                __nv_bfloat16 mo = __float2bfloat16_rn(ds[m]);
                ob[m] = __float2bfloat16_rn(__bfloat162float(hb[m]) + __bfloat162float(mo));
            }
            *reinterpret_cast<uint4*>(&res_out[base]) = out_h;
        }
    }
}

template <int QGPW, bool STREAM_LOAD, bool CHECK_VALID = false>
__device__ __forceinline__ void warp_int4_gemv_128(
    const float* __restrict__ sx,
    const float* __restrict__ sx_qgsum,
    const uint8_t* __restrict__ w_q,
    const __nv_bfloat16* __restrict__ scales,
    const __nv_bfloat16* __restrict__ zeros,
    float* __restrict__ out_accum,
    int N_stride,
    int col_tile_offset,
    int qg_start,
    int valid_cols = 128
) {
    const int lane = threadIdx.x & 31;
    const int col_grp = lane & 15;
    const int row_sub = lane >> 4;
    const int local_col = col_grp * 8;
    if constexpr (CHECK_VALID) {
        if (local_col >= valid_cols) return;
    }
    const int col_base = col_tile_offset + local_col;

    float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
    const int kb_base = qg_start * 16 + row_sub;
    constexpr int STEPS = QGPW * 8;
    const uint64_t pol = STREAM_LOAD ? make_evict_first_policy() : make_evict_last_policy();

    #pragma unroll 8
    for (int step = 0; step < STEPS; ++step) {
        int kb = kb_base + step * 2;
        const void* w_ptr = &w_q[kb * N_stride + col_base];
        uint2 w_vec = load_u2_policy(w_ptr, pol);
        float x_lo = sx[kb * 2];
        float x_hi = sx[kb * 2 + 1];

        uint32_t words[2] = {w_vec.x, w_vec.y};
        #pragma unroll
        for (int w = 0; w < 2; ++w) {
            uint32_t v = words[w];
            uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
            uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
            uint32_t v_shr8 = v >> 8;
            uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
            uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;

            float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
            float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
            float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
            float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));

            g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x;
            g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x;
            g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y;
            g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y;
        }
    }

    uint32_t mask = 0xffffffffu;
    if constexpr (CHECK_VALID) {
        mask = (valid_cols >= 128) ? 0xffffffffu : 0x00ff00ffu;
    }
    #pragma unroll
    for (int c = 0; c < 8; ++c) {
        g_acc[c] += __shfl_down_sync(mask, g_acc[c], 16);
    }

    if (row_sub == 0) {
        int g = qg_start >> 2;
        float xsum = sx_qgsum[qg_start];
        if constexpr (QGPW >= 2) xsum += sx_qgsum[qg_start + 1];
        if constexpr (QGPW >= 4) xsum += sx_qgsum[qg_start + 2] + sx_qgsum[qg_start + 3];

        const void* s_ptr = &scales[g * N_stride + col_base];
        const void* z_ptr = &zeros[g * N_stride + col_base];
        uint4 s_vec = load_u4_policy(s_ptr, pol);
        uint4 z_vec = load_u4_policy(z_ptr, pol);
        const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_vec);
        const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_vec);
        #pragma unroll
        for (int c = 0; c < 8; ++c) {
            float z_eff = __bfloat162float(zb[c]) + 128.0f;
            float val = __bfloat162float(sb[c]) * (g_acc[c] - z_eff * xsum);
            atomicAdd(&out_accum[local_col + c], val);
        }
    }
}

union alignas(16) SharedStorage {
    struct {
        alignas(16) float sx[4096];        // 16 KB
        alignas(16) float sx_qgsum[128];   // 512 B
        alignas(16) float s_red[32];       // 128 B
        alignas(16) int topk_idx[8];
        alignas(16) float topk_w[12];
    } gemv;
    struct {
        alignas(16) float inter[512];      // 2 KB (Half-K = 512 per CTA!)
        alignas(16) float qgsum[16];       // 64 B
        alignas(16) float raw_acc[8][128]; // 4 KB (8-warp smem reduction!)
    } moe_down;
    struct {
        alignas(16) float sq[128];
        alignas(16) float sk[128];
        alignas(16) float sv[128];
        alignas(16) float sdec[128];
        alignas(16) float s_red[8][16];
    } kda_rec;
    struct {
        alignas(16) __nv_bfloat16 ckv[2][16][520]; // 33,280 B (2-stage cp.async double buffer!)
        alignas(16) __nv_bfloat16 krp[2][16][72];  // 4,608 B
        alignas(16) float s_part[4][32][16];       // 8,192 B
        alignas(16) __nv_bfloat16 s_p[32][24];     // 1,536 B
        alignas(16) float s_escale[32];            // 128 B
        alignas(16) float s_m[32];                 // 128 B
        alignas(16) float s_l[32];                 // 128 B
    } mla_tc; // 48,000 B -> 2 blocks/SM (96,000 B <= 101,376 B)!
    struct {
        alignas(16) float scale_s[KV_SPLIT_STRIDE];
        float g_inv_sum;
        alignas(16) float partial[8][64];         // 2 KB (64 cols per CTA across 256 CTAs!)
    } mla_comb;
};

__device__ void run_moe_block(
    FastGridBarrier& grid,
    const MegakernelParams& params,
    Workspace* ws,
    SharedStorage& smem,
    int layer_idx,
    bool write_final_out
) {
    const int lane = threadIdx.x & 31;
    const int wid = threadIdx.x >> 5;
    const MoEPtrs& moe = params.moe[layer_idx];

    // Stage M1: CTAs 0..64 fuse Residual + MoE RMSNorm + 64-Expert Router GEMV + Last-CTA Top-8!
    // Only 65 CTAs read attn_o_accum (no 376-CTA hotspot, zero serial tail on o_proj!).
    if (blockIdx.x < 65) {
        float reg_h[16];
        float sq_sum = 0.0f;
        #pragma unroll
        for (int pass = 0; pass < 2; ++pass) {
            int vec_idx = pass * 256 + threadIdx.x;
            if (vec_idx < 288) {
                int base = vec_idx * 8;
                uint4 hv = *reinterpret_cast<const uint4*>(&ws->h[base]);
                const __nv_bfloat16* hb = reinterpret_cast<const __nv_bfloat16*>(&hv);
                float4 a0a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][0][base]);
                float4 a0b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][0][base + 4]);
                float4 a1a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][1][base]);
                float4 a1b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][1][base + 4]);
                float4 a2a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][2][base]);
                float4 a2b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][2][base + 4]);
                float4 a3a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][3][base]);
                float4 a3b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][3][base + 4]);
                float as[8] = {
                    a0a.x + a1a.x + a2a.x + a3a.x,
                    a0a.y + a1a.y + a2a.y + a3a.y,
                    a0a.z + a1a.z + a2a.z + a3a.z,
                    a0a.w + a1a.w + a2a.w + a3a.w,
                    a0b.x + a1b.x + a2b.x + a3b.x,
                    a0b.y + a1b.y + a2b.y + a3b.y,
                    a0b.z + a1b.z + a2b.z + a3b.z,
                    a0b.w + a1b.w + a2b.w + a3b.w
                };
                uint4 out_h;
                __nv_bfloat16* ob = reinterpret_cast<__nv_bfloat16*>(&out_h);
                #pragma unroll
                for (int m = 0; m < 8; ++m) {
                    __nv_bfloat16 ao = __float2bfloat16_rn(as[m]);
                    __nv_bfloat16 hm = __float2bfloat16_rn(__bfloat162float(hb[m]) + __bfloat162float(ao));
                    ob[m] = hm;
                    float vf = __bfloat162float(hm);
                    reg_h[pass * 8 + m] = vf;
                    sq_sum += vf * vf;
                }
                if (blockIdx.x == 64) {
                    *reinterpret_cast<uint4*>(&ws->h_after_attn[base]) = out_h;
                }
            }
        }
        sq_sum = warp_sum(sq_sum);
        if (lane == 0) smem.gemv.s_red[wid] = sq_sum;
        __syncthreads();
        if (wid == 0) {
            float v = (lane < 8) ? smem.gemv.s_red[lane] : 0.0f;
            v = warp_sum(v);
            if (lane == 0) smem.gemv.s_red[0] = rsqrtf(v * (1.0f / float(HIDDEN)) + 1.0e-6f);
        }
        __syncthreads();

        float inv_rms = smem.gemv.s_red[0];
        const __nv_bfloat16* mw = params.moe_norm[layer_idx];

        if (blockIdx.x == 64) {
            #pragma unroll
            for (int pass = 0; pass < 2; ++pass) {
                int vec_idx = pass * 256 + threadIdx.x;
                if (vec_idx < 288) {
                    int base = vec_idx * 8;
                    uint4 wv = *reinterpret_cast<const uint4*>(&mw[base]);
                    const __nv_bfloat16* wb = reinterpret_cast<const __nv_bfloat16*>(&wv);
                    uint4 out_xn;
                    __nv_bfloat16* xnb = reinterpret_cast<__nv_bfloat16*>(&out_xn);
                    #pragma unroll
                    for (int m = 0; m < 8; ++m) {
                        __nv_bfloat16 xn = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m]));
                        xnb[m] = xn;
                        smem.gemv.sx[base + m] = __bfloat162float(xn);
                    }
                    *reinterpret_cast<uint4*>(&ws->x_norm[base]) = out_xn;
                }
            }
            __syncthreads();
            for (int qg = wid; qg < (HIDDEN / QGROUP); qg += 8) {
                float s = warp_sum(smem.gemv.sx[qg * 32 + lane]);
                if (lane == 0) ws->x_qgsum[qg] = s;
            }
        } else {
            int e = blockIdx.x;
            const __nv_bfloat16* rw = &moe.router_w[e * HIDDEN];
            float dot = 0.0f;
            #pragma unroll
            for (int pass = 0; pass < 2; ++pass) {
                int vec_idx = pass * 256 + threadIdx.x;
                if (vec_idx < 288) {
                    int base = vec_idx * 8;
                    uint4 wv = *reinterpret_cast<const uint4*>(&mw[base]);
                    uint4 rv = *reinterpret_cast<const uint4*>(&rw[base]);
                    const __nv_bfloat16* wb = reinterpret_cast<const __nv_bfloat16*>(&wv);
                    const __nv_bfloat16* rb = reinterpret_cast<const __nv_bfloat16*>(&rv);
                    #pragma unroll
                    for (int m = 0; m < 8; ++m) {
                        __nv_bfloat16 xn = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m]));
                        dot += __bfloat162float(xn) * __bfloat162float(rb[m]);
                    }
                }
            }
            dot = warp_sum(dot);
            if (lane == 0) smem.gemv.s_red[8 + wid] = dot;
            __syncthreads();
            if (wid == 0) {
                float s = (lane < 8) ? smem.gemv.s_red[8 + lane] : 0.0f;
                s = warp_sum(s);
                uint32_t prev_done = 0u;
                if (lane == 0) {
                    float logit = __bfloat162float(__float2bfloat16_rn(s));
                    asm volatile("st.global.release.gpu.f32 [%0], %1;\n"
                                 :: "l"(&ws->router_logits[e]), "f"(logit) : "memory");
                    asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;\n"
                                 : "=r"(prev_done)
                                 : "l"(&ws->router_done[layer_idx])
                                 : "memory");
                }
                prev_done = __shfl_sync(0xffffffff, prev_done, 0);
                if (prev_done == 63u) {
                    int i0 = 2 * lane;
                    int i1 = 2 * lane + 1;
                    float v0, v1;
                    asm volatile("ld.global.acquire.gpu.v2.f32 {%0, %1}, [%2];\n"
                                 : "=f"(v0), "=f"(v1)
                                 : "l"(&ws->router_logits[i0]));
                    float lmax = fmaxf(v0, v1);
                    #pragma unroll
                    for (int off = 16; off > 0; off >>= 1) {
                        lmax = fmaxf(lmax, __shfl_xor_sync(0xffffffff, lmax, off));
                    }
                    float p0 = __expf(v0 - lmax);
                    float p1 = __expf(v1 - lmax);
                    float psum = warp_sum(p0 + p1);
                    p0 /= psum;
                    p1 /= psum;

                    float w_sum = 0.0f;
                    #pragma unroll
                    for (int k = 0; k < 8; ++k) {
                        float best_v = (p0 >= p1) ? p0 : p1;
                        int best_i = (p0 >= p1) ? i0 : i1;
                        #pragma unroll
                        for (int off = 16; off > 0; off >>= 1) {
                            float ov = __shfl_xor_sync(0xffffffff, best_v, off);
                            int oi = __shfl_xor_sync(0xffffffff, best_i, off);
                            if (ov > best_v || (ov == best_v && oi < best_i)) {
                                best_v = ov;
                                best_i = oi;
                            }
                        }
                        if (lane == 0) {
                            ws->topk_idx[k] = best_i;
                            smem.gemv.topk_w[k] = best_v;
                        }
                        w_sum += best_v;
                        if (i0 == best_i) p0 = -1.0f;
                        if (i1 == best_i) p1 = -1.0f;
                    }
                    if (lane == 0) {
                        float scale = params.routed_scaling / (w_sum + 1.0e-9f);
                        #pragma unroll
                        for (int k = 0; k < 8; ++k) {
                            ws->topk_w[k] = smem.gemv.topk_w[k] * scale;
                        }
                        ws->topk_w[8] = 1.0f;
                    }
                }
            }
        }
    }
    grid.sync();

    // Stage M2: Row-Contiguous 1-Wave Gate+Up GEMV (zero Top-K stall, 4.5 KB x_norm load!)
    if (blockIdx.x < 324) {
        int slot = blockIdx.x / 36;
        int rem = blockIdx.x % 36;
        int is_up = rem / 18;
        int kc = rem % 18;
        int ltn = wid; // 0..7

        if (threadIdx.x == 0 && slot < 8) {
            smem.gemv.topk_idx[0] = ws->topk_idx[slot];
        }
        load_x_norm_to_smem(ws, smem.gemv.sx, smem.gemv.sx_qgsum);

        size_t wq_exp_stride = size_t(HIDDEN / 2) * MOE_INTER;
        size_t sz_exp_stride = size_t(HIDDEN / GROUP) * MOE_INTER;
        float* out_ptr = &ws->moe_gu_accum[layer_idx][kc & 1][slot * 2048 + is_up * 1024 + ltn * 128];

        if (slot < 8) {
            int exp_id = smem.gemv.topk_idx[0];
            const QuantLinearPtrs* ql = is_up ? &moe.up : &moe.gate;
            warp_int4_gemv_128<4, true>(
                smem.gemv.sx, smem.gemv.sx_qgsum,
                ql->w_q + exp_id * wq_exp_stride,
                ql->scales + exp_id * sz_exp_stride,
                ql->zeros + exp_id * sz_exp_stride,
                out_ptr, MOE_INTER, ltn * 128, kc * 4
            );
        } else {
            const QuantLinearPtrs* ql = is_up ? &moe.s_up : &moe.s_gate;
            warp_int4_gemv_128<4, false>(
                smem.gemv.sx, smem.gemv.sx_qgsum,
                ql->w_q, ql->scales, ql->zeros,
                out_ptr, MOE_INTER, ltn * 128, kc * 4
            );
        }
    }
    grid.sync();

    // Stage M3: Half-K SiLU*Up + 8-Warp In-CTA Shared-Memory Reduction Down GEMV + Last-CTA Residual!
    if (blockIdx.x < 324) {
        int slot = blockIdx.x / 36;
        int rem = blockIdx.x % 36;
        int k_half = rem / 18; // 0..1 (512 K-channels per CTA!)
        int ltn = rem % 18;    // 0..17 (128 output cols per CTA!)
        float w_s = ws->topk_w[slot];
        int exp_id = (slot < 8) ? ws->topk_idx[slot] : 0;

        {
            int m = threadIdx.x * 2;
            int g_idx = slot * 2048 + k_half * 512 + m;
            float2 g0 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][0][g_idx]);
            float2 g1 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][1][g_idx]);
            float2 u0 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][0][g_idx + 1024]);
            float2 u1 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][1][g_idx + 1024]);
            float2 prod = make_float2(
                w_s * silu_f(g0.x + g1.x) * (u0.x + u1.x),
                w_s * silu_f(g0.y + g1.y) * (u0.y + u1.y)
            );
            *reinterpret_cast<float2*>(&smem.moe_down.inter[m]) = prod;
        }
        __syncthreads();

        for (int qg = wid; qg < 16; qg += 8) {
            float s = warp_sum(smem.moe_down.inter[qg * 32 + lane]);
            if (lane == 0) smem.moe_down.qgsum[qg] = s;
        }
        __syncthreads();

        size_t wq_exp_stride = size_t(MOE_INTER / 2) * HIDDEN;
        size_t sz_exp_stride = size_t(MOE_INTER / GROUP) * HIDDEN;
        const uint8_t* w_q_ptr = ((slot < 8) ? (moe.down.w_q + exp_id * wq_exp_stride) : moe.s_down.w_q)
                               + size_t(k_half * 256) * HIDDEN;
        const __nv_bfloat16* s_ptr_base = ((slot < 8) ? (moe.down.scales + exp_id * sz_exp_stride) : moe.s_down.scales)
                                        + size_t(k_half * 4) * HIDDEN;
        const __nv_bfloat16* z_ptr_base = ((slot < 8) ? (moe.down.zeros + exp_id * sz_exp_stride) : moe.s_down.zeros)
                                        + size_t(k_half * 4) * HIDDEN;
        const uint64_t pol = (slot < 8) ? make_evict_first_policy() : make_evict_last_policy();

        const int col_grp = lane & 15;
        const int row_sub = lane >> 4;
        const int local_col = col_grp * 8;
        const int col_base = ltn * 128 + local_col;
        const int kb_base = wid * 32 + row_sub;

        float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
        #pragma unroll 8
        for (int step = 0; step < 16; ++step) {
            int kb = kb_base + step * 2;
            uint2 w_vec = load_u2_policy(&w_q_ptr[kb * HIDDEN + col_base], pol);
            float x_lo = smem.moe_down.inter[kb * 2];
            float x_hi = smem.moe_down.inter[kb * 2 + 1];

            uint32_t words[2] = {w_vec.x, w_vec.y};
            #pragma unroll
            for (int w = 0; w < 2; ++w) {
                uint32_t v = words[w];
                uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
                uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
                uint32_t v_shr8 = v >> 8;
                uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
                uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;

                float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
                float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
                float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
                float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));

                g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x;
                g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x;
                g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y;
                g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y;
            }
        }
        #pragma unroll
        for (int c = 0; c < 8; ++c) {
            g_acc[c] += __shfl_down_sync(0xffffffffu, g_acc[c], 16);
        }
        if (row_sub == 0) {
            *reinterpret_cast<float4*>(&smem.moe_down.raw_acc[wid][local_col])     = make_float4(g_acc[0], g_acc[1], g_acc[2], g_acc[3]);
            *reinterpret_cast<float4*>(&smem.moe_down.raw_acc[wid][local_col + 4]) = make_float4(g_acc[4], g_acc[5], g_acc[6], g_acc[7]);
        }
        __syncthreads();

        if (wid < 4) {
            int g_loc = wid;
            int col4 = lane * 4;
            int g_col = ltn * 128 + col4;
            float xsum = smem.moe_down.qgsum[g_loc * 4 + 0] + smem.moe_down.qgsum[g_loc * 4 + 1]
                       + smem.moe_down.qgsum[g_loc * 4 + 2] + smem.moe_down.qgsum[g_loc * 4 + 3];
            uint2 s_vec = load_u2_policy(&s_ptr_base[g_loc * HIDDEN + g_col], pol);
            uint2 z_vec = load_u2_policy(&z_ptr_base[g_loc * HIDDEN + g_col], pol);
            const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_vec);
            const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_vec);
            float4 r0 = *reinterpret_cast<const float4*>(&smem.moe_down.raw_acc[2 * g_loc][col4]);
            float4 r1 = *reinterpret_cast<const float4*>(&smem.moe_down.raw_acc[2 * g_loc + 1][col4]);
            float raw[4] = {r0.x + r1.x, r0.y + r1.y, r0.z + r1.z, r0.w + r1.w};
            float deq[4];
            #pragma unroll
            for (int c = 0; c < 4; ++c) {
                deq[c] = __bfloat162float(sb[c]) * (raw[c] - (__bfloat162float(zb[c]) + 128.0f) * xsum);
            }
            *reinterpret_cast<float4*>(&smem.moe_down.raw_acc[2 * g_loc][col4]) = make_float4(deq[0], deq[1], deq[2], deq[3]);
        }
        __syncthreads();

        if (threadIdx.x < 128) {
            int col = threadIdx.x;
            float sum = smem.moe_down.raw_acc[0][col] + smem.moe_down.raw_acc[2][col]
                      + smem.moe_down.raw_acc[4][col] + smem.moe_down.raw_acc[6][col];
            atomicAdd(&ws->moe_down_accum[layer_idx][(slot * 2 + k_half) & 3][ltn * 128 + col], sum);
        }

        __syncthreads();
        if (threadIdx.x == 0) {
            uint32_t prev;
            asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;\n"
                         : "=r"(prev)
                         : "l"(&ws->moe_down_done[layer_idx])
                         : "memory");
            smem.moe_down.qgsum[0] = __uint_as_float(prev);
        }
        __syncthreads();
        uint32_t prev_m3 = __float_as_uint(smem.moe_down.qgsum[0]);
        if (prev_m3 == 323u) {
            reduce4_only_2304(
                ws->h_after_attn, ws->moe_down_accum[layer_idx],
                write_final_out ? params.hidden_out : ws->h
            );
        }
    }
    grid.sync();
}

__device__ void run_kda_layer(
    FastGridBarrier& grid,
    const MegakernelParams& params,
    Workspace* ws,
    SharedStorage& smem,
    int layer_idx
) {
    const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x;
    const int global_warp = global_tid >> 5;
    const int lane = threadIdx.x & 31;
    const int wid = threadIdx.x >> 5;
    const KDAPtrs& kda = params.kda[layer_idx];
    const KDAStatePtrs& st = params.kda_st[layer_idx];

    // Stage K1: CTAs 0..287 run RMSNorm + Row-Contiguous (q,k,v,g) GEMV; idle CTAs 288..291 compute beta[0..31]!
    if (blockIdx.x < 288) {
        cta_rmsnorm_from_h_2304(
            ws->h, params.attn_norm[layer_idx],
            smem.gemv.sx, smem.gemv.sx_qgsum, smem.gemv.s_red
        );

        int ltn = global_warp & 31;
        int rem = global_warp >> 5; // 0..71
        int which = rem / 18;       // 0..3
        int kc = rem % 18;          // 0..17
        int tn = which * 32 + ltn;

        const QuantLinearPtrs* ql = (which == 0) ? &kda.q_proj :
                                    (which == 1) ? &kda.k_proj :
                                    (which == 2) ? &kda.v_proj : &kda.g_proj;

        warp_int4_gemv_128<4, false>(
            smem.gemv.sx, smem.gemv.sx_qgsum,
            ql->w_q, ql->scales, ql->zeros,
            &ws->kda_qkvg[layer_idx][kc & 1][tn * 128],
            KDA_C, ltn * 128, kc * 4
        );
    } else if (blockIdx.x < 292) {
        cta_rmsnorm_from_h_2304(
            ws->h, params.attn_norm[layer_idx],
            smem.gemv.sx, nullptr, smem.gemv.s_red
        );
        int h = (blockIdx.x - 288) * 8 + wid;
        const __nv_bfloat16* bw = &kda.beta_w[h * HIDDEN];
        float dot = 0.0f;
        for (int k = lane * 8; k < HIDDEN; k += 256) {
            uint4 bv = *reinterpret_cast<const uint4*>(&bw[k]);
            const __nv_bfloat16* bb = reinterpret_cast<const __nv_bfloat16*>(&bv);
            #pragma unroll
            for (int m = 0; m < 8; ++m) {
                dot += smem.gemv.sx[k + m] * __bfloat162float(bb[m]);
            }
        }
        dot = warp_sum(dot);
        if (lane == 0) {
            float bx = __bfloat162float(__float2bfloat16_rn(dot));
            ws->beta[h] = 1.0f / (1.0f + __expf(-bx));
        }
    }
    grid.sync();

    // Stage K2: Short Conv + Decay + Recurrent S[32, 128, 128] update!
    {
        const int col_grp = lane & 7;
        const int row_sub = lane >> 3;
        const int i_tid = wid * 4 + row_sub;
        const float q_scale = rsqrtf(float(KDA_DK));

        if (blockIdx.x < 256) {
            int h = blockIdx.x >> 3;
            int j_tile = blockIdx.x & 7;
            int j_base = j_tile * 16;

            if (threadIdx.x < 128) {
                int d = threadIdx.x;
                int c = h * 128 + d;

                float q_sum = ws->kda_qkvg[layer_idx][0][c] + ws->kda_qkvg[layer_idx][1][c];
                float q_raw = __bfloat162float(__float2bfloat16_rn(q_sum));
                const __nv_bfloat16* cw_q = &kda.conv_w[(0 * KDA_C + c) * 4];
                float q_c = __bfloat162float(st.cq[0 * KDA_C + c]) * __bfloat162float(cw_q[0])
                          + __bfloat162float(st.cq[1 * KDA_C + c]) * __bfloat162float(cw_q[1])
                          + __bfloat162float(st.cq[2 * KDA_C + c]) * __bfloat162float(cw_q[2])
                          + q_raw * __bfloat162float(cw_q[3]);
                smem.kda_rec.sq[d] = __bfloat162float(__float2bfloat16_rn(silu_f(q_c))) * q_scale;

                float k_sum = ws->kda_qkvg[layer_idx][0][KDA_C + c] + ws->kda_qkvg[layer_idx][1][KDA_C + c];
                float k_raw = __bfloat162float(__float2bfloat16_rn(k_sum));
                const __nv_bfloat16* cw_k = &kda.conv_w[(1 * KDA_C + c) * 4];
                float k_c = __bfloat162float(st.ck[0 * KDA_C + c]) * __bfloat162float(cw_k[0])
                          + __bfloat162float(st.ck[1 * KDA_C + c]) * __bfloat162float(cw_k[1])
                          + __bfloat162float(st.ck[2 * KDA_C + c]) * __bfloat162float(cw_k[2])
                          + k_raw * __bfloat162float(cw_k[3]);
                smem.kda_rec.sk[d] = __bfloat162float(__float2bfloat16_rn(silu_f(k_c)));

                float v_sum = ws->kda_qkvg[layer_idx][0][2 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][2 * KDA_C + c];
                float v_raw = __bfloat162float(__float2bfloat16_rn(v_sum));
                const __nv_bfloat16* cw_v = &kda.conv_w[(2 * KDA_C + c) * 4];
                float v_c = __bfloat162float(st.cv[0 * KDA_C + c]) * __bfloat162float(cw_v[0])
                          + __bfloat162float(st.cv[1 * KDA_C + c]) * __bfloat162float(cw_v[1])
                          + __bfloat162float(st.cv[2 * KDA_C + c]) * __bfloat162float(cw_v[2])
                          + v_raw * __bfloat162float(cw_v[3]);
                smem.kda_rec.sv[d] = __bfloat162float(__float2bfloat16_rn(silu_f(v_c)));

                float g_sum = ws->kda_qkvg[layer_idx][0][3 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][3 * KDA_C + c];
                float gx = __bfloat162float(__float2bfloat16_rn(g_sum));
                float sp = (gx > 20.0f) ? gx : log1pf(__expf(gx));
                smem.kda_rec.sdec[d] = __expf(-sp);
            }
            __syncthreads();

            float* S_head = &st.S[h * 128 * 128];
            int j0 = j_base + col_grp * 2;

            float2 s_reg[4];
            float2 pred = {0.f, 0.f};
            #pragma unroll
            for (int it = 0; it < 4; ++it) {
                int i = it * 32 + i_tid;
                float2 sv = *reinterpret_cast<const float2*>(&S_head[i * 128 + j0]);
                float dec = smem.kda_rec.sdec[i];
                sv.x *= dec;
                sv.y *= dec;
                s_reg[it] = sv;
                float kv = smem.kda_rec.sk[i];
                pred.x += sv.x * kv;
                pred.y += sv.y * kv;
            }

            pred.x += __shfl_down_sync(0xffffffff, pred.x, 8);
            pred.y += __shfl_down_sync(0xffffffff, pred.y, 8);
            pred.x += __shfl_down_sync(0xffffffff, pred.x, 16);
            pred.y += __shfl_down_sync(0xffffffff, pred.y, 16);
            if (row_sub == 0) {
                smem.kda_rec.s_red[wid][col_grp * 2 + 0] = pred.x;
                smem.kda_rec.s_red[wid][col_grp * 2 + 1] = pred.y;
            }
            __syncthreads();

            float beta_h = ws->beta[h];
            float p0 = 0.f, p1 = 0.f;
            #pragma unroll
            for (int w = 0; w < 8; ++w) {
                p0 += smem.kda_rec.s_red[w][col_grp * 2 + 0];
                p1 += smem.kda_rec.s_red[w][col_grp * 2 + 1];
            }
            float d0 = beta_h * (smem.kda_rec.sv[j0 + 0] - p0);
            float d1 = beta_h * (smem.kda_rec.sv[j0 + 1] - p1);
            __syncthreads();

            float2 o_part = {0.f, 0.f};
            #pragma unroll
            for (int it = 0; it < 4; ++it) {
                int i = it * 32 + i_tid;
                float kv = smem.kda_rec.sk[i];
                float qv = smem.kda_rec.sq[i];
                float2 sv = s_reg[it];
                sv.x += kv * d0;
                sv.y += kv * d1;
                *reinterpret_cast<float2*>(&S_head[i * 128 + j0]) = sv;
                o_part.x += sv.x * qv;
                o_part.y += sv.y * qv;
            }

            o_part.x += __shfl_down_sync(0xffffffff, o_part.x, 8);
            o_part.y += __shfl_down_sync(0xffffffff, o_part.y, 8);
            o_part.x += __shfl_down_sync(0xffffffff, o_part.x, 16);
            o_part.y += __shfl_down_sync(0xffffffff, o_part.y, 16);
            if (row_sub == 0) {
                smem.kda_rec.s_red[wid][col_grp * 2 + 0] = o_part.x;
                smem.kda_rec.s_red[wid][col_grp * 2 + 1] = o_part.y;
            }
            __syncthreads();

            if (threadIdx.x < 16) {
                int jl = threadIdx.x;
                float osum = 0.f;
                #pragma unroll
                for (int w = 0; w < 8; ++w) osum += smem.kda_rec.s_red[w][jl];
                ws->attn_o_bf16[h * 128 + j_base + jl] = __float2bfloat16_rn(osum);
            }
        }
    }
    grid.sync();

    // Stage K3: CTAs 0..143 run QGPW=2 o_proj; idle CTAs 144..159 shift cq, ck, cv off the critical path!
    if (blockIdx.x < 144) {
        #pragma unroll
        for (int pass = 0; pass < 2; ++pass) {
            int base = (pass * 256 + threadIdx.x) * 8;
            uint4 ov = *reinterpret_cast<const uint4*>(&ws->attn_o_bf16[base]);
            const __nv_bfloat16* ob = reinterpret_cast<const __nv_bfloat16*>(&ov);
            #pragma unroll
            for (int m = 0; m < 8; ++m) smem.gemv.sx[base + m] = __bfloat162float(ob[m]);
        }
        __syncthreads();
        for (int qg = wid; qg < (KDA_C / QGROUP); qg += 8) {
            float s = warp_sum(smem.gemv.sx[qg * 32 + lane]);
            if (lane == 0) smem.gemv.sx_qgsum[qg] = s;
        }
        __syncthreads();

        int tn = global_warp % 18;
        int kc = global_warp / 18; // 0..63
        warp_int4_gemv_128<2, false>(
            smem.gemv.sx, smem.gemv.sx_qgsum,
            kda.o_proj.w_q, kda.o_proj.scales, kda.o_proj.zeros,
            &ws->attn_o_accum[layer_idx][kc & 3][tn * 128],
            HIDDEN, tn * 128, kc * 2
        );
    } else if (blockIdx.x < 160) {
        int c = (blockIdx.x - 144) * 256 + threadIdx.x;
        __nv_bfloat16 q1 = st.cq[1 * KDA_C + c], q2 = st.cq[2 * KDA_C + c];
        __nv_bfloat16 qn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][c] + ws->kda_qkvg[layer_idx][1][c]);
        st.cq[0 * KDA_C + c] = q1; st.cq[1 * KDA_C + c] = q2; st.cq[2 * KDA_C + c] = qn;

        __nv_bfloat16 k1 = st.ck[1 * KDA_C + c], k2 = st.ck[2 * KDA_C + c];
        __nv_bfloat16 kn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][KDA_C + c] + ws->kda_qkvg[layer_idx][1][KDA_C + c]);
        st.ck[0 * KDA_C + c] = k1; st.ck[1 * KDA_C + c] = k2; st.ck[2 * KDA_C + c] = kn;

        __nv_bfloat16 v1 = st.cv[1 * KDA_C + c], v2 = st.cv[2 * KDA_C + c];
        __nv_bfloat16 vn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][2 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][2 * KDA_C + c]);
        st.cv[0 * KDA_C + c] = v1; st.cv[1 * KDA_C + c] = v2; st.cv[2 * KDA_C + c] = vn;
    }
    grid.sync();
}

__device__ void run_mla_layer(
    FastGridBarrier& grid,
    const MegakernelParams& params,
    Workspace* ws,
    SharedStorage& smem,
    int layer_idx
) {
    const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x;
    const int global_warp = global_tid >> 5;
    const int lane = threadIdx.x & 31;
    const int wid = threadIdx.x >> 5;
    const MLAPtrs& mla = params.mla;
    const int pos = params.pos;
    const int L = pos + 1;
    const float mla_scale = rsqrtf(float(QK_NOPE + QK_ROPE));

    // Stage A1: Active-CTA RMSNorm + Row-Contiguous q_proj (48 tiles) + kv_a (5 tiles)
    if (blockIdx.x < 239) {
        cta_rmsnorm_from_h_2304(
            ws->h, params.attn_norm[layer_idx],
            smem.gemv.sx, smem.gemv.sx_qgsum, smem.gemv.s_red
        );
        if (global_warp < 53 * 36) {
            int tn = global_warp % 53;
            int kc = global_warp / 53; // 0..35
            if (tn < 48) {
                warp_int4_gemv_128<2, false>(
                    smem.gemv.sx, smem.gemv.sx_qgsum,
                    mla.q_proj.w_q, mla.q_proj.scales, mla.q_proj.zeros,
                    &ws->mla_qkv[kc & 3][tn * 128],
                    6144, tn * 128, kc * 2
                );
            } else {
                int ltn = tn - 48;
                int valid = (ltn < 4) ? 128 : 64;
                warp_int4_gemv_128<2, false, true>(
                    smem.gemv.sx, smem.gemv.sx_qgsum,
                    mla.kv_a.w_q, mla.kv_a.scales, mla.kv_a.zeros,
                    &ws->mla_qkv[kc & 3][6144 + ltn * 128],
                    576, ltn * 128, kc * 2, valid
                );
            }
        }
    }
    grid.sync();

    // Stage A2: CTAs 0..255 compute Absorbed q_latent_bf16; idle CTAs 256..262 write c_kv_out & RoPE!
    if (blockIdx.x < 256) {
        int h = global_warp >> 6;
        int quad = global_warp & 63;
        int g = quad >> 4;
        int d_base = lane * 4;
        int col = h * 256 + d_base;

        float4 q0 = *reinterpret_cast<const float4*>(&ws->mla_qkv[0][h * 192 + d_base]);
        float4 q1 = *reinterpret_cast<const float4*>(&ws->mla_qkv[1][h * 192 + d_base]);
        float4 q2 = *reinterpret_cast<const float4*>(&ws->mla_qkv[2][h * 192 + d_base]);
        float4 q3 = *reinterpret_cast<const float4*>(&ws->mla_qkv[3][h * 192 + d_base]);
        float qn[4] = {
            __bfloat162float(__float2bfloat16_rn(q0.x + q1.x + q2.x + q3.x)),
            __bfloat162float(__float2bfloat16_rn(q0.y + q1.y + q2.y + q3.y)),
            __bfloat162float(__float2bfloat16_rn(q0.z + q1.z + q2.z + q3.z)),
            __bfloat162float(__float2bfloat16_rn(q0.w + q1.w + q2.w + q3.w))
        };
        uint2 s_raw = *reinterpret_cast<const uint2*>(&mla.kv_b.scales[g * 8192 + col]);
        uint2 z_raw = *reinterpret_cast<const uint2*>(&mla.kv_b.zeros[g * 8192 + col]);
        const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_raw);
        const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_raw);

        float qs[4];
        float qz_local = 0.0f;
        #pragma unroll
        for (int m = 0; m < 4; ++m) {
            qs[m] = qn[m] * __bfloat162float(sb[m]);
            qz_local += qs[m] * (__bfloat162float(zb[m]) + 128.0f);
        }
        float qz_sum = warp_sum(qz_local);

        int kb_start = quad * 4;
        uint4 packed;
        #pragma unroll
        for (int s = 0; s < 4; ++s) {
            int kb = kb_start + s;
            uint32_t v = *reinterpret_cast<const uint32_t*>(&mla.kv_b.w_q[kb * 8192 + col]);
            uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
            uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
            uint32_t v_shr8 = v >> 8;
            uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
            uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;

            float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
            float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
            float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
            float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));

            float d_lo = warp_sum(qs[0] * f02_lo.x + qs[1] * f13_lo.x + qs[2] * f02_lo.y + qs[3] * f13_lo.y);
            float d_hi = warp_sum(qs[0] * f02_hi.x + qs[1] * f13_hi.x + qs[2] * f02_hi.y + qs[3] * f13_hi.y);
            __nv_bfloat162 b2 = __floats2bfloat162_rn((d_lo - qz_sum) * mla_scale, (d_hi - qz_sum) * mla_scale);
            uint32_t u2;
            memcpy(&u2, &b2, sizeof(uint32_t));
            if (s == 0) packed.x = u2;
            else if (s == 1) packed.y = u2;
            else if (s == 2) packed.z = u2;
            else packed.w = u2;
        }
        if (lane == 0) {
            *reinterpret_cast<uint4*>(&ws->q_latent_bf16[h * KV_LORA + 2 * kb_start]) = packed;
        }
    } else if (blockIdx.x < 258) {
        int c = (blockIdx.x - 256) * 256 + threadIdx.x;
        float v = ws->mla_qkv[0][6144 + c] + ws->mla_qkv[1][6144 + c]
                + ws->mla_qkv[2][6144 + c] + ws->mla_qkv[3][6144 + c];
        params.c_kv_out[size_t(pos) * KV_LORA + c] = __float2bfloat16_rn(v);
    } else if (blockIdx.x < 263) {
        int rtid = (blockIdx.x - 258) * 256 + threadIdx.x;
        if (rtid < 33 * 32) {
            int who = rtid >> 5;
            int pair = rtid & 31;
            float inv = powf(10000.0f, -float(2 * pair) * (1.0f / 64.0f));
            float ang = float(pos) * inv;
            float cs, sn;
            __sincosf(ang, &sn, &cs);

            int idx_e = (who == 32) ? (6144 + 512 + 2 * pair) : (who * 192 + 128 + 2 * pair);
            float ev_s = ws->mla_qkv[0][idx_e] + ws->mla_qkv[1][idx_e] + ws->mla_qkv[2][idx_e] + ws->mla_qkv[3][idx_e];
            float od_s = ws->mla_qkv[0][idx_e + 1] + ws->mla_qkv[1][idx_e + 1] + ws->mla_qkv[2][idx_e + 1] + ws->mla_qkv[3][idx_e + 1];
            float ev = __bfloat162float(__float2bfloat16_rn(ev_s));
            float od = __bfloat162float(__float2bfloat16_rn(od_s));

            __nv_bfloat16 r0 = __float2bfloat16_rn(ev * cs - od * sn);
            __nv_bfloat16 r1 = __float2bfloat16_rn(od * cs + ev * sn);
            if (who == 32) {
                params.k_rope_out[size_t(pos) * QK_ROPE + 2 * pair + 0] = r0;
                params.k_rope_out[size_t(pos) * QK_ROPE + 2 * pair + 1] = r1;
            } else {
                ws->q_rope_bf16[who * QK_ROPE + 2 * pair + 0] = __float2bfloat16_rn(__bfloat162float(r0) * mla_scale);
                ws->q_rope_bf16[who * QK_ROPE + 2 * pair + 1] = __float2bfloat16_rn(__bfloat162float(r1) * mla_scale);
            }
        }
    }
    grid.sync();

    // Stage A3: 2-Stage cp.async (L2::evict_first) + Hoisted-Q Register SM120 Tensor Core Flash-Decoding!
    const int total_tiles16 = max(1, (L + 15) >> 4);
    const int num_splits = min(MAX_KV_SPLITS, total_tiles16);
    if (blockIdx.x < num_splits) {
        const int split_id = blockIdx.x;
        const int gid = lane >> 2;
        const int tid = lane & 3;
        const int h_half = wid >> 2;
        const int sub_w = wid & 3;
        const int h_base = h_half * 16;

        int t_start = (total_tiles16 * split_id) / num_splits;
        int t_end = (total_tiles16 * (split_id + 1)) / num_splits;
        int n_iters = t_end - t_start;
        int l_start = t_start * 16;
        int l_end = min(L, t_end * 16);

        if (threadIdx.x < 32) {
            smem.mla_tc.s_m[threadIdx.x] = -1.0e30f;
            smem.mla_tc.s_l[threadIdx.x] = 0.0f;
        }
        float o_acc[16][4];
        #pragma unroll
        for (int n = 0; n < 16; ++n) {
            o_acc[n][0] = 0.f; o_acc[n][1] = 0.f; o_acc[n][2] = 0.f; o_acc[n][3] = 0.f;
        }

        uint32_t q_reg[8][4];
        #pragma unroll
        for (int kc = 0; kc < 8; ++kc) {
            int k_col = (sub_w * 8 + kc) * 16;
            q_reg[kc][0] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid) * KV_LORA + k_col + 2 * tid]);
            q_reg[kc][1] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid + 8) * KV_LORA + k_col + 2 * tid]);
            q_reg[kc][2] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid) * KV_LORA + k_col + 8 + 2 * tid]);
            q_reg[kc][3] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid + 8) * KV_LORA + k_col + 8 + 2 * tid]);
        }
        uint32_t qr_reg[4];
        {
            int r_col = sub_w * 16;
            qr_reg[0] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid) * QK_ROPE + r_col + 2 * tid]);
            qr_reg[1] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid + 8) * QK_ROPE + r_col + 2 * tid]);
            qr_reg[2] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid) * QK_ROPE + r_col + 8 + 2 * tid]);
            qr_reg[3] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid + 8) * QK_ROPE + r_col + 8 + 2 * tid]);
        }

        const uint64_t evict_policy = make_evict_first_policy();
        auto issue_tile = [&](int stage, int l_base) {
            int valid_t = min(16, l_end - l_base);
            #pragma unroll
            for (int p = 0; p < 4; ++p) {
                int idx = p * 256 + threadIdx.x;
                int t = idx >> 6;
                int c8 = (idx & 63) * 8;
                bool ok = (t < valid_t);
                const void* gptr = &params.c_kv_out[size_t(ok ? (l_base + t) : 0) * KV_LORA + c8];
                cp_async_16_evict_first(&smem.mla_tc.ckv[stage][t][c8], gptr, ok, evict_policy);
            }
            if (threadIdx.x < 128) {
                int t = threadIdx.x >> 3;
                int r8 = (threadIdx.x & 7) * 8;
                bool ok = (t < valid_t);
                const void* gptr = &params.k_rope_out[size_t(ok ? (l_base + t) : 0) * QK_ROPE + r8];
                cp_async_16_evict_first(&smem.mla_tc.krp[stage][t][r8], gptr, ok, evict_policy);
            }
            cp_async_commit();
        };

        issue_tile(0, l_start);

        for (int iter = 0; iter < n_iters; ++iter) {
            int stage = iter & 1;
            int l_base = l_start + iter * 16;
            int valid_t = min(16, l_end - l_base);

            if (iter + 1 < n_iters) {
                issue_tile(stage ^ 1, l_base + 16);
                cp_async_wait<1>();
            } else {
                cp_async_wait<0>();
            }
            __syncthreads();

            float s0[4] = {0.f, 0.f, 0.f, 0.f};
            float s1[4] = {0.f, 0.f, 0.f, 0.f};

            #pragma unroll
            for (int kc = 0; kc < 8; ++kc) {
                int k_col = (sub_w * 8 + kc) * 16;
                uint32_t b0_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][gid][k_col + 2 * tid]);
                uint32_t b1_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][gid][k_col + 8 + 2 * tid]);
                mma_m16n8k16_bf16(s0[0], s0[1], s0[2], s0[3], q_reg[kc][0], q_reg[kc][1], q_reg[kc][2], q_reg[kc][3], b0_0, b1_0);

                uint32_t b0_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][8 + gid][k_col + 2 * tid]);
                uint32_t b1_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][8 + gid][k_col + 8 + 2 * tid]);
                mma_m16n8k16_bf16(s1[0], s1[1], s1[2], s1[3], q_reg[kc][0], q_reg[kc][1], q_reg[kc][2], q_reg[kc][3], b0_1, b1_1);
            }
            {
                int r_col = sub_w * 16;
                uint32_t b0_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][gid][r_col + 2 * tid]);
                uint32_t b1_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][gid][r_col + 8 + 2 * tid]);
                mma_m16n8k16_bf16(s0[0], s0[1], s0[2], s0[3], qr_reg[0], qr_reg[1], qr_reg[2], qr_reg[3], b0_0, b1_0);

                uint32_t b0_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][8 + gid][r_col + 2 * tid]);
                uint32_t b1_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][8 + gid][r_col + 8 + 2 * tid]);
                mma_m16n8k16_bf16(s1[0], s1[1], s1[2], s1[3], qr_reg[0], qr_reg[1], qr_reg[2], qr_reg[3], b0_1, b1_1);
            }

            *reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid][2 * tid])         = make_float2(s0[0], s0[1]);
            *reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid + 8][2 * tid])     = make_float2(s0[2], s0[3]);
            *reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid][8 + 2 * tid])     = make_float2(s1[0], s1[1]);
            *reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid + 8][8 + 2 * tid]) = make_float2(s1[2], s1[3]);
            __syncthreads();

            #pragma unroll
            for (int hp = 0; hp < 2; ++hp) {
                int h = wid * 4 + hp * 2 + (lane >> 4);
                int t = lane & 15;
                float sc = (t < valid_t)
                    ? (smem.mla_tc.s_part[0][h][t] + smem.mla_tc.s_part[1][h][t]
                     + smem.mla_tc.s_part[2][h][t] + smem.mla_tc.s_part[3][h][t])
                    : -1.0e30f;

                float tmax = sc;
                #pragma unroll
                for (int off = 8; off > 0; off >>= 1) {
                    tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffff, tmax, off));
                }
                float m_old = smem.mla_tc.s_m[h];
                float m_new = fmaxf(m_old, tmax);
                float e_prev = (m_old == -1.0e30f) ? 0.0f : __expf(m_old - m_new);
                float p = (t < valid_t) ? __expf(sc - m_new) : 0.0f;

                float psum = p;
                #pragma unroll
                for (int off = 8; off > 0; off >>= 1) {
                    psum += __shfl_xor_sync(0xffffffff, psum, off);
                }
                smem.mla_tc.s_p[h][t] = __float2bfloat16_rn(p);
                if (t == 0) {
                    smem.mla_tc.s_escale[h] = e_prev;
                    smem.mla_tc.s_m[h] = m_new;
                    smem.mla_tc.s_l[h] = smem.mla_tc.s_l[h] * e_prev + psum;
                }
            }
            __syncthreads();

            float e_top = smem.mla_tc.s_escale[h_base + gid];
            float e_bot = smem.mla_tc.s_escale[h_base + gid + 8];

            uint32_t pa0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid][2 * tid]);
            uint32_t pa1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid + 8][2 * tid]);
            uint32_t pa2 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid][8 + 2 * tid]);
            uint32_t pa3 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid + 8][8 + 2 * tid]);

            int c_sub = sub_w * 128;
            const __nv_bfloat16* ckv_row = &smem.mla_tc.ckv[stage][lane & 15][c_sub];
            #pragma unroll
            for (int n = 0; n < 8; ++n) {
                o_acc[n][0] *= e_top;
                o_acc[n][1] *= e_top;
                o_acc[n][2] *= e_bot;
                o_acc[n][3] *= e_bot;

                uint32_t vb0, vb1;
                ldmatrix_x2_trans(vb0, vb1, ckv_row + n * 8);
                mma_m16n8k16_bf16(o_acc[n][0], o_acc[n][1], o_acc[n][2], o_acc[n][3], pa0, pa1, pa2, pa3, vb0, vb1);
            }
            asm volatile("" ::: "memory");
            #pragma unroll
            for (int n = 8; n < 16; ++n) {
                o_acc[n][0] *= e_top;
                o_acc[n][1] *= e_top;
                o_acc[n][2] *= e_bot;
                o_acc[n][3] *= e_bot;

                uint32_t vb0, vb1;
                ldmatrix_x2_trans(vb0, vb1, ckv_row + n * 8);
                mma_m16n8k16_bf16(o_acc[n][0], o_acc[n][1], o_acc[n][2], o_acc[n][3], pa0, pa1, pa2, pa3, vb0, vb1);
            }
        }

        if (threadIdx.x < 32) {
            int h = threadIdx.x;
            ws->attn_m[h * KV_SPLIT_STRIDE + split_id] = smem.mla_tc.s_m[h];
            ws->attn_l[h * KV_SPLIT_STRIDE + split_id] = smem.mla_tc.s_l[h];
        }
        float inv_l_top = 1.0f / fmaxf(smem.mla_tc.s_l[h_base + gid], 1.0e-20f);
        float inv_l_bot = 1.0f / fmaxf(smem.mla_tc.s_l[h_base + gid + 8], 1.0e-20f);
        int c_sub = sub_w * 128;
        __syncthreads();
        #pragma unroll
        for (int n = 0; n < 16; ++n) {
            int col = c_sub + n * 8 + 2 * tid;
            __nv_bfloat162 v_top = __floats2bfloat162_rn(o_acc[n][0] * inv_l_top, o_acc[n][1] * inv_l_top);
            __nv_bfloat162 v_bot = __floats2bfloat162_rn(o_acc[n][2] * inv_l_bot, o_acc[n][3] * inv_l_bot);
            *reinterpret_cast<__nv_bfloat162*>(&smem.mla_tc.ckv[h_half][gid][col]) = v_top;
            *reinterpret_cast<__nv_bfloat162*>(&smem.mla_tc.ckv[h_half][gid + 8][col]) = v_bot;
        }
        __syncthreads();
        #pragma unroll
        for (int p = 0; p < 8; ++p) {
            int idx = p * 256 + threadIdx.x;
            int h = idx >> 6;
            int c8 = (idx & 63) * 8;
            uint4 v = *reinterpret_cast<const uint4*>(&smem.mla_tc.ckv[h >> 4][h & 15][c8]);
            store_u4_streaming(&ws->attn_o_splits[(split_id * 32 + h) * KV_LORA + c8], v);
        }
    }
    grid.sync();

    // Stage A4: 256-CTA Coalesced Streaming Split-KV Combine (32 heads * 8 tiles of 64 = 256 CTAs!)
    if (blockIdx.x < 256) {
        int h = blockIdx.x >> 3;
        int c_base = (blockIdx.x & 7) * 64;

        if (wid == 0) {
            const float* m_row = &ws->attn_m[h * KV_SPLIT_STRIDE];
            const float* l_row = &ws->attn_l[h * KV_SPLIT_STRIDE];
            float g_max = -1.0e30f;
            for (int s = lane; s < num_splits; s += 32) {
                if (l_row[s] > 0.0f) {
                    g_max = fmaxf(g_max, m_row[s]);
                }
            }
            #pragma unroll
            for (int off = 16; off > 0; off >>= 1) {
                g_max = fmaxf(g_max, __shfl_xor_sync(0xffffffff, g_max, off));
            }
            float g_sum = 0.0f;
            for (int s = lane; s < num_splits; s += 32) {
                float ls = l_row[s];
                float w = (ls > 0.0f) ? (ls * __expf(m_row[s] - g_max)) : 0.0f;
                smem.mla_comb.scale_s[s] = w;
                g_sum += w;
            }
            g_sum = warp_sum(g_sum);
            if (lane == 0) smem.mla_comb.g_inv_sum = 1.0f / fmaxf(g_sum, 1.0e-20f);
        }
        __syncthreads();

        float2 acc = {0.f, 0.f};
        int col2 = c_base + lane * 2;
        for (int s = wid; s < num_splits; s += 8) {
            float w = smem.mla_comb.scale_s[s];
            if (w > 0.0f) {
                uint32_t bv = load_u32_streaming(&ws->attn_o_splits[(s * 32 + h) * KV_LORA + col2]);
                float2 f01 = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&bv));
                acc.x += w * f01.x;
                acc.y += w * f01.y;
            }
        }
        *reinterpret_cast<float2*>(&smem.mla_comb.partial[wid][lane * 2]) = acc;
        __syncthreads();

        if (threadIdx.x < 64) {
            int cl = threadIdx.x;
            float tot = 0.0f;
            #pragma unroll
            for (int w = 0; w < 8; ++w) tot += smem.mla_comb.partial[w][cl];
            ws->o_latent[h * KV_LORA + c_base + cl] = tot * smem.mla_comb.g_inv_sum;
        }
    }
    grid.sync();

    // Stage A5: 1-Wave TILE_N=128 Absorbed o = o_latent @ W_uv (32 heads * 32 chunks of 16 = 1,024 warps, 4-way split!)
    if (global_warp < 32 * 32) {
        int h = global_warp & 31;       // h innermost -> contiguous 4096B row across 32 warps!
        int sg = global_warp >> 5;      // 0..31 (8 kb rows = 16 c's per sixteenth-group!)
        int g = sg >> 3;                // 0..3

        const float* ol = &ws->o_latent[h * KV_LORA];
        float xsum = 0.0f;
        if (lane < 16) xsum = ol[sg * 16 + lane];
        #pragma unroll
        for (int off = 8; off > 0; off >>= 1) xsum += __shfl_xor_sync(0xffffffff, xsum, off);
        xsum = __shfl_sync(0xffffffff, xsum, 0);

        int col_grp = lane & 15;
        int row_sub = lane >> 4;
        int local_col = col_grp * 8;
        int col_base = h * 256 + 128 + local_col;

        float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
        int kb_base = sg * 8 + row_sub;
        const uint64_t pol_last = make_evict_last_policy();
        #pragma unroll
        for (int step = 0; step < 4; ++step) {
            int kb = kb_base + step * 2;
            uint2 w_vec = load_u2_policy(&mla.kv_b.w_q[kb * 8192 + col_base], pol_last);
            float x_lo = ol[kb * 2];
            float x_hi = ol[kb * 2 + 1];

            uint32_t words[2] = {w_vec.x, w_vec.y};
            #pragma unroll
            for (int w = 0; w < 2; ++w) {
                uint32_t v = words[w];
                uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
                uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
                uint32_t v_shr8 = v >> 8;
                uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
                uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;

                float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
                float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
                float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
                float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));

                g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x;
                g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x;
                g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y;
                g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y;
            }
        }
        #pragma unroll
        for (int c = 0; c < 8; ++c) {
            g_acc[c] += __shfl_down_sync(0xffffffff, g_acc[c], 16);
        }
        if (row_sub == 0) {
            uint4 s_vec = load_u4_policy(&mla.kv_b.scales[g * 8192 + col_base], pol_last);
            uint4 z_vec = load_u4_policy(&mla.kv_b.zeros[g * 8192 + col_base], pol_last);
            const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_vec);
            const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_vec);
            float* op = &ws->mla_o_raw[sg & 3][h * V_HEAD + local_col];
            #pragma unroll
            for (int c = 0; c < 8; ++c) {
                float z_eff = __bfloat162float(zb[c]) + 128.0f;
                float val = __bfloat162float(sb[c]) * (g_acc[c] - z_eff * xsum);
                atomicAdd(&op[c], val);
            }
        }
    }
    grid.sync();

    // Stage A6: CTAs 0..143 run QGPW=2 Row-Contiguous o_proj!
    if (blockIdx.x < 144) {
        #pragma unroll
        for (int pass = 0; pass < 4; ++pass) {
            int base = (pass * 256 + threadIdx.x) * 4;
            float4 o0 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[0][base]);
            float4 o1 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[1][base]);
            float4 o2 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[2][base]);
            float4 o3 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[3][base]);
            smem.gemv.sx[base + 0] = __bfloat162float(__float2bfloat16_rn(o0.x + o1.x + o2.x + o3.x));
            smem.gemv.sx[base + 1] = __bfloat162float(__float2bfloat16_rn(o0.y + o1.y + o2.y + o3.y));
            smem.gemv.sx[base + 2] = __bfloat162float(__float2bfloat16_rn(o0.z + o1.z + o2.z + o3.z));
            smem.gemv.sx[base + 3] = __bfloat162float(__float2bfloat16_rn(o0.w + o1.w + o2.w + o3.w));
        }
        __syncthreads();
        for (int qg = wid; qg < (4096 / QGROUP); qg += 8) {
            float s = warp_sum(smem.gemv.sx[qg * 32 + lane]);
            if (lane == 0) smem.gemv.sx_qgsum[qg] = s;
        }
        __syncthreads();

        int tn = global_warp % 18;
        int kc = global_warp / 18; // 0..63
        warp_int4_gemv_128<2, false>(
            smem.gemv.sx, smem.gemv.sx_qgsum,
            mla.o_proj.w_q, mla.o_proj.scales, mla.o_proj.zeros,
            &ws->attn_o_accum[layer_idx][kc & 3][tn * 128],
            HIDDEN, tn * 128, kc * 2
        );
    }
    grid.sync();
}

__global__ __launch_bounds__(BLOCK_THREADS, 2)
void kimi_decode_megakernel(const __grid_constant__ MegakernelParams params) {
    Workspace* ws = reinterpret_cast<Workspace*>(params.workspace);
    FastGridBarrier grid{ws->grid_bar};
    extern __shared__ char dyn_smem[];
    SharedStorage& smem = *reinterpret_cast<SharedStorage*>(dyn_smem);

    const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x;
    const int total_threads = gridDim.x * BLOCK_THREADS;

    constexpr size_t ZERO_FLOATS = offsetof(Workspace, h) / sizeof(float);
    float4* ws_zero_f4 = reinterpret_cast<float4*>(ws);
    for (size_t i = global_tid; i < ZERO_FLOATS / 4; i += total_threads) {
        ws_zero_f4[i] = make_float4(0.f, 0.f, 0.f, 0.f);
    }
    if (global_tid < HIDDEN / 8) {
        int i = global_tid * 8;
        *reinterpret_cast<uint4*>(&ws->h[i]) = *reinterpret_cast<const uint4*>(&params.hidden_in[i]);
    }

    // Streaming copy on step 0 prevents evicting the 105 MB dense weights from L2 cache!
    if (params.c_kv_in != params.c_kv_out && params.pos > 0) {
        size_t n_ckv = size_t(params.pos) * KV_LORA;
        const uint4* src_c = reinterpret_cast<const uint4*>(params.c_kv_in);
        uint4* dst_c = reinterpret_cast<uint4*>(params.c_kv_out);
        for (size_t i = global_tid; i < n_ckv / 8; i += total_threads) {
            store_u4_streaming(&dst_c[i], load_u4_streaming(&src_c[i]));
        }
    }
    if (params.k_rope_in != params.k_rope_out && params.pos > 0) {
        size_t n_kr = size_t(params.pos) * QK_ROPE;
        const uint4* src_k = reinterpret_cast<const uint4*>(params.k_rope_in);
        uint4* dst_k = reinterpret_cast<uint4*>(params.k_rope_out);
        for (size_t i = global_tid; i < n_kr / 8; i += total_threads) {
            store_u4_streaming(&dst_k[i], load_u4_streaming(&src_k[i]));
        }
    }
    grid.sync();

    run_kda_layer(grid, params, ws, smem, 0);
    run_moe_block(grid, params, ws, smem, 0, false);

    run_kda_layer(grid, params, ws, smem, 1);
    run_moe_block(grid, params, ws, smem, 1, false);

    run_kda_layer(grid, params, ws, smem, 2);
    run_moe_block(grid, params, ws, smem, 2, false);

    run_mla_layer(grid, params, ws, smem, 3);
    run_moe_block(grid, params, ws, smem, 3, true);
}

extern "C" size_t kimi_workspace_bytes() {
    return sizeof(Workspace) + 4096;
}

extern "C" size_t kimi_params_bytes() {
    return sizeof(MegakernelParams);
}

extern "C" int kimi_build_base_params(
    const uint64_t* weight_ptrs,
    float routed_scaling,
    void* out_params,
    int device_id
) {
    cudaSetDevice(device_id);
    int smem_bytes = sizeof(SharedStorage);
    cudaFuncSetAttribute(kimi_decode_megakernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes);
    int blocks_per_sm = 0;
    cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks_per_sm, kimi_decode_megakernel, BLOCK_THREADS, smem_bytes);
    cudaDeviceProp prop;
    cudaGetDeviceProperties(&prop, device_id);
    int max_blocks = blocks_per_sm * prop.multiProcessorCount;
    int target = 2 * prop.multiProcessorCount; // 376 blocks
    int num_blocks = (max_blocks < target) ? max_blocks : target;

    MegakernelParams p = {};
    int idx = 0;
    for (int i = 0; i < 4; ++i) p.attn_norm[i] = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
    for (int i = 0; i < 4; ++i) p.moe_norm[i]  = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);

    auto read_ql = [&](QuantLinearPtrs& ql) {
        ql.w_q    = reinterpret_cast<const uint8_t*>(weight_ptrs[idx++]);
        ql.scales = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
        ql.zeros  = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
    };

    for (int k = 0; k < 3; ++k) {
        read_ql(p.kda[k].q_proj);
        read_ql(p.kda[k].k_proj);
        read_ql(p.kda[k].v_proj);
        read_ql(p.kda[k].g_proj);
        p.kda[k].beta_w = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
        p.kda[k].conv_w = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
        read_ql(p.kda[k].o_proj);
    }

    read_ql(p.mla.q_proj);
    read_ql(p.mla.kv_a);
    read_ql(p.mla.kv_b);
    read_ql(p.mla.o_proj);

    for (int i = 0; i < 4; ++i) {
        p.moe[i].router_w = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
        read_ql(p.moe[i].gate);
        read_ql(p.moe[i].up);
        read_ql(p.moe[i].down);
        read_ql(p.moe[i].s_gate);
        read_ql(p.moe[i].s_up);
        read_ql(p.moe[i].s_down);
    }
    p.routed_scaling = routed_scaling;
    memcpy(out_params, &p, sizeof(MegakernelParams));
    return num_blocks;
}

extern "C" void launch_kimi_fast(
    const void* base_params,
    const uint64_t* state_ptrs,
    int pos,
    int num_blocks,
    cudaStream_t stream
) {
    MegakernelParams p;
    memcpy(&p, base_params, sizeof(MegakernelParams));
    int sidx = 0;
    for (int k = 0; k < 3; ++k) {
        p.kda_st[k].S  = reinterpret_cast<float*>(state_ptrs[sidx++]);
        p.kda_st[k].cq = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
        p.kda_st[k].ck = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
        p.kda_st[k].cv = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
    }
    p.c_kv_in    = reinterpret_cast<const __nv_bfloat16*>(state_ptrs[sidx++]);
    p.c_kv_out   = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
    p.k_rope_in  = reinterpret_cast<const __nv_bfloat16*>(state_ptrs[sidx++]);
    p.k_rope_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
    p.hidden_in  = reinterpret_cast<const __nv_bfloat16*>(state_ptrs[sidx++]);
    p.hidden_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
    p.workspace  = reinterpret_cast<void*>(state_ptrs[sidx++]);
    p.pos = pos;

    kimi_decode_megakernel<<<num_blocks, BLOCK_THREADS, sizeof(SharedStorage), stream>>>(p);
}
"""

_CPP_SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <stdint.h>
#include <tuple>

extern "C" size_t kimi_workspace_bytes();
extern "C" size_t kimi_params_bytes();
extern "C" int kimi_build_base_params(
    const uint64_t* weight_ptrs,
    float routed_scaling,
    void* out_params,
    int device_id
);
extern "C" void launch_kimi_fast(
    const void* base_params,
    const uint64_t* state_ptrs,
    int pos,
    int num_blocks,
    cudaStream_t stream
);

int64_t get_workspace_bytes() {
    return static_cast<int64_t>(kimi_workspace_bytes());
}

std::tuple<torch::Tensor, int64_t> build_base_params(
    torch::Tensor weight_ptrs_cpu,
    double routed_scaling,
    int64_t device_id
) {
    auto buf = torch::empty({static_cast<int64_t>(kimi_params_bytes())}, torch::dtype(torch::kUInt8));
    int nb = kimi_build_base_params(
        reinterpret_cast<const uint64_t*>(weight_ptrs_cpu.data_ptr<int64_t>()),
        static_cast<float>(routed_scaling),
        buf.data_ptr(),
        static_cast<int>(device_id)
    );
    return std::make_tuple(buf, static_cast<int64_t>(nb));
}

static inline torch::Tensor grow_cache_2d(const torch::Tensor& tin, int64_t new_len, int64_t cols) {
    size_t req_bytes = static_cast<size_t>(new_len * cols) * 2;
    if (tin.is_contiguous() && tin.storage_offset() == 0 && tin.storage().nbytes() >= req_bytes) {
        auto out = at::detail::make_tensor<c10::TensorImpl>(
            c10::Storage(tin.storage()), tin.key_set(), tin.dtype()
        );
        int64_t sizes[2] = {new_len, cols};
        int64_t strides[2] = {cols, 1};
        out.unsafeGetTensorImpl()->set_sizes_and_strides(
            c10::IntArrayRef(sizes, 2), c10::IntArrayRef(strides, 2)
        );
        return out;
    }
    auto buf = at::empty({new_len + 64, cols}, tin.options());
    return buf.slice(0, 0, new_len);
}

std::tuple<torch::Tensor, torch::Tensor, torch::Tensor> step_fast(
    torch::Tensor base_params_cpu,
    torch::Tensor s0_S, torch::Tensor s0_cq, torch::Tensor s0_ck, torch::Tensor s0_cv,
    torch::Tensor s1_S, torch::Tensor s1_cq, torch::Tensor s1_ck, torch::Tensor s1_cv,
    torch::Tensor s2_S, torch::Tensor s2_cq, torch::Tensor s2_ck, torch::Tensor s2_cv,
    torch::Tensor c_kv_in, torch::Tensor k_rope_in,
    torch::Tensor hidden_in,
    torch::Tensor workspace,
    int64_t num_blocks
) {
    int64_t pos = c_kv_in.size(0);
    int64_t new_len = pos + 1;
    torch::Tensor c_kv_out = grow_cache_2d(c_kv_in, new_len, 512);
    torch::Tensor k_rope_out = grow_cache_2d(k_rope_in, new_len, 64);
    torch::Tensor hidden_out = at::empty_like(hidden_in);

    uint64_t state_ptrs[19] = {
        reinterpret_cast<uint64_t>(s0_S.data_ptr()),
        reinterpret_cast<uint64_t>(s0_cq.data_ptr()),
        reinterpret_cast<uint64_t>(s0_ck.data_ptr()),
        reinterpret_cast<uint64_t>(s0_cv.data_ptr()),
        reinterpret_cast<uint64_t>(s1_S.data_ptr()),
        reinterpret_cast<uint64_t>(s1_cq.data_ptr()),
        reinterpret_cast<uint64_t>(s1_ck.data_ptr()),
        reinterpret_cast<uint64_t>(s1_cv.data_ptr()),
        reinterpret_cast<uint64_t>(s2_S.data_ptr()),
        reinterpret_cast<uint64_t>(s2_cq.data_ptr()),
        reinterpret_cast<uint64_t>(s2_ck.data_ptr()),
        reinterpret_cast<uint64_t>(s2_cv.data_ptr()),
        reinterpret_cast<uint64_t>(c_kv_in.data_ptr()),
        reinterpret_cast<uint64_t>(c_kv_out.data_ptr()),
        reinterpret_cast<uint64_t>(k_rope_in.data_ptr()),
        reinterpret_cast<uint64_t>(k_rope_out.data_ptr()),
        reinterpret_cast<uint64_t>(hidden_in.data_ptr()),
        reinterpret_cast<uint64_t>(hidden_out.data_ptr()),
        reinterpret_cast<uint64_t>(workspace.data_ptr()),
    };
    cudaStream_t stream = at::cuda::getCurrentCUDAStream(hidden_in.device().index()).stream();
    launch_kimi_fast(
        base_params_cpu.data_ptr(),
        state_ptrs,
        static_cast<int>(pos),
        static_cast<int>(num_blocks),
        stream
    );
    return std::make_tuple(hidden_out, c_kv_out, k_rope_out);
}
"""

_EXT = None


def _get_ext():
    global _EXT
    if _EXT is None:
        _EXT = load_inline(
            name="kimi_linear_megakernel_v9",
            cpp_sources=_CPP_SRC,
            cuda_sources=_CUDA_SRC,
            functions=["get_workspace_bytes", "build_base_params", "step_fast"],
            no_implicit_headers=True,
            extra_cuda_cflags=["-O3", "--use_fast_math"],
        )
    return _EXT


class Model(nn.Module):
    def __init__(self, cfg: Config):
        super().__init__()
        self.cfg = cfg
        self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern)
        self._ws: torch.Tensor | None = None
        self._base_params: torch.Tensor | None = None
        self._num_blocks: int = 376
        _get_ext()

    def _apply(self, fn):
        self._base_params = None
        self._ws = None
        return super()._apply(fn)

    def reset_parameters(self):
        g = torch.Generator(device="cpu").manual_seed(1234)
        for mod in self.modules():
            if isinstance(mod, (QuantLinear, QuantExperts)):
                mod.init_random(g)
            elif isinstance(mod, nn.Linear):
                nn.init.normal_(mod.weight, 0.0, 0.02, generator=g)
            elif isinstance(mod, KDA):
                nn.init.normal_(mod.conv_w, 0.0, 0.1, generator=g)
        self._base_params = None

    def _prepare(self, device: torch.device) -> None:
        ext = _get_ext()
        if self._ws is None or self._ws.device != device:
            self._ws = torch.zeros(ext.get_workspace_bytes(), dtype=torch.uint8, device=device)

        ptrs: list[int] = []
        for blk in self.blocks:
            ptrs.append(blk.attn_norm.data_ptr())
        for blk in self.blocks:
            ptrs.append(blk.moe_norm.data_ptr())

        def add_ql(ql):
            ptrs.append(ql.w_q.data_ptr())
            ptrs.append(ql.scales.data_ptr())
            ptrs.append(ql.zeros.data_ptr())

        for k in range(3):
            attn: KDA = self.blocks[k].attn
            add_ql(attn.q_proj)
            add_ql(attn.k_proj)
            add_ql(attn.v_proj)
            add_ql(attn.g_proj)
            ptrs.append(attn.beta_proj.weight.data_ptr())
            ptrs.append(attn.conv_w.data_ptr())
            add_ql(attn.o_proj)

        mla: MLA = self.blocks[3].attn
        add_ql(mla.q_proj)
        add_ql(mla.kv_a)
        add_ql(mla.kv_b)
        add_ql(mla.o_proj)

        for i in range(4):
            moe: MoE = self.blocks[i].moe
            ptrs.append(moe.router.weight.data_ptr())
            add_ql(moe.gate)
            add_ql(moe.up)
            add_ql(moe.down)
            add_ql(moe.s_gate)
            add_ql(moe.s_up)
            add_ql(moe.s_down)

        dev_idx = device.index if device.index is not None else 0
        w_t = torch.tensor(ptrs, dtype=torch.int64, device="cpu").contiguous()
        self._base_params, nb = ext.build_base_params(w_t, self.cfg.routed_scaling, dev_idx)
        self._num_blocks = int(nb)

    def step(self, hidden: torch.Tensor, state: list):
        if self._base_params is None:
            self._prepare(hidden.device)

        s0, s1, s2, s3 = state[0], state[1], state[2], state[3]
        hidden_out, c_kv_out, k_rope_out = _EXT.step_fast(
            self._base_params,
            s0["S"], s0["cq"], s0["ck"], s0["cv"],
            s1["S"], s1["cq"], s1["ck"], s1["cv"],
            s2["S"], s2["cq"], s2["ck"], s2["cv"],
            s3["c_kv"], s3["k_rope"],
            hidden,
            self._ws,
            self._num_blocks,
        )
        s3["c_kv"] = c_kv_out
        s3["k_rope"] = k_rope_out
        return hidden_out, state


def init_state(cfg: Config, context_len: int, seed: int) -> list:
    dev = torch.device("cuda:0")
    g = torch.Generator(device=dev).manual_seed(seed)
    H, Dk = cfg.kda_heads, cfg.kda_head_dim
    C = H * Dk
    state = []
    for kind in cfg.pattern:
        if kind == "K":
            state.append({
                "S": torch.randn(H, Dk, Dk, device=dev, generator=g) * 0.05,
                "cq": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
                "ck": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
                "cv": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
            })
        else:
            state.append({
                "c_kv": torch.randn(context_len, cfg.kv_lora, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
                "k_rope": torch.randn(context_len, cfg.qk_rope, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
            })
    return state


def init_token(cfg: Config, seed: int) -> torch.Tensor:
    dev = torch.device("cuda:0")
    g = torch.Generator(device=dev).manual_seed(seed + 1)
    return torch.randn(cfg.hidden, device=dev, generator=g, dtype=cfg.dtype) * 0.25

20261110_113616_agy_gemini-4-carbon-high_02_kimi_linear_decode