"""Single-launch CUDA megakernel for Kimi-Linear W4A16 hybrid decode.""" from __future__ import annotations from dataclasses import dataclass, field import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline OP_TYPE = "kimi_linear_w4a16_decode" HARDWARE_REQUIRED = ["RTX_PRO_6000"] EPS = 1.0e-6 GROUP_SIZE = 128 @dataclass(frozen=True) class Config: hidden: int = 2304 kda_heads: int = 32 kda_head_dim: int = 128 short_conv: int = 4 mla_heads: int = 32 kv_lora: int = 512 qk_nope: int = 128 qk_rope: int = 64 v_head: int = 128 rope_theta: float = 10000.0 n_experts: int = 64 n_active: int = 8 n_shared: int = 1 moe_inter: int = 1024 routed_scaling: float = 2.446 group: int = 128 pattern: tuple = ("K", "K", "K", "M") dtype: torch.dtype = field(default=torch.bfloat16) def build_config(shape: dict) -> Config: return Config(n_experts=int(shape.get("n_experts", 64))) def _pack_int4(w_q: torch.Tensor) -> torch.Tensor: lo = w_q[0::2] & 0xF hi = w_q[1::2] & 0xF return (lo | (hi << 4)).contiguous() def quantize(w_io: torch.Tensor, group: int = GROUP_SIZE): K, N = w_io.shape ng = K // group wg = w_io.view(ng, group, N).float() wmin = wg.min(dim=1, keepdim=True).values wmax = wg.max(dim=1, keepdim=True).values scales = (wmax - wmin).clamp_min(1e-8) / 15.0 zeros = (-wmin / scales).round().clamp(0, 15) w_q = ((wg / scales) + zeros).round().clamp(0, 15).to(torch.uint8).view(K, N) return ( _pack_int4(w_q), scales.squeeze(1).to(torch.bfloat16), zeros.squeeze(1).to(torch.bfloat16), ) class QuantLinear(nn.Module): def __init__(self, in_f: int, out_f: int, group: int = GROUP_SIZE): super().__init__() assert in_f % group == 0 and in_f % 2 == 0 self.in_f, self.out_f, self.group = in_f, out_f, group ng = in_f // group self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8)) self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16)) self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16)) def init_random(self, gen: torch.Generator, std: float = 0.02) -> None: w = torch.randn(self.in_f, self.out_f, generator=gen) * std wq, s, z = quantize(w, self.group) self.w_q.copy_(wq) self.scales.copy_(s) self.zeros.copy_(z) class QuantExperts(nn.Module): def __init__(self, n: int, in_f: int, out_f: int, group: int = GROUP_SIZE): super().__init__() self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group ng = in_f // group self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8)) self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16)) self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16)) def init_random(self, gen: torch.Generator, std: float = 0.02) -> None: for e in range(self.n): w = torch.randn(self.in_f, self.out_f, generator=gen) * std wq, s, z = quantize(w, self.group) self.w_q[e].copy_(wq) self.scales[e].copy_(s) self.zeros[e].copy_(z) class KDA(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden self.q_proj = QuantLinear(d, H * Dk, cfg.group) self.k_proj = QuantLinear(d, H * Dk, cfg.group) self.v_proj = QuantLinear(d, H * Dk, cfg.group) self.g_proj = QuantLinear(d, H * Dk, cfg.group) self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype) self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype)) self.o_proj = QuantLinear(H * Dk, d, cfg.group) class MLA(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg H, d = cfg.mla_heads, cfg.hidden self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group) self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group) self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group) self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group) class MoE(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype) self.gate = QuantExperts(E, d, m, cfg.group) self.up = QuantExperts(E, d, m, cfg.group) self.down = QuantExperts(E, m, d, cfg.group) self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group) self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group) self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group) class Block(nn.Module): def __init__(self, cfg: Config, kind: str): super().__init__() self.kind = kind self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype)) self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype)) self.attn = KDA(cfg) if kind == "K" else MLA(cfg) self.moe = MoE(cfg) _CUDA_SRC = r""" #include #include #include #include #include constexpr int HIDDEN = 2304; constexpr int KDA_DK = 128; constexpr int KDA_C = 4096; constexpr int KV_LORA = 512; constexpr int QK_NOPE = 128; constexpr int QK_ROPE = 64; constexpr int V_HEAD = 128; constexpr int MOE_INTER = 1024; constexpr int GROUP = 128; constexpr int QGROUP = 32; constexpr int BLOCK_THREADS = 256; constexpr int MAX_KV_SPLITS = 128; constexpr int KV_SPLIT_STRIDE = 128; // Sense-flipping single-word GPU release-acquire barrier across all resident CTAs struct FastGridBarrier { uint32_t* arrived; __device__ __forceinline__ void sync() { __barrier_sync(0); if (threadIdx.x == 0) { uint32_t nb = (blockIdx.x == 0) ? (0x80000000u - (gridDim.x - 1u)) : 1u; uint32_t old_arrive; asm volatile("atom.add.release.gpu.u32 %0, [%1], %2;\n" : "=r"(old_arrive) : "l"(arrived), "r"(nb) : "memory"); uint32_t cur_arrive; do { asm volatile("ld.acquire.gpu.u32 %0, [%1];\n" : "=r"(cur_arrive) : "l"(arrived) : "memory"); } while (((old_arrive ^ cur_arrive) & 0x80000000u) == 0u); } __barrier_sync(0); } }; struct QuantLinearPtrs { const uint8_t* w_q; const __nv_bfloat16* scales; const __nv_bfloat16* zeros; }; struct KDAPtrs { QuantLinearPtrs q_proj; QuantLinearPtrs k_proj; QuantLinearPtrs v_proj; QuantLinearPtrs g_proj; const __nv_bfloat16* beta_w; // [32, 2304] const __nv_bfloat16* conv_w; // [3, 4096, 4] QuantLinearPtrs o_proj; // [4096, 2304] }; struct MLAPtrs { QuantLinearPtrs q_proj; // [2304, 6144] QuantLinearPtrs kv_a; // [2304, 576] QuantLinearPtrs kv_b; // [512, 8192] QuantLinearPtrs o_proj; // [4096, 2304] }; struct MoEPtrs { const __nv_bfloat16* router_w; // [64, 2304] QuantLinearPtrs gate; // [64, 1152, 1024] QuantLinearPtrs up; // [64, 1152, 1024] QuantLinearPtrs down; // [64, 512, 2304] QuantLinearPtrs s_gate; // [1, 1152, 1024] QuantLinearPtrs s_up; // [1, 1152, 1024] QuantLinearPtrs s_down; // [1, 512, 2304] }; struct KDAStatePtrs { float* S; // [32, 128, 128] __nv_bfloat16* cq; // [3, 4096] __nv_bfloat16* ck; // [3, 4096] __nv_bfloat16* cv; // [3, 4096] }; struct MegakernelParams { const __nv_bfloat16* attn_norm[4]; const __nv_bfloat16* moe_norm[4]; KDAPtrs kda[3]; MLAPtrs mla; MoEPtrs moe[4]; KDAStatePtrs kda_st[3]; const __nv_bfloat16* c_kv_in; __nv_bfloat16* c_kv_out; const __nv_bfloat16* k_rope_in; __nv_bfloat16* k_rope_out; int pos; float routed_scaling; const __nv_bfloat16* hidden_in; __nv_bfloat16* hidden_out; void* workspace; }; struct alignas(16) Workspace { // Dedicated per-layer split atomicAdd buffers & arrival counters (zeroed once at start of step!) alignas(16) float kda_qkvg[3][2][16384]; // 384 KB (2-way split!) alignas(16) float mla_qkv[4][6720]; // 105 KB (4-way split!) alignas(16) float mla_o_raw[4][4096]; // 64 KB (4-way split!) alignas(16) float attn_o_accum[4][4][2304]; // 144 KB (4-way split!) alignas(16) float moe_gu_accum[4][2][9 * 2048]; // 576 KB (2-way split!) alignas(16) float moe_down_accum[4][4][2304]; // 144 KB (4-way split!) alignas(16) uint32_t router_done[4]; // 16 B alignas(16) uint32_t moe_down_done[4]; // 16 B // End of zeroed section (~1.4 MB -> ~1.4 us across 376 CTAs!) alignas(16) __nv_bfloat16 h[2304]; alignas(16) __nv_bfloat16 h_after_attn[2304]; alignas(16) __nv_bfloat16 x_norm[2304]; alignas(16) float x_qgsum[80]; alignas(16) float beta[32]; alignas(16) __nv_bfloat16 attn_o_bf16[4096]; alignas(16) float router_logits[64]; alignas(16) int topk_idx[8]; alignas(16) float topk_w[12]; alignas(16) __nv_bfloat16 q_latent_bf16[32 * 512]; alignas(16) __nv_bfloat16 q_rope_bf16[32 * 64]; alignas(16) float o_latent[32 * 512]; alignas(16) float attn_m[32 * KV_SPLIT_STRIDE]; alignas(16) float attn_l[32 * KV_SPLIT_STRIDE]; alignas(16) __nv_bfloat16 attn_o_splits[MAX_KV_SPLITS * 32 * 512]; alignas(16) uint32_t grid_bar[4]; }; __device__ __forceinline__ float silu_f(float x) { return x / (1.0f + __expf(-x)); } __device__ __forceinline__ float warp_sum(float v) { #pragma unroll for (int offset = 16; offset > 0; offset >>= 1) { v += __shfl_xor_sync(0xffffffff, v, offset); } return v; } __device__ __forceinline__ uint64_t make_evict_first_policy() { uint64_t policy; asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;\n" : "=l"(policy)); return policy; } __device__ __forceinline__ uint64_t make_evict_last_policy() { uint64_t policy; asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;\n" : "=l"(policy)); return policy; } __device__ __forceinline__ uint32_t load_u32_policy(const void* ptr, uint64_t policy) { uint32_t ret; asm volatile("ld.global.L1::no_allocate.L2::cache_hint.u32 %0, [%1], %2;\n" : "=r"(ret) : "l"(ptr), "l"(policy)); return ret; } __device__ __forceinline__ uint2 load_u2_policy(const void* ptr, uint64_t policy) { uint2 ret; asm volatile("ld.global.L1::no_allocate.L2::cache_hint.v2.u32 {%0, %1}, [%2], %3;\n" : "=r"(ret.x), "=r"(ret.y) : "l"(ptr), "l"(policy)); return ret; } __device__ __forceinline__ uint4 load_u4_policy(const void* ptr, uint64_t policy) { uint4 ret; asm volatile("ld.global.L1::no_allocate.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;\n" : "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w) : "l"(ptr), "l"(policy)); return ret; } __device__ __forceinline__ void cp_async_16_evict_first(void* smem_ptr, const void* glob_ptr, bool pred, uint64_t policy) { uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_ptr)); int src_bytes = pred ? 16 : 0; asm volatile( "cp.async.cg.shared.global.L2::cache_hint [%0], [%1], 16, %2, %3;\n" :: "r"(smem_addr), "l"(glob_ptr), "r"(src_bytes), "l"(policy) ); } __device__ __forceinline__ void cp_async_commit() { asm volatile("cp.async.commit_group;\n" ::: "memory"); } template __device__ __forceinline__ void cp_async_wait() { asm volatile("cp.async.wait_group %0;\n" :: "n"(N) : "memory"); } __device__ __forceinline__ uint32_t load_u32_streaming(const void* ptr) { uint32_t ret; asm volatile("ld.global.cs.u32 %0, [%1];\n" : "=r"(ret) : "l"(ptr)); return ret; } __device__ __forceinline__ void store_u32_streaming(void* ptr, uint32_t v) { asm volatile("st.global.cs.u32 [%0], %1;\n" :: "l"(ptr), "r"(v) : "memory"); } __device__ __forceinline__ uint2 load_u2_streaming(const void* ptr) { uint2 ret; asm volatile("ld.global.cs.v2.u32 {%0, %1}, [%2];\n" : "=r"(ret.x), "=r"(ret.y) : "l"(ptr)); return ret; } __device__ __forceinline__ uint4 load_u4_streaming(const void* ptr) { uint4 ret; asm volatile("ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n" : "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w) : "l"(ptr)); return ret; } __device__ __forceinline__ void store_u4_streaming(void* ptr, uint4 v) { asm volatile("st.global.cs.v4.u32 [%0], {%1, %2, %3, %4};\n" : : "l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w) : "memory"); } __device__ __forceinline__ void mma_m16n8k16_bf16( float& d0, float& d1, float& d2, float& d3, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1 ) { asm volatile( "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n" : "=f"(d0), "=f"(d1), "=f"(d2), "=f"(d3) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1), "f"(d0), "f"(d1), "f"(d2), "f"(d3) ); } __device__ __forceinline__ void ldmatrix_x2_trans(uint32_t& b0, uint32_t& b1, const void* smem_ptr) { uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile( "ldmatrix.sync.aligned.m8n8.x2.trans.shared.b16 {%0, %1}, [%2];\n" : "=r"(b0), "=r"(b1) : "r"(addr) ); } __device__ __forceinline__ void load_x_norm_to_smem( const Workspace* ws, float* __restrict__ sx, float* __restrict__ sx_qgsum ) { #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 xv = *reinterpret_cast(&ws->x_norm[base]); const __nv_bfloat16* xb = reinterpret_cast(&xv); #pragma unroll for (int m = 0; m < 8; ++m) { sx[base + m] = __bfloat162float(xb[m]); } } } if (threadIdx.x < (HIDDEN / QGROUP)) { sx_qgsum[threadIdx.x] = ws->x_qgsum[threadIdx.x]; } __syncthreads(); } __device__ __forceinline__ void cta_rmsnorm_from_h_2304( const __nv_bfloat16* __restrict__ h_in, const __nv_bfloat16* __restrict__ norm_w, float* __restrict__ sx, float* __restrict__ sx_qgsum, // nullptr if qgsum not needed float* __restrict__ s_red ) { float reg_h[16]; float sq_sum = 0.0f; #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 hv = *reinterpret_cast(&h_in[base]); const __nv_bfloat16* hb = reinterpret_cast(&hv); #pragma unroll for (int m = 0; m < 8; ++m) { float vf = __bfloat162float(hb[m]); reg_h[pass * 8 + m] = vf; sq_sum += vf * vf; } } } int wid = threadIdx.x >> 5; int lane = threadIdx.x & 31; sq_sum = warp_sum(sq_sum); if (lane == 0) s_red[wid] = sq_sum; __syncthreads(); if (wid == 0) { float v = (lane < 8) ? s_red[lane] : 0.0f; v = warp_sum(v); if (lane == 0) s_red[0] = rsqrtf(v * (1.0f / float(HIDDEN)) + 1.0e-6f); } __syncthreads(); float inv_rms = s_red[0]; #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 wv = *reinterpret_cast(&norm_w[base]); const __nv_bfloat16* wb = reinterpret_cast(&wv); #pragma unroll for (int m = 0; m < 8; ++m) { __nv_bfloat16 b = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m])); sx[base + m] = __bfloat162float(b); } } } __syncthreads(); if (sx_qgsum != nullptr) { for (int qg = wid; qg < (HIDDEN / QGROUP); qg += 8) { float s = warp_sum(sx[qg * 32 + lane]); if (lane == 0) sx_qgsum[qg] = s; } __syncthreads(); } } __device__ __forceinline__ void reduce4_only_2304( const __nv_bfloat16* __restrict__ res_in, const float (&accum)[4][HIDDEN], __nv_bfloat16* __restrict__ res_out ) { #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 hv = *reinterpret_cast(&res_in[base]); const __nv_bfloat16* hb = reinterpret_cast(&hv); float4 d0a = *reinterpret_cast(&accum[0][base]); float4 d0b = *reinterpret_cast(&accum[0][base + 4]); float4 d1a = *reinterpret_cast(&accum[1][base]); float4 d1b = *reinterpret_cast(&accum[1][base + 4]); float4 d2a = *reinterpret_cast(&accum[2][base]); float4 d2b = *reinterpret_cast(&accum[2][base + 4]); float4 d3a = *reinterpret_cast(&accum[3][base]); float4 d3b = *reinterpret_cast(&accum[3][base + 4]); float ds[8] = { d0a.x + d1a.x + d2a.x + d3a.x, d0a.y + d1a.y + d2a.y + d3a.y, d0a.z + d1a.z + d2a.z + d3a.z, d0a.w + d1a.w + d2a.w + d3a.w, d0b.x + d1b.x + d2b.x + d3b.x, d0b.y + d1b.y + d2b.y + d3b.y, d0b.z + d1b.z + d2b.z + d3b.z, d0b.w + d1b.w + d2b.w + d3b.w }; uint4 out_h; __nv_bfloat16* ob = reinterpret_cast<__nv_bfloat16*>(&out_h); #pragma unroll for (int m = 0; m < 8; ++m) { __nv_bfloat16 mo = __float2bfloat16_rn(ds[m]); ob[m] = __float2bfloat16_rn(__bfloat162float(hb[m]) + __bfloat162float(mo)); } *reinterpret_cast(&res_out[base]) = out_h; } } } template __device__ __forceinline__ void warp_int4_gemv_128( const float* __restrict__ sx, const float* __restrict__ sx_qgsum, const uint8_t* __restrict__ w_q, const __nv_bfloat16* __restrict__ scales, const __nv_bfloat16* __restrict__ zeros, float* __restrict__ out_accum, int N_stride, int col_tile_offset, int qg_start, int valid_cols = 128 ) { const int lane = threadIdx.x & 31; const int col_grp = lane & 15; const int row_sub = lane >> 4; const int local_col = col_grp * 8; if constexpr (CHECK_VALID) { if (local_col >= valid_cols) return; } const int col_base = col_tile_offset + local_col; float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; const int kb_base = qg_start * 16 + row_sub; constexpr int STEPS = QGPW * 8; const uint64_t pol = STREAM_LOAD ? make_evict_first_policy() : make_evict_last_policy(); #pragma unroll 8 for (int step = 0; step < STEPS; ++step) { int kb = kb_base + step * 2; const void* w_ptr = &w_q[kb * N_stride + col_base]; uint2 w_vec = load_u2_policy(w_ptr, pol); float x_lo = sx[kb * 2]; float x_hi = sx[kb * 2 + 1]; uint32_t words[2] = {w_vec.x, w_vec.y}; #pragma unroll for (int w = 0; w < 2; ++w) { uint32_t v = words[w]; uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u; uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u; uint32_t v_shr8 = v >> 8; uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u; uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u; float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u)); float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u)); float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u)); float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u)); g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x; g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x; g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y; g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y; } } uint32_t mask = 0xffffffffu; if constexpr (CHECK_VALID) { mask = (valid_cols >= 128) ? 0xffffffffu : 0x00ff00ffu; } #pragma unroll for (int c = 0; c < 8; ++c) { g_acc[c] += __shfl_down_sync(mask, g_acc[c], 16); } if (row_sub == 0) { int g = qg_start >> 2; float xsum = sx_qgsum[qg_start]; if constexpr (QGPW >= 2) xsum += sx_qgsum[qg_start + 1]; if constexpr (QGPW >= 4) xsum += sx_qgsum[qg_start + 2] + sx_qgsum[qg_start + 3]; const void* s_ptr = &scales[g * N_stride + col_base]; const void* z_ptr = &zeros[g * N_stride + col_base]; uint4 s_vec = load_u4_policy(s_ptr, pol); uint4 z_vec = load_u4_policy(z_ptr, pol); const __nv_bfloat16* sb = reinterpret_cast(&s_vec); const __nv_bfloat16* zb = reinterpret_cast(&z_vec); #pragma unroll for (int c = 0; c < 8; ++c) { float z_eff = __bfloat162float(zb[c]) + 128.0f; float val = __bfloat162float(sb[c]) * (g_acc[c] - z_eff * xsum); atomicAdd(&out_accum[local_col + c], val); } } } union alignas(16) SharedStorage { struct { alignas(16) float sx[4096]; // 16 KB alignas(16) float sx_qgsum[128]; // 512 B alignas(16) float s_red[32]; // 128 B alignas(16) int topk_idx[8]; alignas(16) float topk_w[12]; } gemv; struct { alignas(16) float inter[512]; // 2 KB (Half-K = 512 per CTA!) alignas(16) float qgsum[16]; // 64 B alignas(16) float raw_acc[8][128]; // 4 KB (8-warp smem reduction!) } moe_down; struct { alignas(16) float sq[128]; alignas(16) float sk[128]; alignas(16) float sv[128]; alignas(16) float sdec[128]; alignas(16) float s_red[8][16]; } kda_rec; struct { alignas(16) __nv_bfloat16 ckv[2][16][520]; // 33,280 B (2-stage cp.async double buffer!) alignas(16) __nv_bfloat16 krp[2][16][72]; // 4,608 B alignas(16) float s_part[4][32][16]; // 8,192 B alignas(16) __nv_bfloat16 s_p[32][24]; // 1,536 B alignas(16) float s_escale[32]; // 128 B alignas(16) float s_m[32]; // 128 B alignas(16) float s_l[32]; // 128 B } mla_tc; // 48,000 B -> 2 blocks/SM (96,000 B <= 101,376 B)! struct { alignas(16) float scale_s[KV_SPLIT_STRIDE]; float g_inv_sum; alignas(16) float partial[8][64]; // 2 KB (64 cols per CTA across 256 CTAs!) } mla_comb; }; __device__ void run_moe_block( FastGridBarrier& grid, const MegakernelParams& params, Workspace* ws, SharedStorage& smem, int layer_idx, bool write_final_out ) { const int lane = threadIdx.x & 31; const int wid = threadIdx.x >> 5; const MoEPtrs& moe = params.moe[layer_idx]; // Stage M1: CTAs 0..64 fuse Residual + MoE RMSNorm + 64-Expert Router GEMV + Last-CTA Top-8! // Only 65 CTAs read attn_o_accum (no 376-CTA hotspot, zero serial tail on o_proj!). if (blockIdx.x < 65) { float reg_h[16]; float sq_sum = 0.0f; #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 hv = *reinterpret_cast(&ws->h[base]); const __nv_bfloat16* hb = reinterpret_cast(&hv); float4 a0a = *reinterpret_cast(&ws->attn_o_accum[layer_idx][0][base]); float4 a0b = *reinterpret_cast(&ws->attn_o_accum[layer_idx][0][base + 4]); float4 a1a = *reinterpret_cast(&ws->attn_o_accum[layer_idx][1][base]); float4 a1b = *reinterpret_cast(&ws->attn_o_accum[layer_idx][1][base + 4]); float4 a2a = *reinterpret_cast(&ws->attn_o_accum[layer_idx][2][base]); float4 a2b = *reinterpret_cast(&ws->attn_o_accum[layer_idx][2][base + 4]); float4 a3a = *reinterpret_cast(&ws->attn_o_accum[layer_idx][3][base]); float4 a3b = *reinterpret_cast(&ws->attn_o_accum[layer_idx][3][base + 4]); float as[8] = { a0a.x + a1a.x + a2a.x + a3a.x, a0a.y + a1a.y + a2a.y + a3a.y, a0a.z + a1a.z + a2a.z + a3a.z, a0a.w + a1a.w + a2a.w + a3a.w, a0b.x + a1b.x + a2b.x + a3b.x, a0b.y + a1b.y + a2b.y + a3b.y, a0b.z + a1b.z + a2b.z + a3b.z, a0b.w + a1b.w + a2b.w + a3b.w }; uint4 out_h; __nv_bfloat16* ob = reinterpret_cast<__nv_bfloat16*>(&out_h); #pragma unroll for (int m = 0; m < 8; ++m) { __nv_bfloat16 ao = __float2bfloat16_rn(as[m]); __nv_bfloat16 hm = __float2bfloat16_rn(__bfloat162float(hb[m]) + __bfloat162float(ao)); ob[m] = hm; float vf = __bfloat162float(hm); reg_h[pass * 8 + m] = vf; sq_sum += vf * vf; } if (blockIdx.x == 64) { *reinterpret_cast(&ws->h_after_attn[base]) = out_h; } } } sq_sum = warp_sum(sq_sum); if (lane == 0) smem.gemv.s_red[wid] = sq_sum; __syncthreads(); if (wid == 0) { float v = (lane < 8) ? smem.gemv.s_red[lane] : 0.0f; v = warp_sum(v); if (lane == 0) smem.gemv.s_red[0] = rsqrtf(v * (1.0f / float(HIDDEN)) + 1.0e-6f); } __syncthreads(); float inv_rms = smem.gemv.s_red[0]; const __nv_bfloat16* mw = params.moe_norm[layer_idx]; if (blockIdx.x == 64) { #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 wv = *reinterpret_cast(&mw[base]); const __nv_bfloat16* wb = reinterpret_cast(&wv); uint4 out_xn; __nv_bfloat16* xnb = reinterpret_cast<__nv_bfloat16*>(&out_xn); #pragma unroll for (int m = 0; m < 8; ++m) { __nv_bfloat16 xn = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m])); xnb[m] = xn; smem.gemv.sx[base + m] = __bfloat162float(xn); } *reinterpret_cast(&ws->x_norm[base]) = out_xn; } } __syncthreads(); for (int qg = wid; qg < (HIDDEN / QGROUP); qg += 8) { float s = warp_sum(smem.gemv.sx[qg * 32 + lane]); if (lane == 0) ws->x_qgsum[qg] = s; } } else { int e = blockIdx.x; const __nv_bfloat16* rw = &moe.router_w[e * HIDDEN]; float dot = 0.0f; #pragma unroll for (int pass = 0; pass < 2; ++pass) { int vec_idx = pass * 256 + threadIdx.x; if (vec_idx < 288) { int base = vec_idx * 8; uint4 wv = *reinterpret_cast(&mw[base]); uint4 rv = *reinterpret_cast(&rw[base]); const __nv_bfloat16* wb = reinterpret_cast(&wv); const __nv_bfloat16* rb = reinterpret_cast(&rv); #pragma unroll for (int m = 0; m < 8; ++m) { __nv_bfloat16 xn = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m])); dot += __bfloat162float(xn) * __bfloat162float(rb[m]); } } } dot = warp_sum(dot); if (lane == 0) smem.gemv.s_red[8 + wid] = dot; __syncthreads(); if (wid == 0) { float s = (lane < 8) ? smem.gemv.s_red[8 + lane] : 0.0f; s = warp_sum(s); uint32_t prev_done = 0u; if (lane == 0) { float logit = __bfloat162float(__float2bfloat16_rn(s)); asm volatile("st.global.release.gpu.f32 [%0], %1;\n" :: "l"(&ws->router_logits[e]), "f"(logit) : "memory"); asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;\n" : "=r"(prev_done) : "l"(&ws->router_done[layer_idx]) : "memory"); } prev_done = __shfl_sync(0xffffffff, prev_done, 0); if (prev_done == 63u) { int i0 = 2 * lane; int i1 = 2 * lane + 1; float v0, v1; asm volatile("ld.global.acquire.gpu.v2.f32 {%0, %1}, [%2];\n" : "=f"(v0), "=f"(v1) : "l"(&ws->router_logits[i0])); float lmax = fmaxf(v0, v1); #pragma unroll for (int off = 16; off > 0; off >>= 1) { lmax = fmaxf(lmax, __shfl_xor_sync(0xffffffff, lmax, off)); } float p0 = __expf(v0 - lmax); float p1 = __expf(v1 - lmax); float psum = warp_sum(p0 + p1); p0 /= psum; p1 /= psum; float w_sum = 0.0f; #pragma unroll for (int k = 0; k < 8; ++k) { float best_v = (p0 >= p1) ? p0 : p1; int best_i = (p0 >= p1) ? i0 : i1; #pragma unroll for (int off = 16; off > 0; off >>= 1) { float ov = __shfl_xor_sync(0xffffffff, best_v, off); int oi = __shfl_xor_sync(0xffffffff, best_i, off); if (ov > best_v || (ov == best_v && oi < best_i)) { best_v = ov; best_i = oi; } } if (lane == 0) { ws->topk_idx[k] = best_i; smem.gemv.topk_w[k] = best_v; } w_sum += best_v; if (i0 == best_i) p0 = -1.0f; if (i1 == best_i) p1 = -1.0f; } if (lane == 0) { float scale = params.routed_scaling / (w_sum + 1.0e-9f); #pragma unroll for (int k = 0; k < 8; ++k) { ws->topk_w[k] = smem.gemv.topk_w[k] * scale; } ws->topk_w[8] = 1.0f; } } } } } grid.sync(); // Stage M2: Row-Contiguous 1-Wave Gate+Up GEMV (zero Top-K stall, 4.5 KB x_norm load!) if (blockIdx.x < 324) { int slot = blockIdx.x / 36; int rem = blockIdx.x % 36; int is_up = rem / 18; int kc = rem % 18; int ltn = wid; // 0..7 if (threadIdx.x == 0 && slot < 8) { smem.gemv.topk_idx[0] = ws->topk_idx[slot]; } load_x_norm_to_smem(ws, smem.gemv.sx, smem.gemv.sx_qgsum); size_t wq_exp_stride = size_t(HIDDEN / 2) * MOE_INTER; size_t sz_exp_stride = size_t(HIDDEN / GROUP) * MOE_INTER; float* out_ptr = &ws->moe_gu_accum[layer_idx][kc & 1][slot * 2048 + is_up * 1024 + ltn * 128]; if (slot < 8) { int exp_id = smem.gemv.topk_idx[0]; const QuantLinearPtrs* ql = is_up ? &moe.up : &moe.gate; warp_int4_gemv_128<4, true>( smem.gemv.sx, smem.gemv.sx_qgsum, ql->w_q + exp_id * wq_exp_stride, ql->scales + exp_id * sz_exp_stride, ql->zeros + exp_id * sz_exp_stride, out_ptr, MOE_INTER, ltn * 128, kc * 4 ); } else { const QuantLinearPtrs* ql = is_up ? &moe.s_up : &moe.s_gate; warp_int4_gemv_128<4, false>( smem.gemv.sx, smem.gemv.sx_qgsum, ql->w_q, ql->scales, ql->zeros, out_ptr, MOE_INTER, ltn * 128, kc * 4 ); } } grid.sync(); // Stage M3: Half-K SiLU*Up + 8-Warp In-CTA Shared-Memory Reduction Down GEMV + Last-CTA Residual! if (blockIdx.x < 324) { int slot = blockIdx.x / 36; int rem = blockIdx.x % 36; int k_half = rem / 18; // 0..1 (512 K-channels per CTA!) int ltn = rem % 18; // 0..17 (128 output cols per CTA!) float w_s = ws->topk_w[slot]; int exp_id = (slot < 8) ? ws->topk_idx[slot] : 0; { int m = threadIdx.x * 2; int g_idx = slot * 2048 + k_half * 512 + m; float2 g0 = *reinterpret_cast(&ws->moe_gu_accum[layer_idx][0][g_idx]); float2 g1 = *reinterpret_cast(&ws->moe_gu_accum[layer_idx][1][g_idx]); float2 u0 = *reinterpret_cast(&ws->moe_gu_accum[layer_idx][0][g_idx + 1024]); float2 u1 = *reinterpret_cast(&ws->moe_gu_accum[layer_idx][1][g_idx + 1024]); float2 prod = make_float2( w_s * silu_f(g0.x + g1.x) * (u0.x + u1.x), w_s * silu_f(g0.y + g1.y) * (u0.y + u1.y) ); *reinterpret_cast(&smem.moe_down.inter[m]) = prod; } __syncthreads(); for (int qg = wid; qg < 16; qg += 8) { float s = warp_sum(smem.moe_down.inter[qg * 32 + lane]); if (lane == 0) smem.moe_down.qgsum[qg] = s; } __syncthreads(); size_t wq_exp_stride = size_t(MOE_INTER / 2) * HIDDEN; size_t sz_exp_stride = size_t(MOE_INTER / GROUP) * HIDDEN; const uint8_t* w_q_ptr = ((slot < 8) ? (moe.down.w_q + exp_id * wq_exp_stride) : moe.s_down.w_q) + size_t(k_half * 256) * HIDDEN; const __nv_bfloat16* s_ptr_base = ((slot < 8) ? (moe.down.scales + exp_id * sz_exp_stride) : moe.s_down.scales) + size_t(k_half * 4) * HIDDEN; const __nv_bfloat16* z_ptr_base = ((slot < 8) ? (moe.down.zeros + exp_id * sz_exp_stride) : moe.s_down.zeros) + size_t(k_half * 4) * HIDDEN; const uint64_t pol = (slot < 8) ? make_evict_first_policy() : make_evict_last_policy(); const int col_grp = lane & 15; const int row_sub = lane >> 4; const int local_col = col_grp * 8; const int col_base = ltn * 128 + local_col; const int kb_base = wid * 32 + row_sub; float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; #pragma unroll 8 for (int step = 0; step < 16; ++step) { int kb = kb_base + step * 2; uint2 w_vec = load_u2_policy(&w_q_ptr[kb * HIDDEN + col_base], pol); float x_lo = smem.moe_down.inter[kb * 2]; float x_hi = smem.moe_down.inter[kb * 2 + 1]; uint32_t words[2] = {w_vec.x, w_vec.y}; #pragma unroll for (int w = 0; w < 2; ++w) { uint32_t v = words[w]; uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u; uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u; uint32_t v_shr8 = v >> 8; uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u; uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u; float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u)); float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u)); float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u)); float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u)); g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x; g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x; g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y; g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y; } } #pragma unroll for (int c = 0; c < 8; ++c) { g_acc[c] += __shfl_down_sync(0xffffffffu, g_acc[c], 16); } if (row_sub == 0) { *reinterpret_cast(&smem.moe_down.raw_acc[wid][local_col]) = make_float4(g_acc[0], g_acc[1], g_acc[2], g_acc[3]); *reinterpret_cast(&smem.moe_down.raw_acc[wid][local_col + 4]) = make_float4(g_acc[4], g_acc[5], g_acc[6], g_acc[7]); } __syncthreads(); if (wid < 4) { int g_loc = wid; int col4 = lane * 4; int g_col = ltn * 128 + col4; float xsum = smem.moe_down.qgsum[g_loc * 4 + 0] + smem.moe_down.qgsum[g_loc * 4 + 1] + smem.moe_down.qgsum[g_loc * 4 + 2] + smem.moe_down.qgsum[g_loc * 4 + 3]; uint2 s_vec = load_u2_policy(&s_ptr_base[g_loc * HIDDEN + g_col], pol); uint2 z_vec = load_u2_policy(&z_ptr_base[g_loc * HIDDEN + g_col], pol); const __nv_bfloat16* sb = reinterpret_cast(&s_vec); const __nv_bfloat16* zb = reinterpret_cast(&z_vec); float4 r0 = *reinterpret_cast(&smem.moe_down.raw_acc[2 * g_loc][col4]); float4 r1 = *reinterpret_cast(&smem.moe_down.raw_acc[2 * g_loc + 1][col4]); float raw[4] = {r0.x + r1.x, r0.y + r1.y, r0.z + r1.z, r0.w + r1.w}; float deq[4]; #pragma unroll for (int c = 0; c < 4; ++c) { deq[c] = __bfloat162float(sb[c]) * (raw[c] - (__bfloat162float(zb[c]) + 128.0f) * xsum); } *reinterpret_cast(&smem.moe_down.raw_acc[2 * g_loc][col4]) = make_float4(deq[0], deq[1], deq[2], deq[3]); } __syncthreads(); if (threadIdx.x < 128) { int col = threadIdx.x; float sum = smem.moe_down.raw_acc[0][col] + smem.moe_down.raw_acc[2][col] + smem.moe_down.raw_acc[4][col] + smem.moe_down.raw_acc[6][col]; atomicAdd(&ws->moe_down_accum[layer_idx][(slot * 2 + k_half) & 3][ltn * 128 + col], sum); } __syncthreads(); if (threadIdx.x == 0) { uint32_t prev; asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;\n" : "=r"(prev) : "l"(&ws->moe_down_done[layer_idx]) : "memory"); smem.moe_down.qgsum[0] = __uint_as_float(prev); } __syncthreads(); uint32_t prev_m3 = __float_as_uint(smem.moe_down.qgsum[0]); if (prev_m3 == 323u) { reduce4_only_2304( ws->h_after_attn, ws->moe_down_accum[layer_idx], write_final_out ? params.hidden_out : ws->h ); } } grid.sync(); } __device__ void run_kda_layer( FastGridBarrier& grid, const MegakernelParams& params, Workspace* ws, SharedStorage& smem, int layer_idx ) { const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x; const int global_warp = global_tid >> 5; const int lane = threadIdx.x & 31; const int wid = threadIdx.x >> 5; const KDAPtrs& kda = params.kda[layer_idx]; const KDAStatePtrs& st = params.kda_st[layer_idx]; // Stage K1: CTAs 0..287 run RMSNorm + Row-Contiguous (q,k,v,g) GEMV; idle CTAs 288..291 compute beta[0..31]! if (blockIdx.x < 288) { cta_rmsnorm_from_h_2304( ws->h, params.attn_norm[layer_idx], smem.gemv.sx, smem.gemv.sx_qgsum, smem.gemv.s_red ); int ltn = global_warp & 31; int rem = global_warp >> 5; // 0..71 int which = rem / 18; // 0..3 int kc = rem % 18; // 0..17 int tn = which * 32 + ltn; const QuantLinearPtrs* ql = (which == 0) ? &kda.q_proj : (which == 1) ? &kda.k_proj : (which == 2) ? &kda.v_proj : &kda.g_proj; warp_int4_gemv_128<4, false>( smem.gemv.sx, smem.gemv.sx_qgsum, ql->w_q, ql->scales, ql->zeros, &ws->kda_qkvg[layer_idx][kc & 1][tn * 128], KDA_C, ltn * 128, kc * 4 ); } else if (blockIdx.x < 292) { cta_rmsnorm_from_h_2304( ws->h, params.attn_norm[layer_idx], smem.gemv.sx, nullptr, smem.gemv.s_red ); int h = (blockIdx.x - 288) * 8 + wid; const __nv_bfloat16* bw = &kda.beta_w[h * HIDDEN]; float dot = 0.0f; for (int k = lane * 8; k < HIDDEN; k += 256) { uint4 bv = *reinterpret_cast(&bw[k]); const __nv_bfloat16* bb = reinterpret_cast(&bv); #pragma unroll for (int m = 0; m < 8; ++m) { dot += smem.gemv.sx[k + m] * __bfloat162float(bb[m]); } } dot = warp_sum(dot); if (lane == 0) { float bx = __bfloat162float(__float2bfloat16_rn(dot)); ws->beta[h] = 1.0f / (1.0f + __expf(-bx)); } } grid.sync(); // Stage K2: Short Conv + Decay + Recurrent S[32, 128, 128] update! { const int col_grp = lane & 7; const int row_sub = lane >> 3; const int i_tid = wid * 4 + row_sub; const float q_scale = rsqrtf(float(KDA_DK)); if (blockIdx.x < 256) { int h = blockIdx.x >> 3; int j_tile = blockIdx.x & 7; int j_base = j_tile * 16; if (threadIdx.x < 128) { int d = threadIdx.x; int c = h * 128 + d; float q_sum = ws->kda_qkvg[layer_idx][0][c] + ws->kda_qkvg[layer_idx][1][c]; float q_raw = __bfloat162float(__float2bfloat16_rn(q_sum)); const __nv_bfloat16* cw_q = &kda.conv_w[(0 * KDA_C + c) * 4]; float q_c = __bfloat162float(st.cq[0 * KDA_C + c]) * __bfloat162float(cw_q[0]) + __bfloat162float(st.cq[1 * KDA_C + c]) * __bfloat162float(cw_q[1]) + __bfloat162float(st.cq[2 * KDA_C + c]) * __bfloat162float(cw_q[2]) + q_raw * __bfloat162float(cw_q[3]); smem.kda_rec.sq[d] = __bfloat162float(__float2bfloat16_rn(silu_f(q_c))) * q_scale; float k_sum = ws->kda_qkvg[layer_idx][0][KDA_C + c] + ws->kda_qkvg[layer_idx][1][KDA_C + c]; float k_raw = __bfloat162float(__float2bfloat16_rn(k_sum)); const __nv_bfloat16* cw_k = &kda.conv_w[(1 * KDA_C + c) * 4]; float k_c = __bfloat162float(st.ck[0 * KDA_C + c]) * __bfloat162float(cw_k[0]) + __bfloat162float(st.ck[1 * KDA_C + c]) * __bfloat162float(cw_k[1]) + __bfloat162float(st.ck[2 * KDA_C + c]) * __bfloat162float(cw_k[2]) + k_raw * __bfloat162float(cw_k[3]); smem.kda_rec.sk[d] = __bfloat162float(__float2bfloat16_rn(silu_f(k_c))); float v_sum = ws->kda_qkvg[layer_idx][0][2 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][2 * KDA_C + c]; float v_raw = __bfloat162float(__float2bfloat16_rn(v_sum)); const __nv_bfloat16* cw_v = &kda.conv_w[(2 * KDA_C + c) * 4]; float v_c = __bfloat162float(st.cv[0 * KDA_C + c]) * __bfloat162float(cw_v[0]) + __bfloat162float(st.cv[1 * KDA_C + c]) * __bfloat162float(cw_v[1]) + __bfloat162float(st.cv[2 * KDA_C + c]) * __bfloat162float(cw_v[2]) + v_raw * __bfloat162float(cw_v[3]); smem.kda_rec.sv[d] = __bfloat162float(__float2bfloat16_rn(silu_f(v_c))); float g_sum = ws->kda_qkvg[layer_idx][0][3 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][3 * KDA_C + c]; float gx = __bfloat162float(__float2bfloat16_rn(g_sum)); float sp = (gx > 20.0f) ? gx : log1pf(__expf(gx)); smem.kda_rec.sdec[d] = __expf(-sp); } __syncthreads(); float* S_head = &st.S[h * 128 * 128]; int j0 = j_base + col_grp * 2; float2 s_reg[4]; float2 pred = {0.f, 0.f}; #pragma unroll for (int it = 0; it < 4; ++it) { int i = it * 32 + i_tid; float2 sv = *reinterpret_cast(&S_head[i * 128 + j0]); float dec = smem.kda_rec.sdec[i]; sv.x *= dec; sv.y *= dec; s_reg[it] = sv; float kv = smem.kda_rec.sk[i]; pred.x += sv.x * kv; pred.y += sv.y * kv; } pred.x += __shfl_down_sync(0xffffffff, pred.x, 8); pred.y += __shfl_down_sync(0xffffffff, pred.y, 8); pred.x += __shfl_down_sync(0xffffffff, pred.x, 16); pred.y += __shfl_down_sync(0xffffffff, pred.y, 16); if (row_sub == 0) { smem.kda_rec.s_red[wid][col_grp * 2 + 0] = pred.x; smem.kda_rec.s_red[wid][col_grp * 2 + 1] = pred.y; } __syncthreads(); float beta_h = ws->beta[h]; float p0 = 0.f, p1 = 0.f; #pragma unroll for (int w = 0; w < 8; ++w) { p0 += smem.kda_rec.s_red[w][col_grp * 2 + 0]; p1 += smem.kda_rec.s_red[w][col_grp * 2 + 1]; } float d0 = beta_h * (smem.kda_rec.sv[j0 + 0] - p0); float d1 = beta_h * (smem.kda_rec.sv[j0 + 1] - p1); __syncthreads(); float2 o_part = {0.f, 0.f}; #pragma unroll for (int it = 0; it < 4; ++it) { int i = it * 32 + i_tid; float kv = smem.kda_rec.sk[i]; float qv = smem.kda_rec.sq[i]; float2 sv = s_reg[it]; sv.x += kv * d0; sv.y += kv * d1; *reinterpret_cast(&S_head[i * 128 + j0]) = sv; o_part.x += sv.x * qv; o_part.y += sv.y * qv; } o_part.x += __shfl_down_sync(0xffffffff, o_part.x, 8); o_part.y += __shfl_down_sync(0xffffffff, o_part.y, 8); o_part.x += __shfl_down_sync(0xffffffff, o_part.x, 16); o_part.y += __shfl_down_sync(0xffffffff, o_part.y, 16); if (row_sub == 0) { smem.kda_rec.s_red[wid][col_grp * 2 + 0] = o_part.x; smem.kda_rec.s_red[wid][col_grp * 2 + 1] = o_part.y; } __syncthreads(); if (threadIdx.x < 16) { int jl = threadIdx.x; float osum = 0.f; #pragma unroll for (int w = 0; w < 8; ++w) osum += smem.kda_rec.s_red[w][jl]; ws->attn_o_bf16[h * 128 + j_base + jl] = __float2bfloat16_rn(osum); } } } grid.sync(); // Stage K3: CTAs 0..143 run QGPW=2 o_proj; idle CTAs 144..159 shift cq, ck, cv off the critical path! if (blockIdx.x < 144) { #pragma unroll for (int pass = 0; pass < 2; ++pass) { int base = (pass * 256 + threadIdx.x) * 8; uint4 ov = *reinterpret_cast(&ws->attn_o_bf16[base]); const __nv_bfloat16* ob = reinterpret_cast(&ov); #pragma unroll for (int m = 0; m < 8; ++m) smem.gemv.sx[base + m] = __bfloat162float(ob[m]); } __syncthreads(); for (int qg = wid; qg < (KDA_C / QGROUP); qg += 8) { float s = warp_sum(smem.gemv.sx[qg * 32 + lane]); if (lane == 0) smem.gemv.sx_qgsum[qg] = s; } __syncthreads(); int tn = global_warp % 18; int kc = global_warp / 18; // 0..63 warp_int4_gemv_128<2, false>( smem.gemv.sx, smem.gemv.sx_qgsum, kda.o_proj.w_q, kda.o_proj.scales, kda.o_proj.zeros, &ws->attn_o_accum[layer_idx][kc & 3][tn * 128], HIDDEN, tn * 128, kc * 2 ); } else if (blockIdx.x < 160) { int c = (blockIdx.x - 144) * 256 + threadIdx.x; __nv_bfloat16 q1 = st.cq[1 * KDA_C + c], q2 = st.cq[2 * KDA_C + c]; __nv_bfloat16 qn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][c] + ws->kda_qkvg[layer_idx][1][c]); st.cq[0 * KDA_C + c] = q1; st.cq[1 * KDA_C + c] = q2; st.cq[2 * KDA_C + c] = qn; __nv_bfloat16 k1 = st.ck[1 * KDA_C + c], k2 = st.ck[2 * KDA_C + c]; __nv_bfloat16 kn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][KDA_C + c] + ws->kda_qkvg[layer_idx][1][KDA_C + c]); st.ck[0 * KDA_C + c] = k1; st.ck[1 * KDA_C + c] = k2; st.ck[2 * KDA_C + c] = kn; __nv_bfloat16 v1 = st.cv[1 * KDA_C + c], v2 = st.cv[2 * KDA_C + c]; __nv_bfloat16 vn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][2 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][2 * KDA_C + c]); st.cv[0 * KDA_C + c] = v1; st.cv[1 * KDA_C + c] = v2; st.cv[2 * KDA_C + c] = vn; } grid.sync(); } __device__ void run_mla_layer( FastGridBarrier& grid, const MegakernelParams& params, Workspace* ws, SharedStorage& smem, int layer_idx ) { const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x; const int global_warp = global_tid >> 5; const int lane = threadIdx.x & 31; const int wid = threadIdx.x >> 5; const MLAPtrs& mla = params.mla; const int pos = params.pos; const int L = pos + 1; const float mla_scale = rsqrtf(float(QK_NOPE + QK_ROPE)); // Stage A1: Active-CTA RMSNorm + Row-Contiguous q_proj (48 tiles) + kv_a (5 tiles) if (blockIdx.x < 239) { cta_rmsnorm_from_h_2304( ws->h, params.attn_norm[layer_idx], smem.gemv.sx, smem.gemv.sx_qgsum, smem.gemv.s_red ); if (global_warp < 53 * 36) { int tn = global_warp % 53; int kc = global_warp / 53; // 0..35 if (tn < 48) { warp_int4_gemv_128<2, false>( smem.gemv.sx, smem.gemv.sx_qgsum, mla.q_proj.w_q, mla.q_proj.scales, mla.q_proj.zeros, &ws->mla_qkv[kc & 3][tn * 128], 6144, tn * 128, kc * 2 ); } else { int ltn = tn - 48; int valid = (ltn < 4) ? 128 : 64; warp_int4_gemv_128<2, false, true>( smem.gemv.sx, smem.gemv.sx_qgsum, mla.kv_a.w_q, mla.kv_a.scales, mla.kv_a.zeros, &ws->mla_qkv[kc & 3][6144 + ltn * 128], 576, ltn * 128, kc * 2, valid ); } } } grid.sync(); // Stage A2: CTAs 0..255 compute Absorbed q_latent_bf16; idle CTAs 256..262 write c_kv_out & RoPE! if (blockIdx.x < 256) { int h = global_warp >> 6; int quad = global_warp & 63; int g = quad >> 4; int d_base = lane * 4; int col = h * 256 + d_base; float4 q0 = *reinterpret_cast(&ws->mla_qkv[0][h * 192 + d_base]); float4 q1 = *reinterpret_cast(&ws->mla_qkv[1][h * 192 + d_base]); float4 q2 = *reinterpret_cast(&ws->mla_qkv[2][h * 192 + d_base]); float4 q3 = *reinterpret_cast(&ws->mla_qkv[3][h * 192 + d_base]); float qn[4] = { __bfloat162float(__float2bfloat16_rn(q0.x + q1.x + q2.x + q3.x)), __bfloat162float(__float2bfloat16_rn(q0.y + q1.y + q2.y + q3.y)), __bfloat162float(__float2bfloat16_rn(q0.z + q1.z + q2.z + q3.z)), __bfloat162float(__float2bfloat16_rn(q0.w + q1.w + q2.w + q3.w)) }; uint2 s_raw = *reinterpret_cast(&mla.kv_b.scales[g * 8192 + col]); uint2 z_raw = *reinterpret_cast(&mla.kv_b.zeros[g * 8192 + col]); const __nv_bfloat16* sb = reinterpret_cast(&s_raw); const __nv_bfloat16* zb = reinterpret_cast(&z_raw); float qs[4]; float qz_local = 0.0f; #pragma unroll for (int m = 0; m < 4; ++m) { qs[m] = qn[m] * __bfloat162float(sb[m]); qz_local += qs[m] * (__bfloat162float(zb[m]) + 128.0f); } float qz_sum = warp_sum(qz_local); int kb_start = quad * 4; uint4 packed; #pragma unroll for (int s = 0; s < 4; ++s) { int kb = kb_start + s; uint32_t v = *reinterpret_cast(&mla.kv_b.w_q[kb * 8192 + col]); uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u; uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u; uint32_t v_shr8 = v >> 8; uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u; uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u; float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u)); float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u)); float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u)); float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u)); float d_lo = warp_sum(qs[0] * f02_lo.x + qs[1] * f13_lo.x + qs[2] * f02_lo.y + qs[3] * f13_lo.y); float d_hi = warp_sum(qs[0] * f02_hi.x + qs[1] * f13_hi.x + qs[2] * f02_hi.y + qs[3] * f13_hi.y); __nv_bfloat162 b2 = __floats2bfloat162_rn((d_lo - qz_sum) * mla_scale, (d_hi - qz_sum) * mla_scale); uint32_t u2; memcpy(&u2, &b2, sizeof(uint32_t)); if (s == 0) packed.x = u2; else if (s == 1) packed.y = u2; else if (s == 2) packed.z = u2; else packed.w = u2; } if (lane == 0) { *reinterpret_cast(&ws->q_latent_bf16[h * KV_LORA + 2 * kb_start]) = packed; } } else if (blockIdx.x < 258) { int c = (blockIdx.x - 256) * 256 + threadIdx.x; float v = ws->mla_qkv[0][6144 + c] + ws->mla_qkv[1][6144 + c] + ws->mla_qkv[2][6144 + c] + ws->mla_qkv[3][6144 + c]; params.c_kv_out[size_t(pos) * KV_LORA + c] = __float2bfloat16_rn(v); } else if (blockIdx.x < 263) { int rtid = (blockIdx.x - 258) * 256 + threadIdx.x; if (rtid < 33 * 32) { int who = rtid >> 5; int pair = rtid & 31; float inv = powf(10000.0f, -float(2 * pair) * (1.0f / 64.0f)); float ang = float(pos) * inv; float cs, sn; __sincosf(ang, &sn, &cs); int idx_e = (who == 32) ? (6144 + 512 + 2 * pair) : (who * 192 + 128 + 2 * pair); float ev_s = ws->mla_qkv[0][idx_e] + ws->mla_qkv[1][idx_e] + ws->mla_qkv[2][idx_e] + ws->mla_qkv[3][idx_e]; float od_s = ws->mla_qkv[0][idx_e + 1] + ws->mla_qkv[1][idx_e + 1] + ws->mla_qkv[2][idx_e + 1] + ws->mla_qkv[3][idx_e + 1]; float ev = __bfloat162float(__float2bfloat16_rn(ev_s)); float od = __bfloat162float(__float2bfloat16_rn(od_s)); __nv_bfloat16 r0 = __float2bfloat16_rn(ev * cs - od * sn); __nv_bfloat16 r1 = __float2bfloat16_rn(od * cs + ev * sn); if (who == 32) { params.k_rope_out[size_t(pos) * QK_ROPE + 2 * pair + 0] = r0; params.k_rope_out[size_t(pos) * QK_ROPE + 2 * pair + 1] = r1; } else { ws->q_rope_bf16[who * QK_ROPE + 2 * pair + 0] = __float2bfloat16_rn(__bfloat162float(r0) * mla_scale); ws->q_rope_bf16[who * QK_ROPE + 2 * pair + 1] = __float2bfloat16_rn(__bfloat162float(r1) * mla_scale); } } } grid.sync(); // Stage A3: 2-Stage cp.async (L2::evict_first) + Hoisted-Q Register SM120 Tensor Core Flash-Decoding! const int total_tiles16 = max(1, (L + 15) >> 4); const int num_splits = min(MAX_KV_SPLITS, total_tiles16); if (blockIdx.x < num_splits) { const int split_id = blockIdx.x; const int gid = lane >> 2; const int tid = lane & 3; const int h_half = wid >> 2; const int sub_w = wid & 3; const int h_base = h_half * 16; int t_start = (total_tiles16 * split_id) / num_splits; int t_end = (total_tiles16 * (split_id + 1)) / num_splits; int n_iters = t_end - t_start; int l_start = t_start * 16; int l_end = min(L, t_end * 16); if (threadIdx.x < 32) { smem.mla_tc.s_m[threadIdx.x] = -1.0e30f; smem.mla_tc.s_l[threadIdx.x] = 0.0f; } float o_acc[16][4]; #pragma unroll for (int n = 0; n < 16; ++n) { o_acc[n][0] = 0.f; o_acc[n][1] = 0.f; o_acc[n][2] = 0.f; o_acc[n][3] = 0.f; } uint32_t q_reg[8][4]; #pragma unroll for (int kc = 0; kc < 8; ++kc) { int k_col = (sub_w * 8 + kc) * 16; q_reg[kc][0] = *reinterpret_cast(&ws->q_latent_bf16[(h_base + gid) * KV_LORA + k_col + 2 * tid]); q_reg[kc][1] = *reinterpret_cast(&ws->q_latent_bf16[(h_base + gid + 8) * KV_LORA + k_col + 2 * tid]); q_reg[kc][2] = *reinterpret_cast(&ws->q_latent_bf16[(h_base + gid) * KV_LORA + k_col + 8 + 2 * tid]); q_reg[kc][3] = *reinterpret_cast(&ws->q_latent_bf16[(h_base + gid + 8) * KV_LORA + k_col + 8 + 2 * tid]); } uint32_t qr_reg[4]; { int r_col = sub_w * 16; qr_reg[0] = *reinterpret_cast(&ws->q_rope_bf16[(h_base + gid) * QK_ROPE + r_col + 2 * tid]); qr_reg[1] = *reinterpret_cast(&ws->q_rope_bf16[(h_base + gid + 8) * QK_ROPE + r_col + 2 * tid]); qr_reg[2] = *reinterpret_cast(&ws->q_rope_bf16[(h_base + gid) * QK_ROPE + r_col + 8 + 2 * tid]); qr_reg[3] = *reinterpret_cast(&ws->q_rope_bf16[(h_base + gid + 8) * QK_ROPE + r_col + 8 + 2 * tid]); } const uint64_t evict_policy = make_evict_first_policy(); auto issue_tile = [&](int stage, int l_base) { int valid_t = min(16, l_end - l_base); #pragma unroll for (int p = 0; p < 4; ++p) { int idx = p * 256 + threadIdx.x; int t = idx >> 6; int c8 = (idx & 63) * 8; bool ok = (t < valid_t); const void* gptr = ¶ms.c_kv_out[size_t(ok ? (l_base + t) : 0) * KV_LORA + c8]; cp_async_16_evict_first(&smem.mla_tc.ckv[stage][t][c8], gptr, ok, evict_policy); } if (threadIdx.x < 128) { int t = threadIdx.x >> 3; int r8 = (threadIdx.x & 7) * 8; bool ok = (t < valid_t); const void* gptr = ¶ms.k_rope_out[size_t(ok ? (l_base + t) : 0) * QK_ROPE + r8]; cp_async_16_evict_first(&smem.mla_tc.krp[stage][t][r8], gptr, ok, evict_policy); } cp_async_commit(); }; issue_tile(0, l_start); for (int iter = 0; iter < n_iters; ++iter) { int stage = iter & 1; int l_base = l_start + iter * 16; int valid_t = min(16, l_end - l_base); if (iter + 1 < n_iters) { issue_tile(stage ^ 1, l_base + 16); cp_async_wait<1>(); } else { cp_async_wait<0>(); } __syncthreads(); float s0[4] = {0.f, 0.f, 0.f, 0.f}; float s1[4] = {0.f, 0.f, 0.f, 0.f}; #pragma unroll for (int kc = 0; kc < 8; ++kc) { int k_col = (sub_w * 8 + kc) * 16; uint32_t b0_0 = *reinterpret_cast(&smem.mla_tc.ckv[stage][gid][k_col + 2 * tid]); uint32_t b1_0 = *reinterpret_cast(&smem.mla_tc.ckv[stage][gid][k_col + 8 + 2 * tid]); mma_m16n8k16_bf16(s0[0], s0[1], s0[2], s0[3], q_reg[kc][0], q_reg[kc][1], q_reg[kc][2], q_reg[kc][3], b0_0, b1_0); uint32_t b0_1 = *reinterpret_cast(&smem.mla_tc.ckv[stage][8 + gid][k_col + 2 * tid]); uint32_t b1_1 = *reinterpret_cast(&smem.mla_tc.ckv[stage][8 + gid][k_col + 8 + 2 * tid]); mma_m16n8k16_bf16(s1[0], s1[1], s1[2], s1[3], q_reg[kc][0], q_reg[kc][1], q_reg[kc][2], q_reg[kc][3], b0_1, b1_1); } { int r_col = sub_w * 16; uint32_t b0_0 = *reinterpret_cast(&smem.mla_tc.krp[stage][gid][r_col + 2 * tid]); uint32_t b1_0 = *reinterpret_cast(&smem.mla_tc.krp[stage][gid][r_col + 8 + 2 * tid]); mma_m16n8k16_bf16(s0[0], s0[1], s0[2], s0[3], qr_reg[0], qr_reg[1], qr_reg[2], qr_reg[3], b0_0, b1_0); uint32_t b0_1 = *reinterpret_cast(&smem.mla_tc.krp[stage][8 + gid][r_col + 2 * tid]); uint32_t b1_1 = *reinterpret_cast(&smem.mla_tc.krp[stage][8 + gid][r_col + 8 + 2 * tid]); mma_m16n8k16_bf16(s1[0], s1[1], s1[2], s1[3], qr_reg[0], qr_reg[1], qr_reg[2], qr_reg[3], b0_1, b1_1); } *reinterpret_cast(&smem.mla_tc.s_part[sub_w][h_base + gid][2 * tid]) = make_float2(s0[0], s0[1]); *reinterpret_cast(&smem.mla_tc.s_part[sub_w][h_base + gid + 8][2 * tid]) = make_float2(s0[2], s0[3]); *reinterpret_cast(&smem.mla_tc.s_part[sub_w][h_base + gid][8 + 2 * tid]) = make_float2(s1[0], s1[1]); *reinterpret_cast(&smem.mla_tc.s_part[sub_w][h_base + gid + 8][8 + 2 * tid]) = make_float2(s1[2], s1[3]); __syncthreads(); #pragma unroll for (int hp = 0; hp < 2; ++hp) { int h = wid * 4 + hp * 2 + (lane >> 4); int t = lane & 15; float sc = (t < valid_t) ? (smem.mla_tc.s_part[0][h][t] + smem.mla_tc.s_part[1][h][t] + smem.mla_tc.s_part[2][h][t] + smem.mla_tc.s_part[3][h][t]) : -1.0e30f; float tmax = sc; #pragma unroll for (int off = 8; off > 0; off >>= 1) { tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffff, tmax, off)); } float m_old = smem.mla_tc.s_m[h]; float m_new = fmaxf(m_old, tmax); float e_prev = (m_old == -1.0e30f) ? 0.0f : __expf(m_old - m_new); float p = (t < valid_t) ? __expf(sc - m_new) : 0.0f; float psum = p; #pragma unroll for (int off = 8; off > 0; off >>= 1) { psum += __shfl_xor_sync(0xffffffff, psum, off); } smem.mla_tc.s_p[h][t] = __float2bfloat16_rn(p); if (t == 0) { smem.mla_tc.s_escale[h] = e_prev; smem.mla_tc.s_m[h] = m_new; smem.mla_tc.s_l[h] = smem.mla_tc.s_l[h] * e_prev + psum; } } __syncthreads(); float e_top = smem.mla_tc.s_escale[h_base + gid]; float e_bot = smem.mla_tc.s_escale[h_base + gid + 8]; uint32_t pa0 = *reinterpret_cast(&smem.mla_tc.s_p[h_base + gid][2 * tid]); uint32_t pa1 = *reinterpret_cast(&smem.mla_tc.s_p[h_base + gid + 8][2 * tid]); uint32_t pa2 = *reinterpret_cast(&smem.mla_tc.s_p[h_base + gid][8 + 2 * tid]); uint32_t pa3 = *reinterpret_cast(&smem.mla_tc.s_p[h_base + gid + 8][8 + 2 * tid]); int c_sub = sub_w * 128; const __nv_bfloat16* ckv_row = &smem.mla_tc.ckv[stage][lane & 15][c_sub]; #pragma unroll for (int n = 0; n < 8; ++n) { o_acc[n][0] *= e_top; o_acc[n][1] *= e_top; o_acc[n][2] *= e_bot; o_acc[n][3] *= e_bot; uint32_t vb0, vb1; ldmatrix_x2_trans(vb0, vb1, ckv_row + n * 8); mma_m16n8k16_bf16(o_acc[n][0], o_acc[n][1], o_acc[n][2], o_acc[n][3], pa0, pa1, pa2, pa3, vb0, vb1); } asm volatile("" ::: "memory"); #pragma unroll for (int n = 8; n < 16; ++n) { o_acc[n][0] *= e_top; o_acc[n][1] *= e_top; o_acc[n][2] *= e_bot; o_acc[n][3] *= e_bot; uint32_t vb0, vb1; ldmatrix_x2_trans(vb0, vb1, ckv_row + n * 8); mma_m16n8k16_bf16(o_acc[n][0], o_acc[n][1], o_acc[n][2], o_acc[n][3], pa0, pa1, pa2, pa3, vb0, vb1); } } if (threadIdx.x < 32) { int h = threadIdx.x; ws->attn_m[h * KV_SPLIT_STRIDE + split_id] = smem.mla_tc.s_m[h]; ws->attn_l[h * KV_SPLIT_STRIDE + split_id] = smem.mla_tc.s_l[h]; } float inv_l_top = 1.0f / fmaxf(smem.mla_tc.s_l[h_base + gid], 1.0e-20f); float inv_l_bot = 1.0f / fmaxf(smem.mla_tc.s_l[h_base + gid + 8], 1.0e-20f); int c_sub = sub_w * 128; __syncthreads(); #pragma unroll for (int n = 0; n < 16; ++n) { int col = c_sub + n * 8 + 2 * tid; __nv_bfloat162 v_top = __floats2bfloat162_rn(o_acc[n][0] * inv_l_top, o_acc[n][1] * inv_l_top); __nv_bfloat162 v_bot = __floats2bfloat162_rn(o_acc[n][2] * inv_l_bot, o_acc[n][3] * inv_l_bot); *reinterpret_cast<__nv_bfloat162*>(&smem.mla_tc.ckv[h_half][gid][col]) = v_top; *reinterpret_cast<__nv_bfloat162*>(&smem.mla_tc.ckv[h_half][gid + 8][col]) = v_bot; } __syncthreads(); #pragma unroll for (int p = 0; p < 8; ++p) { int idx = p * 256 + threadIdx.x; int h = idx >> 6; int c8 = (idx & 63) * 8; uint4 v = *reinterpret_cast(&smem.mla_tc.ckv[h >> 4][h & 15][c8]); store_u4_streaming(&ws->attn_o_splits[(split_id * 32 + h) * KV_LORA + c8], v); } } grid.sync(); // Stage A4: 256-CTA Coalesced Streaming Split-KV Combine (32 heads * 8 tiles of 64 = 256 CTAs!) if (blockIdx.x < 256) { int h = blockIdx.x >> 3; int c_base = (blockIdx.x & 7) * 64; if (wid == 0) { const float* m_row = &ws->attn_m[h * KV_SPLIT_STRIDE]; const float* l_row = &ws->attn_l[h * KV_SPLIT_STRIDE]; float g_max = -1.0e30f; for (int s = lane; s < num_splits; s += 32) { if (l_row[s] > 0.0f) { g_max = fmaxf(g_max, m_row[s]); } } #pragma unroll for (int off = 16; off > 0; off >>= 1) { g_max = fmaxf(g_max, __shfl_xor_sync(0xffffffff, g_max, off)); } float g_sum = 0.0f; for (int s = lane; s < num_splits; s += 32) { float ls = l_row[s]; float w = (ls > 0.0f) ? (ls * __expf(m_row[s] - g_max)) : 0.0f; smem.mla_comb.scale_s[s] = w; g_sum += w; } g_sum = warp_sum(g_sum); if (lane == 0) smem.mla_comb.g_inv_sum = 1.0f / fmaxf(g_sum, 1.0e-20f); } __syncthreads(); float2 acc = {0.f, 0.f}; int col2 = c_base + lane * 2; for (int s = wid; s < num_splits; s += 8) { float w = smem.mla_comb.scale_s[s]; if (w > 0.0f) { uint32_t bv = load_u32_streaming(&ws->attn_o_splits[(s * 32 + h) * KV_LORA + col2]); float2 f01 = __bfloat1622float2(*reinterpret_cast(&bv)); acc.x += w * f01.x; acc.y += w * f01.y; } } *reinterpret_cast(&smem.mla_comb.partial[wid][lane * 2]) = acc; __syncthreads(); if (threadIdx.x < 64) { int cl = threadIdx.x; float tot = 0.0f; #pragma unroll for (int w = 0; w < 8; ++w) tot += smem.mla_comb.partial[w][cl]; ws->o_latent[h * KV_LORA + c_base + cl] = tot * smem.mla_comb.g_inv_sum; } } grid.sync(); // Stage A5: 1-Wave TILE_N=128 Absorbed o = o_latent @ W_uv (32 heads * 32 chunks of 16 = 1,024 warps, 4-way split!) if (global_warp < 32 * 32) { int h = global_warp & 31; // h innermost -> contiguous 4096B row across 32 warps! int sg = global_warp >> 5; // 0..31 (8 kb rows = 16 c's per sixteenth-group!) int g = sg >> 3; // 0..3 const float* ol = &ws->o_latent[h * KV_LORA]; float xsum = 0.0f; if (lane < 16) xsum = ol[sg * 16 + lane]; #pragma unroll for (int off = 8; off > 0; off >>= 1) xsum += __shfl_xor_sync(0xffffffff, xsum, off); xsum = __shfl_sync(0xffffffff, xsum, 0); int col_grp = lane & 15; int row_sub = lane >> 4; int local_col = col_grp * 8; int col_base = h * 256 + 128 + local_col; float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; int kb_base = sg * 8 + row_sub; const uint64_t pol_last = make_evict_last_policy(); #pragma unroll for (int step = 0; step < 4; ++step) { int kb = kb_base + step * 2; uint2 w_vec = load_u2_policy(&mla.kv_b.w_q[kb * 8192 + col_base], pol_last); float x_lo = ol[kb * 2]; float x_hi = ol[kb * 2 + 1]; uint32_t words[2] = {w_vec.x, w_vec.y}; #pragma unroll for (int w = 0; w < 2; ++w) { uint32_t v = words[w]; uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u; uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u; uint32_t v_shr8 = v >> 8; uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u; uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u; float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u)); float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u)); float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u)); float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u)); g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x; g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x; g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y; g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y; } } #pragma unroll for (int c = 0; c < 8; ++c) { g_acc[c] += __shfl_down_sync(0xffffffff, g_acc[c], 16); } if (row_sub == 0) { uint4 s_vec = load_u4_policy(&mla.kv_b.scales[g * 8192 + col_base], pol_last); uint4 z_vec = load_u4_policy(&mla.kv_b.zeros[g * 8192 + col_base], pol_last); const __nv_bfloat16* sb = reinterpret_cast(&s_vec); const __nv_bfloat16* zb = reinterpret_cast(&z_vec); float* op = &ws->mla_o_raw[sg & 3][h * V_HEAD + local_col]; #pragma unroll for (int c = 0; c < 8; ++c) { float z_eff = __bfloat162float(zb[c]) + 128.0f; float val = __bfloat162float(sb[c]) * (g_acc[c] - z_eff * xsum); atomicAdd(&op[c], val); } } } grid.sync(); // Stage A6: CTAs 0..143 run QGPW=2 Row-Contiguous o_proj! if (blockIdx.x < 144) { #pragma unroll for (int pass = 0; pass < 4; ++pass) { int base = (pass * 256 + threadIdx.x) * 4; float4 o0 = *reinterpret_cast(&ws->mla_o_raw[0][base]); float4 o1 = *reinterpret_cast(&ws->mla_o_raw[1][base]); float4 o2 = *reinterpret_cast(&ws->mla_o_raw[2][base]); float4 o3 = *reinterpret_cast(&ws->mla_o_raw[3][base]); smem.gemv.sx[base + 0] = __bfloat162float(__float2bfloat16_rn(o0.x + o1.x + o2.x + o3.x)); smem.gemv.sx[base + 1] = __bfloat162float(__float2bfloat16_rn(o0.y + o1.y + o2.y + o3.y)); smem.gemv.sx[base + 2] = __bfloat162float(__float2bfloat16_rn(o0.z + o1.z + o2.z + o3.z)); smem.gemv.sx[base + 3] = __bfloat162float(__float2bfloat16_rn(o0.w + o1.w + o2.w + o3.w)); } __syncthreads(); for (int qg = wid; qg < (4096 / QGROUP); qg += 8) { float s = warp_sum(smem.gemv.sx[qg * 32 + lane]); if (lane == 0) smem.gemv.sx_qgsum[qg] = s; } __syncthreads(); int tn = global_warp % 18; int kc = global_warp / 18; // 0..63 warp_int4_gemv_128<2, false>( smem.gemv.sx, smem.gemv.sx_qgsum, mla.o_proj.w_q, mla.o_proj.scales, mla.o_proj.zeros, &ws->attn_o_accum[layer_idx][kc & 3][tn * 128], HIDDEN, tn * 128, kc * 2 ); } grid.sync(); } __global__ __launch_bounds__(BLOCK_THREADS, 2) void kimi_decode_megakernel(const __grid_constant__ MegakernelParams params) { Workspace* ws = reinterpret_cast(params.workspace); FastGridBarrier grid{ws->grid_bar}; extern __shared__ char dyn_smem[]; SharedStorage& smem = *reinterpret_cast(dyn_smem); const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x; const int total_threads = gridDim.x * BLOCK_THREADS; constexpr size_t ZERO_FLOATS = offsetof(Workspace, h) / sizeof(float); float4* ws_zero_f4 = reinterpret_cast(ws); for (size_t i = global_tid; i < ZERO_FLOATS / 4; i += total_threads) { ws_zero_f4[i] = make_float4(0.f, 0.f, 0.f, 0.f); } if (global_tid < HIDDEN / 8) { int i = global_tid * 8; *reinterpret_cast(&ws->h[i]) = *reinterpret_cast(¶ms.hidden_in[i]); } // Streaming copy on step 0 prevents evicting the 105 MB dense weights from L2 cache! if (params.c_kv_in != params.c_kv_out && params.pos > 0) { size_t n_ckv = size_t(params.pos) * KV_LORA; const uint4* src_c = reinterpret_cast(params.c_kv_in); uint4* dst_c = reinterpret_cast(params.c_kv_out); for (size_t i = global_tid; i < n_ckv / 8; i += total_threads) { store_u4_streaming(&dst_c[i], load_u4_streaming(&src_c[i])); } } if (params.k_rope_in != params.k_rope_out && params.pos > 0) { size_t n_kr = size_t(params.pos) * QK_ROPE; const uint4* src_k = reinterpret_cast(params.k_rope_in); uint4* dst_k = reinterpret_cast(params.k_rope_out); for (size_t i = global_tid; i < n_kr / 8; i += total_threads) { store_u4_streaming(&dst_k[i], load_u4_streaming(&src_k[i])); } } grid.sync(); run_kda_layer(grid, params, ws, smem, 0); run_moe_block(grid, params, ws, smem, 0, false); run_kda_layer(grid, params, ws, smem, 1); run_moe_block(grid, params, ws, smem, 1, false); run_kda_layer(grid, params, ws, smem, 2); run_moe_block(grid, params, ws, smem, 2, false); run_mla_layer(grid, params, ws, smem, 3); run_moe_block(grid, params, ws, smem, 3, true); } extern "C" size_t kimi_workspace_bytes() { return sizeof(Workspace) + 4096; } extern "C" size_t kimi_params_bytes() { return sizeof(MegakernelParams); } extern "C" int kimi_build_base_params( const uint64_t* weight_ptrs, float routed_scaling, void* out_params, int device_id ) { cudaSetDevice(device_id); int smem_bytes = sizeof(SharedStorage); cudaFuncSetAttribute(kimi_decode_megakernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes); int blocks_per_sm = 0; cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks_per_sm, kimi_decode_megakernel, BLOCK_THREADS, smem_bytes); cudaDeviceProp prop; cudaGetDeviceProperties(&prop, device_id); int max_blocks = blocks_per_sm * prop.multiProcessorCount; int target = 2 * prop.multiProcessorCount; // 376 blocks int num_blocks = (max_blocks < target) ? max_blocks : target; MegakernelParams p = {}; int idx = 0; for (int i = 0; i < 4; ++i) p.attn_norm[i] = reinterpret_cast(weight_ptrs[idx++]); for (int i = 0; i < 4; ++i) p.moe_norm[i] = reinterpret_cast(weight_ptrs[idx++]); auto read_ql = [&](QuantLinearPtrs& ql) { ql.w_q = reinterpret_cast(weight_ptrs[idx++]); ql.scales = reinterpret_cast(weight_ptrs[idx++]); ql.zeros = reinterpret_cast(weight_ptrs[idx++]); }; for (int k = 0; k < 3; ++k) { read_ql(p.kda[k].q_proj); read_ql(p.kda[k].k_proj); read_ql(p.kda[k].v_proj); read_ql(p.kda[k].g_proj); p.kda[k].beta_w = reinterpret_cast(weight_ptrs[idx++]); p.kda[k].conv_w = reinterpret_cast(weight_ptrs[idx++]); read_ql(p.kda[k].o_proj); } read_ql(p.mla.q_proj); read_ql(p.mla.kv_a); read_ql(p.mla.kv_b); read_ql(p.mla.o_proj); for (int i = 0; i < 4; ++i) { p.moe[i].router_w = reinterpret_cast(weight_ptrs[idx++]); read_ql(p.moe[i].gate); read_ql(p.moe[i].up); read_ql(p.moe[i].down); read_ql(p.moe[i].s_gate); read_ql(p.moe[i].s_up); read_ql(p.moe[i].s_down); } p.routed_scaling = routed_scaling; memcpy(out_params, &p, sizeof(MegakernelParams)); return num_blocks; } extern "C" void launch_kimi_fast( const void* base_params, const uint64_t* state_ptrs, int pos, int num_blocks, cudaStream_t stream ) { MegakernelParams p; memcpy(&p, base_params, sizeof(MegakernelParams)); int sidx = 0; for (int k = 0; k < 3; ++k) { p.kda_st[k].S = reinterpret_cast(state_ptrs[sidx++]); p.kda_st[k].cq = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]); p.kda_st[k].ck = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]); p.kda_st[k].cv = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]); } p.c_kv_in = reinterpret_cast(state_ptrs[sidx++]); p.c_kv_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]); p.k_rope_in = reinterpret_cast(state_ptrs[sidx++]); p.k_rope_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]); p.hidden_in = reinterpret_cast(state_ptrs[sidx++]); p.hidden_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]); p.workspace = reinterpret_cast(state_ptrs[sidx++]); p.pos = pos; kimi_decode_megakernel<<>>(p); } """ _CPP_SRC = r""" #include #include #include #include extern "C" size_t kimi_workspace_bytes(); extern "C" size_t kimi_params_bytes(); extern "C" int kimi_build_base_params( const uint64_t* weight_ptrs, float routed_scaling, void* out_params, int device_id ); extern "C" void launch_kimi_fast( const void* base_params, const uint64_t* state_ptrs, int pos, int num_blocks, cudaStream_t stream ); int64_t get_workspace_bytes() { return static_cast(kimi_workspace_bytes()); } std::tuple build_base_params( torch::Tensor weight_ptrs_cpu, double routed_scaling, int64_t device_id ) { auto buf = torch::empty({static_cast(kimi_params_bytes())}, torch::dtype(torch::kUInt8)); int nb = kimi_build_base_params( reinterpret_cast(weight_ptrs_cpu.data_ptr()), static_cast(routed_scaling), buf.data_ptr(), static_cast(device_id) ); return std::make_tuple(buf, static_cast(nb)); } static inline torch::Tensor grow_cache_2d(const torch::Tensor& tin, int64_t new_len, int64_t cols) { size_t req_bytes = static_cast(new_len * cols) * 2; if (tin.is_contiguous() && tin.storage_offset() == 0 && tin.storage().nbytes() >= req_bytes) { auto out = at::detail::make_tensor( c10::Storage(tin.storage()), tin.key_set(), tin.dtype() ); int64_t sizes[2] = {new_len, cols}; int64_t strides[2] = {cols, 1}; out.unsafeGetTensorImpl()->set_sizes_and_strides( c10::IntArrayRef(sizes, 2), c10::IntArrayRef(strides, 2) ); return out; } auto buf = at::empty({new_len + 64, cols}, tin.options()); return buf.slice(0, 0, new_len); } std::tuple step_fast( torch::Tensor base_params_cpu, torch::Tensor s0_S, torch::Tensor s0_cq, torch::Tensor s0_ck, torch::Tensor s0_cv, torch::Tensor s1_S, torch::Tensor s1_cq, torch::Tensor s1_ck, torch::Tensor s1_cv, torch::Tensor s2_S, torch::Tensor s2_cq, torch::Tensor s2_ck, torch::Tensor s2_cv, torch::Tensor c_kv_in, torch::Tensor k_rope_in, torch::Tensor hidden_in, torch::Tensor workspace, int64_t num_blocks ) { int64_t pos = c_kv_in.size(0); int64_t new_len = pos + 1; torch::Tensor c_kv_out = grow_cache_2d(c_kv_in, new_len, 512); torch::Tensor k_rope_out = grow_cache_2d(k_rope_in, new_len, 64); torch::Tensor hidden_out = at::empty_like(hidden_in); uint64_t state_ptrs[19] = { reinterpret_cast(s0_S.data_ptr()), reinterpret_cast(s0_cq.data_ptr()), reinterpret_cast(s0_ck.data_ptr()), reinterpret_cast(s0_cv.data_ptr()), reinterpret_cast(s1_S.data_ptr()), reinterpret_cast(s1_cq.data_ptr()), reinterpret_cast(s1_ck.data_ptr()), reinterpret_cast(s1_cv.data_ptr()), reinterpret_cast(s2_S.data_ptr()), reinterpret_cast(s2_cq.data_ptr()), reinterpret_cast(s2_ck.data_ptr()), reinterpret_cast(s2_cv.data_ptr()), reinterpret_cast(c_kv_in.data_ptr()), reinterpret_cast(c_kv_out.data_ptr()), reinterpret_cast(k_rope_in.data_ptr()), reinterpret_cast(k_rope_out.data_ptr()), reinterpret_cast(hidden_in.data_ptr()), reinterpret_cast(hidden_out.data_ptr()), reinterpret_cast(workspace.data_ptr()), }; cudaStream_t stream = at::cuda::getCurrentCUDAStream(hidden_in.device().index()).stream(); launch_kimi_fast( base_params_cpu.data_ptr(), state_ptrs, static_cast(pos), static_cast(num_blocks), stream ); return std::make_tuple(hidden_out, c_kv_out, k_rope_out); } """ _EXT = None def _get_ext(): global _EXT if _EXT is None: _EXT = load_inline( name="kimi_linear_megakernel_v9", cpp_sources=_CPP_SRC, cuda_sources=_CUDA_SRC, functions=["get_workspace_bytes", "build_base_params", "step_fast"], no_implicit_headers=True, extra_cuda_cflags=["-O3", "--use_fast_math"], ) return _EXT class Model(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern) self._ws: torch.Tensor | None = None self._base_params: torch.Tensor | None = None self._num_blocks: int = 376 _get_ext() def _apply(self, fn): self._base_params = None self._ws = None return super()._apply(fn) def reset_parameters(self): g = torch.Generator(device="cpu").manual_seed(1234) for mod in self.modules(): if isinstance(mod, (QuantLinear, QuantExperts)): mod.init_random(g) elif isinstance(mod, nn.Linear): nn.init.normal_(mod.weight, 0.0, 0.02, generator=g) elif isinstance(mod, KDA): nn.init.normal_(mod.conv_w, 0.0, 0.1, generator=g) self._base_params = None def _prepare(self, device: torch.device) -> None: ext = _get_ext() if self._ws is None or self._ws.device != device: self._ws = torch.zeros(ext.get_workspace_bytes(), dtype=torch.uint8, device=device) ptrs: list[int] = [] for blk in self.blocks: ptrs.append(blk.attn_norm.data_ptr()) for blk in self.blocks: ptrs.append(blk.moe_norm.data_ptr()) def add_ql(ql): ptrs.append(ql.w_q.data_ptr()) ptrs.append(ql.scales.data_ptr()) ptrs.append(ql.zeros.data_ptr()) for k in range(3): attn: KDA = self.blocks[k].attn add_ql(attn.q_proj) add_ql(attn.k_proj) add_ql(attn.v_proj) add_ql(attn.g_proj) ptrs.append(attn.beta_proj.weight.data_ptr()) ptrs.append(attn.conv_w.data_ptr()) add_ql(attn.o_proj) mla: MLA = self.blocks[3].attn add_ql(mla.q_proj) add_ql(mla.kv_a) add_ql(mla.kv_b) add_ql(mla.o_proj) for i in range(4): moe: MoE = self.blocks[i].moe ptrs.append(moe.router.weight.data_ptr()) add_ql(moe.gate) add_ql(moe.up) add_ql(moe.down) add_ql(moe.s_gate) add_ql(moe.s_up) add_ql(moe.s_down) dev_idx = device.index if device.index is not None else 0 w_t = torch.tensor(ptrs, dtype=torch.int64, device="cpu").contiguous() self._base_params, nb = ext.build_base_params(w_t, self.cfg.routed_scaling, dev_idx) self._num_blocks = int(nb) def step(self, hidden: torch.Tensor, state: list): if self._base_params is None: self._prepare(hidden.device) s0, s1, s2, s3 = state[0], state[1], state[2], state[3] hidden_out, c_kv_out, k_rope_out = _EXT.step_fast( self._base_params, s0["S"], s0["cq"], s0["ck"], s0["cv"], s1["S"], s1["cq"], s1["ck"], s1["cv"], s2["S"], s2["cq"], s2["ck"], s2["cv"], s3["c_kv"], s3["k_rope"], hidden, self._ws, self._num_blocks, ) s3["c_kv"] = c_kv_out s3["k_rope"] = k_rope_out return hidden_out, state def init_state(cfg: Config, context_len: int, seed: int) -> list: dev = torch.device("cuda:0") g = torch.Generator(device=dev).manual_seed(seed) H, Dk = cfg.kda_heads, cfg.kda_head_dim C = H * Dk state = [] for kind in cfg.pattern: if kind == "K": state.append({ "S": torch.randn(H, Dk, Dk, device=dev, generator=g) * 0.05, "cq": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1, "ck": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1, "cv": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1, }) else: state.append({ "c_kv": torch.randn(context_len, cfg.kv_lora, device=dev, generator=g, dtype=cfg.dtype) * 0.1, "k_rope": torch.randn(context_len, cfg.qk_rope, device=dev, generator=g, dtype=cfg.dtype) * 0.1, }) return state def init_token(cfg: Config, seed: int) -> torch.Tensor: dev = torch.device("cuda:0") g = torch.Generator(device=dev).manual_seed(seed + 1) return torch.randn(cfg.hidden, device=dev, generator=g, dtype=cfg.dtype) * 0.25