"""Single-kernel W4A16 hybrid decode megakernel for Kimi-Linear (3x KDA + 1x MLA + 4x MoE) on SM120 Blackwell.""" from __future__ import annotations from dataclasses import dataclass, field import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline OP_TYPE = "kimi_linear_w4a16_decode" HARDWARE_REQUIRED = ["RTX_PRO_6000"] EPS = 1.0e-6 GROUP_SIZE = 128 @dataclass(frozen=True) class Config: hidden: int = 2304 kda_heads: int = 32 kda_head_dim: int = 128 short_conv: int = 4 mla_heads: int = 32 kv_lora: int = 512 qk_nope: int = 128 qk_rope: int = 64 v_head: int = 128 rope_theta: float = 10000.0 n_experts: int = 64 n_active: int = 8 n_shared: int = 1 moe_inter: int = 1024 routed_scaling: float = 2.446 group: int = 128 pattern: tuple = ("K", "K", "K", "M") dtype: torch.dtype = field(default=torch.bfloat16) def build_config(shape: dict) -> Config: return Config(n_experts=int(shape.get("n_experts", 64))) def _pack_int4(w_q: torch.Tensor) -> torch.Tensor: lo = w_q[0::2] & 0xF hi = w_q[1::2] & 0xF return (lo | (hi << 4)).contiguous() def quantize(w_io: torch.Tensor, group: int = GROUP_SIZE): K, N = w_io.shape ng = K // group wg = w_io.view(ng, group, N).float() wmin = wg.min(dim=1, keepdim=True).values wmax = wg.max(dim=1, keepdim=True).values scales = (wmax - wmin).clamp_min(1e-8) / 15.0 zeros = (-wmin / scales).round().clamp(0, 15) w_q = ((wg / scales) + zeros).round().clamp(0, 15).to(torch.uint8).view(K, N) return _pack_int4(w_q), scales.squeeze(1).to(torch.bfloat16), zeros.squeeze(1).to(torch.bfloat16) class QuantLinear(nn.Module): def __init__(self, in_f: int, out_f: int, group: int = GROUP_SIZE): super().__init__() assert in_f % group == 0 and in_f % 2 == 0 self.in_f, self.out_f, self.group = in_f, out_f, group ng = in_f // group self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8)) self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16)) self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16)) class QuantExperts(nn.Module): def __init__(self, n: int, in_f: int, out_f: int, group: int = GROUP_SIZE): super().__init__() self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group ng = in_f // group self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8)) self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16)) self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16)) class KDA(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden self.q_proj = QuantLinear(d, H * Dk, cfg.group) self.k_proj = QuantLinear(d, H * Dk, cfg.group) self.v_proj = QuantLinear(d, H * Dk, cfg.group) self.g_proj = QuantLinear(d, H * Dk, cfg.group) self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype) self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype)) self.o_proj = QuantLinear(H * Dk, d, cfg.group) self.scale = Dk ** -0.5 class MLA(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg H, d = cfg.mla_heads, cfg.hidden self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group) self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group) self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group) self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group) self.scale = (cfg.qk_nope + cfg.qk_rope) ** -0.5 class MoE(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype) self.gate = QuantExperts(E, d, m, cfg.group) self.up = QuantExperts(E, d, m, cfg.group) self.down = QuantExperts(E, m, d, cfg.group) self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group) self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group) self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group) class Block(nn.Module): def __init__(self, cfg: Config, kind: str): super().__init__() self.kind = kind self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype)) self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype)) self.attn = KDA(cfg) if kind == "K" else MLA(cfg) self.moe = MoE(cfg) # --------------------------------------------------------------------------- # # Single-Kernel CUDA Megakernel V11 (compiled via load_inline): # - 188 x 512 cooperative grid (16 warps/SM, 128 regs, 0 stack, 0 spills, 0 MATCH.ANY) # - 80 MiB L2 persisting pool (cudaLimitPersistingL2CacheSize) + adaptive Step-0/steady L2 policy # - Native SM120 centered-int4 BF16x2 FMA2 GEMV with L2::256B prefetch hints # - Input-first overlapped o_proj (12-13 rows/SM, zero extra atomics) # - PRE=2 software-pipelined KDA qkvg + overlapped coalesced conv preload on warps 12..14 # - MoE fused router + shared gate/up + PTX release/acquire barrier + 4-expert pre-issued M3 down # - Spin-free MLA A1+A2 (nibble-transposed int4 W_uk), async cp.async double-buffered Step-0 KV copy, # exp2f FlashMLA A3 (64/96/128 splits + streaming part_o), and slim overlapped A4 # --------------------------------------------------------------------------- # _CPP_SRC = r""" #include #include #include #include struct KdaLayerPtrs { const void* attn_norm; const void* qkvg_wq; const void* qkvg_sz; const void* beta_w; const void* conv_w; const void* o_wq; const void* o_sz; }; struct MlaLayerPtrs { const void* attn_norm; const void* qkva_wq; const void* qkva_sz; const void* wuk_wq; const void* wuk_sz; const void* wuv_wq; const void* wuv_sz; const void* o_wq; const void* o_sz; }; struct MoeLayerPtrs { const void* moe_norm; const void* router_w; const void* s_gate_up_wq; const void* s_gate_up_sz; const void* s_down_wq; const void* s_down_sz; const void* r_gate_up_wq; const void* r_gate_up_sz; const void* r_down_wq; const void* r_down_sz; }; struct MegaParams { KdaLayerPtrs kda[3]; MlaLayerPtrs mla; MoeLayerPtrs moe[4]; float* kda_S[3]; void* kda_cq[3]; void* kda_ck[3]; void* kda_cv[3]; const void* src_c_kv; const void* src_k_rope; void* dst_c_kv; void* dst_k_rope; int ctx_len; int need_copy; const void* in_hidden; void* out_hidden; void* g_hidden; float* g_norm_sq; float* g_q; float* g_k; float* g_v; float* g_decay; float* g_beta; void* g_attn_o; // [4096] bf16 (KDA) float* g_attn_o_f32; // [4096] f32 (MLA Phase A4 atomic accum) float* g_q_full_f32; // [32*512] f32 (MLA spin-free W_uk accum) void* g_q_rope; // [32*64] bf16 (MLA q_rope) void* g_mla_part_ml; // [32 * 192] float2 (coalesced across splits!) void* g_mla_part_o; // [188 * 2056 * 8] bf16 (padded stride!) float* g_router_logits; unsigned int* g_router_done; // [4] u32 void* g_moe_hh; // [9*1024] bf16 float routed_scaling; float rope_theta; int n_experts; }; extern "C" void launch_kimi_megakernel( const MegaParams* h_params, void* dense_arena_ptr, size_t dense_arena_bytes, cudaStream_t stream ); struct PackedHandle { MegaParams base; void* dense_arena_ptr; size_t dense_arena_bytes; }; int64_t create_handle( std::vector kda_tensors, std::vector mla_tensors, std::vector moe_tensors, std::vector scratch, torch::Tensor dense_arena, double routed_scaling, double rope_theta, int64_t n_experts ) { auto* h = new PackedHandle(); h->dense_arena_ptr = dense_arena.data_ptr(); h->dense_arena_bytes = static_cast(dense_arena.numel()) * dense_arena.element_size(); for (int i = 0; i < 3; ++i) { h->base.kda[i].attn_norm = kda_tensors[i * 7 + 0].data_ptr(); h->base.kda[i].qkvg_wq = kda_tensors[i * 7 + 1].data_ptr(); h->base.kda[i].qkvg_sz = kda_tensors[i * 7 + 2].data_ptr(); h->base.kda[i].beta_w = kda_tensors[i * 7 + 3].data_ptr(); h->base.kda[i].conv_w = kda_tensors[i * 7 + 4].data_ptr(); h->base.kda[i].o_wq = kda_tensors[i * 7 + 5].data_ptr(); h->base.kda[i].o_sz = kda_tensors[i * 7 + 6].data_ptr(); } h->base.mla.attn_norm = mla_tensors[0].data_ptr(); h->base.mla.qkva_wq = mla_tensors[1].data_ptr(); h->base.mla.qkva_sz = mla_tensors[2].data_ptr(); h->base.mla.wuk_wq = mla_tensors[3].data_ptr(); h->base.mla.wuk_sz = mla_tensors[4].data_ptr(); h->base.mla.wuv_wq = mla_tensors[5].data_ptr(); h->base.mla.wuv_sz = mla_tensors[6].data_ptr(); h->base.mla.o_wq = mla_tensors[7].data_ptr(); h->base.mla.o_sz = mla_tensors[8].data_ptr(); for (int i = 0; i < 4; ++i) { h->base.moe[i].moe_norm = moe_tensors[i * 10 + 0].data_ptr(); h->base.moe[i].router_w = moe_tensors[i * 10 + 1].data_ptr(); h->base.moe[i].s_gate_up_wq = moe_tensors[i * 10 + 2].data_ptr(); h->base.moe[i].s_gate_up_sz = moe_tensors[i * 10 + 3].data_ptr(); h->base.moe[i].s_down_wq = moe_tensors[i * 10 + 4].data_ptr(); h->base.moe[i].s_down_sz = moe_tensors[i * 10 + 5].data_ptr(); h->base.moe[i].r_gate_up_wq = moe_tensors[i * 10 + 6].data_ptr(); h->base.moe[i].r_gate_up_sz = moe_tensors[i * 10 + 7].data_ptr(); h->base.moe[i].r_down_wq = moe_tensors[i * 10 + 8].data_ptr(); h->base.moe[i].r_down_sz = moe_tensors[i * 10 + 9].data_ptr(); } h->base.g_hidden = scratch[0].data_ptr(); h->base.g_norm_sq = scratch[1].data_ptr(); h->base.g_q = scratch[2].data_ptr(); h->base.g_k = scratch[3].data_ptr(); h->base.g_v = scratch[4].data_ptr(); h->base.g_decay = scratch[5].data_ptr(); h->base.g_beta = scratch[6].data_ptr(); h->base.g_attn_o = scratch[7].data_ptr(); h->base.g_attn_o_f32 = scratch[8].data_ptr(); h->base.g_q_full_f32 = scratch[9].data_ptr(); h->base.g_q_rope = scratch[10].data_ptr(); h->base.g_mla_part_ml = scratch[11].data_ptr(); h->base.g_mla_part_o = scratch[12].data_ptr(); h->base.g_router_logits = scratch[13].data_ptr(); h->base.g_router_done = reinterpret_cast(scratch[14].data_ptr()); h->base.g_moe_hh = scratch[15].data_ptr(); h->base.routed_scaling = static_cast(routed_scaling); h->base.rope_theta = static_cast(rope_theta); h->base.n_experts = static_cast(n_experts); return reinterpret_cast(h); } void destroy_handle(int64_t ptr) { if (ptr != 0) delete reinterpret_cast(ptr); } torch::Tensor step_megakernel( int64_t handle_ptr, torch::Tensor hidden, torch::Tensor S0, torch::Tensor cq0, torch::Tensor ck0, torch::Tensor cv0, torch::Tensor S1, torch::Tensor cq1, torch::Tensor ck1, torch::Tensor cv1, torch::Tensor S2, torch::Tensor cq2, torch::Tensor ck2, torch::Tensor cv2, torch::Tensor src_c_kv, torch::Tensor src_k_rope, torch::Tensor dst_c_kv, torch::Tensor dst_k_rope, int64_t ctx_len, int64_t need_copy ) { auto* h = reinterpret_cast(handle_ptr); auto out = torch::empty_like(hidden); MegaParams p = h->base; p.kda_S[0] = S0.data_ptr(); p.kda_cq[0] = cq0.data_ptr(); p.kda_ck[0] = ck0.data_ptr(); p.kda_cv[0] = cv0.data_ptr(); p.kda_S[1] = S1.data_ptr(); p.kda_cq[1] = cq1.data_ptr(); p.kda_ck[1] = ck1.data_ptr(); p.kda_cv[1] = cv1.data_ptr(); p.kda_S[2] = S2.data_ptr(); p.kda_cq[2] = cq2.data_ptr(); p.kda_ck[2] = ck2.data_ptr(); p.kda_cv[2] = cv2.data_ptr(); p.src_c_kv = src_c_kv.data_ptr(); p.src_k_rope = src_k_rope.data_ptr(); p.dst_c_kv = dst_c_kv.data_ptr(); p.dst_k_rope = dst_k_rope.data_ptr(); p.ctx_len = static_cast(ctx_len); p.need_copy = static_cast(need_copy); p.in_hidden = hidden.data_ptr(); p.out_hidden = out.data_ptr(); launch_kimi_megakernel(&p, h->dense_arena_ptr, h->dense_arena_bytes, at::cuda::getCurrentCUDAStream().stream()); return out; } """ _CUDA_SRC = r""" #include #include #include #include #include namespace cg = cooperative_groups; constexpr int HIDDEN = 2304; constexpr int KDA_H = 32; constexpr int KDA_C = 4096; constexpr int KV_LORA = 512; constexpr int QK_ROPE = 64; constexpr int V_HEAD = 128; constexpr int MOE_INTER = 1024; constexpr int N_ACTIVE = 8; constexpr int BLOCK_THREADS = 512; constexpr int SMEM_BYTES = 86016; // 84 KB constexpr int X_U4_STRIDE = 5; // 4 uint4s (32 bf16s as 16 bf16x2) + 1 uint4 pad -> 0 bank conflicts! constexpr int X_F32_STRIDE = 36; // Used only for MLA W_uv (4*36 floats = 576 B) constexpr int LAT_STRIDE = 132; // Padded 128+4 for MLA A4 smem_part_lat -> 0 bank conflicts! constexpr int O_SMEM_STRIDE = 520; // Padded 512+8 bf16 (65 uint4s) for FlashMLA smem_O -> 0 bank conflicts! constexpr int ML_STRIDE = 192; // Coalesced stride for float2 g_mla_part_ml[MLA_H][ML_STRIDE] constexpr int PART_O_STRIDE_U4 = 2056; // Padded 32*64 + 8 uint4 (32896 B) -> breaks 32 KB L2 set aliasing! struct KdaLayerPtrs { const void* attn_norm; const void* qkvg_wq; const void* qkvg_sz; const void* beta_w; const void* conv_w; const void* o_wq; const void* o_sz; }; struct MlaLayerPtrs { const void* attn_norm; const void* qkva_wq; const void* qkva_sz; const void* wuk_wq; const void* wuk_sz; const void* wuv_wq; const void* wuv_sz; const void* o_wq; const void* o_sz; }; struct MoeLayerPtrs { const void* moe_norm; const void* router_w; const void* s_gate_up_wq; const void* s_gate_up_sz; const void* s_down_wq; const void* s_down_sz; const void* r_gate_up_wq; const void* r_gate_up_sz; const void* r_down_wq; const void* r_down_sz; }; struct MegaParams { KdaLayerPtrs kda[3]; MlaLayerPtrs mla; MoeLayerPtrs moe[4]; float* kda_S[3]; void* kda_cq[3]; void* kda_ck[3]; void* kda_cv[3]; const void* src_c_kv; const void* src_k_rope; void* dst_c_kv; void* dst_k_rope; int ctx_len; int need_copy; const void* in_hidden; void* out_hidden; void* g_hidden; float* g_norm_sq; float* g_q; float* g_k; float* g_v; float* g_decay; float* g_beta; void* g_attn_o; float* g_attn_o_f32; float* g_q_full_f32; void* g_q_rope; void* g_mla_part_ml; void* g_mla_part_o; float* g_router_logits; unsigned int* g_router_done; void* g_moe_hh; float routed_scaling; float rope_theta; int n_experts; }; // --------------------------------------------------------------------------- // Low-level PTX helpers & L2 cache policies // --------------------------------------------------------------------------- __device__ __forceinline__ float bf16_to_f32(__nv_bfloat16 v) { return __bfloat162float(v); } __device__ __forceinline__ __nv_bfloat16 f32_to_bf16(float v) { return __float2bfloat16(v); } __device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) { __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi); return *reinterpret_cast(&v); } __device__ __forceinline__ float silu_f32(float x) { if (x >= 0.0f) return x / (1.0f + __expf(-x)); float e = __expf(x); return x * (e / (1.0f + e)); } __device__ __forceinline__ float sigmoid_f32(float x) { if (x >= 0.0f) return 1.0f / (1.0f + __expf(-x)); float e = __expf(x); return e / (1.0f + e); } __device__ __forceinline__ uint32_t float_to_sortable_u32(float f) { uint32_t u = __float_as_uint(f); uint32_t mask = static_cast(-static_cast(u >> 31)) | 0x80000000u; return u ^ mask; } __device__ __forceinline__ uint4 ld_u4_keep(const uint4* ptr, uint64_t pol) { uint4 v; asm volatile("ld.global.L2::256B.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ uint4 ld_u4_stream(const uint4* ptr, uint64_t pol) { uint4 v; asm volatile("ld.global.L1::evict_first.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ uint4 ld_u4_stream256(const uint4* ptr, uint64_t pol) { uint4 v; asm volatile("ld.global.L1::evict_first.L2::256B.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ uint4 ld_u4_ca(const uint4* ptr) { uint4 v; asm volatile("ld.global.ca.v4.u32 {%0, %1, %2, %3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr)); return v; } __device__ __forceinline__ uint2 ld_u2_keep(const uint2* ptr, uint64_t pol) { uint2 v; asm volatile("ld.global.L2::cache_hint.v2.u32 {%0, %1}, [%2], %3;" : "=r"(v.x), "=r"(v.y) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ void st_u4_stream(uint4* ptr, uint4 v, uint64_t pol) { asm volatile("st.global.cs.L2::cache_hint.v4.u32 [%0], {%1, %2, %3, %4}, %5;" :: "l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w), "l"(pol)); } __device__ __forceinline__ uint32_t ld_u32_keep(const uint32_t* ptr, uint64_t pol) { uint32_t v; asm volatile("ld.global.L2::cache_hint.u32 %0, [%1], %2;" : "=r"(v) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ uint32_t ld_u32_stream(const uint32_t* ptr, uint64_t pol) { uint32_t v; asm volatile("ld.global.L1::evict_first.L2::cache_hint.u32 %0, [%1], %2;" : "=r"(v) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ float ld_f32_keep(const float* ptr, uint64_t pol) { float v; asm volatile("ld.global.L2::cache_hint.f32 %0, [%1], %2;" : "=f"(v) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ void st_f32_keep(float* ptr, float val, uint64_t pol) { asm volatile("st.global.L2::cache_hint.f32 [%0], %1, %2;" :: "l"(ptr), "f"(val), "l"(pol)); } __device__ __forceinline__ void st_f32x2_keep(float2* ptr, float2 val, uint64_t pol) { asm volatile("st.global.L2::cache_hint.v2.f32 [%0], {%1, %2}, %3;" :: "l"(ptr), "f"(val.x), "f"(val.y), "l"(pol)); } __device__ __forceinline__ float2 ld_f32x2_keep(const float2* ptr, uint64_t pol) { float2 v; asm volatile("ld.global.L2::cache_hint.v2.f32 {%0, %1}, [%2], %3;" : "=f"(v.x), "=f"(v.y) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ uint16_t ld_u16_keep(const void* ptr, uint64_t pol) { uint16_t v; asm volatile("ld.global.L2::cache_hint.u16 %0, [%1], %2;" : "=h"(v) : "l"(ptr), "l"(pol)); return v; } __device__ __forceinline__ void st_u16_keep(void* ptr, uint16_t v, uint64_t pol) { asm volatile("st.global.L2::cache_hint.u16 [%0], %1, %2;" :: "l"(ptr), "h"(v), "l"(pol)); } // FP32 biased-128 unpack (for MLA W_uk and W_uv) __device__ __forceinline__ float unpack_biased128(uint32_t v) { uint32_t r; asm("lop3.b32 %0, %1, 0x000F0000, 0x43000000, 0xEA;" : "=r"(r) : "r"(v)); return __uint_as_float(r); } __device__ __forceinline__ float dot8_biased(uint32_t p, const float* xp) { float4 x0 = *reinterpret_cast(xp); float4 x1 = *reinterpret_cast(xp + 4); float acc = x0.x * unpack_biased128(p << 16); acc = fmaf(x0.y, unpack_biased128(p << 12), acc); acc = fmaf(x0.z, unpack_biased128(p << 8), acc); acc = fmaf(x0.w, unpack_biased128(p << 4), acc); acc = fmaf(x1.x, unpack_biased128(p), acc); acc = fmaf(x1.y, unpack_biased128(p >> 4), acc); acc = fmaf(x1.z, unpack_biased128(p >> 8), acc); acc = fmaf(x1.w, unpack_biased128(p >> 12), acc); return acc; } __device__ __forceinline__ float dot32_w4_f32( uint4 w, uint32_t sz, const float* smem_x32, float x_sum32 ) { float dot = dot8_biased(w.x, smem_x32) + dot8_biased(w.y, smem_x32 + 8) + dot8_biased(w.z, smem_x32 + 16) + dot8_biased(w.w, smem_x32 + 24); float scale = __uint_as_float(sz << 16); float zero = __uint_as_float(sz & 0xFFFF0000u); return scale * fmaf(-(zero + 128.0f), x_sum32, dot); } // Native SM120 BF16x2 centered [-8..+7] int4 unpack + FMA2 (8.33 TB/s L2, 0 overflow, 0 DC cancellation!) __device__ __forceinline__ uint32_t unpack_bf2_s8(uint32_t v) { uint32_t b, r; asm("lop3.b32 %0, %1, 0x000F000F, 0x43004300, 0xEA;" : "=r"(b) : "r"(v)); asm("sub.rn.bf16x2 %0, %1, %2;" : "=r"(r) : "r"(b), "r"(0x43084308u)); return r; } __device__ __forceinline__ uint32_t fma_bf2_u32(uint32_t a, uint32_t b, uint32_t c) { uint32_t r; asm("fma.rn.bf16x2 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c)); return r; } __device__ __forceinline__ float dot32_bf2( uint4 w, uint32_t sz, const uint4* sx_u4, float x_sum32 ) { uint32_t acc = 0u; #pragma unroll for (int part = 0; part < 4; ++part) { uint32_t p = (part == 0) ? w.x : ((part == 1) ? w.y : ((part == 2) ? w.z : w.w)); uint4 x4 = sx_u4[part]; acc = fma_bf2_u32(unpack_bf2_s8(p), x4.x, acc); acc = fma_bf2_u32(unpack_bf2_s8(p >> 4), x4.y, acc); acc = fma_bf2_u32(unpack_bf2_s8(p >> 8), x4.z, acc); acc = fma_bf2_u32(unpack_bf2_s8(p >> 12), x4.w, acc); } float dot = __uint_as_float(acc << 16) + __uint_as_float(acc & 0xFFFF0000u); float scale = __uint_as_float(sz << 16); float zero = __uint_as_float(sz & 0xFFFF0000u); return scale * fmaf(8.0f - zero, x_sum32, dot); } __device__ __forceinline__ void dot32_bf2_pair( uint4 wa, uint32_t sza, uint4 wb, uint32_t szb, const uint4* sx_u4, float x_sum32, float& acc_a, float& acc_b ) { uint32_t da = 0u, db = 0u; #pragma unroll for (int part = 0; part < 4; ++part) { uint32_t pa = (part == 0) ? wa.x : ((part == 1) ? wa.y : ((part == 2) ? wa.z : wa.w)); uint32_t pb = (part == 0) ? wb.x : ((part == 1) ? wb.y : ((part == 2) ? wb.z : wb.w)); uint4 x4 = sx_u4[part]; da = fma_bf2_u32(unpack_bf2_s8(pa), x4.x, da); db = fma_bf2_u32(unpack_bf2_s8(pb), x4.x, db); da = fma_bf2_u32(unpack_bf2_s8(pa >> 4), x4.y, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 4), x4.y, db); da = fma_bf2_u32(unpack_bf2_s8(pa >> 8), x4.z, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 8), x4.z, db); da = fma_bf2_u32(unpack_bf2_s8(pa >> 12), x4.w, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 12), x4.w, db); } float daf = __uint_as_float(da << 16) + __uint_as_float(da & 0xFFFF0000u); float dbf = __uint_as_float(db << 16) + __uint_as_float(db & 0xFFFF0000u); float sa = __uint_as_float(sza << 16), za = __uint_as_float(sza & 0xFFFF0000u); float sb = __uint_as_float(szb << 16), zb = __uint_as_float(szb & 0xFFFF0000u); acc_a += sa * fmaf(8.0f - za, x_sum32, daf); acc_b += sb * fmaf(8.0f - zb, x_sum32, dbf); } // Uniform-mask warp/subwarp reductions (100% 0xffffffffu -> ZERO MATCH.ANY!) __device__ __forceinline__ float warp_reduce_sum(float v) { #pragma unroll for (int off = 16; off > 0; off >>= 1) v += __shfl_xor_sync(0xffffffffu, v, off); return v; } __device__ __forceinline__ float subwarp8_reduce_sum(float v) { v += __shfl_xor_sync(0xffffffffu, v, 4); v += __shfl_xor_sync(0xffffffffu, v, 2); v += __shfl_xor_sync(0xffffffffu, v, 1); return v; } __device__ __forceinline__ float subwarp4_reduce_sum(float v) { v += __shfl_xor_sync(0xffffffffu, v, 2); v += __shfl_xor_sync(0xffffffffu, v, 1); return v; } __device__ __forceinline__ float halfwarp16_reduce_max(float v) { v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 8)); v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 4)); v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 2)); v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 1)); return v; } __device__ __forceinline__ float halfwarp16_reduce_sum(float v) { v += __shfl_xor_sync(0xffffffffu, v, 8); v += __shfl_xor_sync(0xffffffffu, v, 4); v += __shfl_xor_sync(0xffffffffu, v, 2); v += __shfl_xor_sync(0xffffffffu, v, 1); return v; } __device__ __forceinline__ void cp_async_cg_16_pol(void* smem_ptr, const void* glob_ptr, uint64_t pol) { uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile("cp.async.cg.shared.global.L2::256B.L2::cache_hint [%0], [%1], 16, %2;\n" :: "r"(smem_addr), "l"(glob_ptr), "l"(pol)); } __device__ __forceinline__ void cp_async_commit() { asm volatile("cp.async.commit_group;\n" ::); } template __device__ __forceinline__ void cp_async_wait() { asm volatile("cp.async.wait_group %0;\n" :: "n"(N)); } __device__ __forceinline__ void ldmatrix_x4(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr) { uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile("ldmatrix.sync.aligned.x4.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr)); } __device__ __forceinline__ void ldmatrix_x2(uint32_t& r0, uint32_t& r1, const void* smem_ptr) { uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile("ldmatrix.sync.aligned.x2.m8n8.shared.b16 {%0, %1}, [%2];\n" : "=r"(r0), "=r"(r1) : "r"(addr)); } __device__ __forceinline__ void ldmatrix_x4_trans(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr) { uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile("ldmatrix.sync.aligned.x4.trans.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr)); } __device__ __forceinline__ void mma_m16n8k16_bf16( float& d0, float& d1, float& d2, float& d3, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1 ) { asm volatile( "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n" : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1) ); } // --------------------------------------------------------------------------- // Vectorized BF16x2 SMEM activation loaders // --------------------------------------------------------------------------- __device__ __forceinline__ uint4 pack_and_store_rmsnorm8( int q, uint4 h8, uint4 w8, float inv_rms, uint4* smem_x_u4, float* smem_x_sum32 ) { float r0 = __uint_as_float(h8.x << 16) * inv_rms * __uint_as_float(w8.x << 16); float r1 = __uint_as_float(h8.x & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.x & 0xFFFF0000u); float r2 = __uint_as_float(h8.y << 16) * inv_rms * __uint_as_float(w8.y << 16); float r3 = __uint_as_float(h8.y & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.y & 0xFFFF0000u); float r4 = __uint_as_float(h8.z << 16) * inv_rms * __uint_as_float(w8.z << 16); float r5 = __uint_as_float(h8.z & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.z & 0xFFFF0000u); float r6 = __uint_as_float(h8.w << 16) * inv_rms * __uint_as_float(w8.w << 16); float r7 = __uint_as_float(h8.w & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.w & 0xFFFF0000u); uint32_t p0 = pack_bf16x2(r0, r4); uint32_t p1 = pack_bf16x2(r1, r5); uint32_t p2 = pack_bf16x2(r2, r6); uint32_t p3 = pack_bf16x2(r3, r7); int c = q >> 2; uint4 x4 = make_uint4(p0, p1, p2, p3); smem_x_u4[c * X_U4_STRIDE + (q & 3)] = x4; float s = (__uint_as_float(p0 << 16) + __uint_as_float(p0 & 0xFFFF0000u)) + (__uint_as_float(p1 << 16) + __uint_as_float(p1 & 0xFFFF0000u)) + (__uint_as_float(p2 << 16) + __uint_as_float(p2 & 0xFFFF0000u)) + (__uint_as_float(p3 << 16) + __uint_as_float(p3 & 0xFFFF0000u)); s = subwarp4_reduce_sum(s); if ((threadIdx.x & 3) == 0) smem_x_sum32[c] = s; return x4; } __device__ __forceinline__ void load_rmsnorm_2304_to_smem_bf2( const __nv_bfloat16* g_in, const __nv_bfloat16* norm_w, float inv_rms, uint4* smem_x_u4, float* smem_x_sum32, int bid, uint64_t pol_keep ) { int tid = threadIdx.x; if (tid < 288) { int shift = (bid % 72) * 4; int q = tid + shift; if (q >= 288) q -= 288; uint4 h8 = ld_u4_ca(reinterpret_cast(g_in) + q); uint4 w8 = ld_u4_keep(reinterpret_cast(norm_w) + q, pol_keep); pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32); } __syncthreads(); } __device__ __forceinline__ void load_input_and_rmsnorm_2304_to_smem_bf2( const __nv_bfloat16* in_hidden, __nv_bfloat16* g_hidden, const __nv_bfloat16* norm_w, uint4* smem_x_u4, float* smem_x_sum32, float* smem_red16, int bid, uint64_t pol_keep ) { int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; uint4 h8 = make_uint4(0, 0, 0, 0); uint4 w8 = make_uint4(0, 0, 0, 0); int q = 0; float local_sq = 0.0f; if (tid < 288) { int shift = (bid % 72) * 4; q = tid + shift; if (q >= 288) q -= 288; h8 = ld_u4_ca(reinterpret_cast(in_hidden) + q); w8 = ld_u4_keep(reinterpret_cast(norm_w) + q, pol_keep); if (bid == 0) reinterpret_cast(g_hidden)[q] = h8; float h0 = __uint_as_float(h8.x << 16), h1 = __uint_as_float(h8.x & 0xFFFF0000u); float h2 = __uint_as_float(h8.y << 16), h3 = __uint_as_float(h8.y & 0xFFFF0000u); float h4 = __uint_as_float(h8.z << 16), h5 = __uint_as_float(h8.z & 0xFFFF0000u); float h6 = __uint_as_float(h8.w << 16), h7 = __uint_as_float(h8.w & 0xFFFF0000u); local_sq = h0*h0 + h1*h1 + h2*h2 + h3*h3 + h4*h4 + h5*h5 + h6*h6 + h7*h7; } local_sq = warp_reduce_sum(local_sq); if (lane == 0) smem_red16[wid] = local_sq; __syncthreads(); if (wid == 0) { float tot = (lane < 9) ? smem_red16[lane] : 0.0f; tot = warp_reduce_sum(tot); if (lane == 0) smem_red16[0] = rsqrtf(tot * (1.0f / float(HIDDEN)) + 1.0e-6f); } __syncthreads(); float inv_rms = smem_red16[0]; if (tid < 288) { pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32); } __syncthreads(); } __device__ __forceinline__ void store_bf16_u4_to_smem_bf2( int q, uint4 v, uint4* smem_u4, float* smem_sum32 ) { uint32_t p0 = __byte_perm(v.x, v.z, 0x5410u); uint32_t p1 = __byte_perm(v.x, v.z, 0x7632u); uint32_t p2 = __byte_perm(v.y, v.w, 0x5410u); uint32_t p3 = __byte_perm(v.y, v.w, 0x7632u); int c = q >> 2; smem_u4[c * X_U4_STRIDE + (q & 3)] = make_uint4(p0, p1, p2, p3); float s = (__uint_as_float(v.x << 16) + __uint_as_float(v.x & 0xFFFF0000u)) + (__uint_as_float(v.y << 16) + __uint_as_float(v.y & 0xFFFF0000u)) + (__uint_as_float(v.z << 16) + __uint_as_float(v.z & 0xFFFF0000u)) + (__uint_as_float(v.w << 16) + __uint_as_float(v.w & 0xFFFF0000u)); s = subwarp4_reduce_sum(s); if ((threadIdx.x & 3) == 0) smem_sum32[c] = s; } // --------------------------------------------------------------------------- // Overlapped o_proj (4096 -> 2304): // - Direct 12-13 rows/block across all 188 SMs (zero remainder atomics) // - Issues g_attn_o_in L2 loads first, overlapped with o_wq[0..1] + g_hidden[n] // --------------------------------------------------------------------------- template __device__ __forceinline__ void run_load_and_oproj( const void* g_attn_o_in, const uint4* o_wq, const uint32_t* o_sz, __nv_bfloat16* g_hidden, float* g_norm_sq, uint4* smem_x_u4, float* smem_x_sum32, float* smem_sq16, int bid, int nblocks, uint64_t pol_keep ) { int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; int n0 = (bid * HIDDEN) / nblocks; int n_rows = ((bid + 1) * HIDDEN) / nblocks - n0; uint4 w0 = make_uint4(0, 0, 0, 0), w1 = make_uint4(0, 0, 0, 0); uint32_t s0 = 0, s1 = 0; const uint4* wr = nullptr; const uint32_t* sr = nullptr; int n = 0; __nv_bfloat16 h_prev = f32_to_bf16(0.0f); int q = (tid + (bid & 127) * 4) & 511; uint4 vo_a = {}, vo_b = {}; if constexpr (!INPUT_IS_F32) { vo_a = ld_u4_ca(reinterpret_cast(g_attn_o_in) + q); } else { const uint4* src = reinterpret_cast(g_attn_o_in); vo_a = ld_u4_ca(src + q * 2 + 0); vo_b = ld_u4_ca(src + q * 2 + 1); } if (wid < n_rows) { n = n0 + wid; wr = o_wq + static_cast(n) * 128; sr = o_sz + static_cast(n) * 32; w0 = ld_u4_keep(wr + lane, pol_keep); w1 = ld_u4_keep(wr + lane + 32, pol_keep); s0 = ld_u32_keep(sr + (lane >> 2), pol_keep); s1 = ld_u32_keep(sr + ((lane + 32) >> 2), pol_keep); if (lane == 0) h_prev = g_hidden[n]; } if constexpr (!INPUT_IS_F32) { store_bf16_u4_to_smem_bf2(q, vo_a, smem_x_u4, smem_x_sum32); } else { float4 a = *reinterpret_cast(&vo_a); float4 b = *reinterpret_cast(&vo_b); uint32_t p0 = pack_bf16x2(a.x, b.x); uint32_t p1 = pack_bf16x2(a.y, b.y); uint32_t p2 = pack_bf16x2(a.z, b.z); uint32_t p3 = pack_bf16x2(a.w, b.w); int c = q >> 2; smem_x_u4[c * X_U4_STRIDE + (q & 3)] = make_uint4(p0, p1, p2, p3); float s = (__uint_as_float(p0 << 16) + __uint_as_float(p0 & 0xFFFF0000u)) + (__uint_as_float(p1 << 16) + __uint_as_float(p1 & 0xFFFF0000u)) + (__uint_as_float(p2 << 16) + __uint_as_float(p2 & 0xFFFF0000u)) + (__uint_as_float(p3 << 16) + __uint_as_float(p3 & 0xFFFF0000u)); s = subwarp4_reduce_sum(s); if ((tid & 3) == 0) smem_x_sum32[c] = s; } __syncthreads(); if (wid < n_rows) { int k0 = lane, k1 = lane + 32, k2 = lane + 64, k3 = lane + 96; uint4 w2 = ld_u4_keep(wr + k2, pol_keep); uint4 w3 = ld_u4_keep(wr + k3, pol_keep); uint32_t s2 = ld_u32_keep(sr + (k2 >> 2), pol_keep); uint32_t s3 = ld_u32_keep(sr + (k3 >> 2), pol_keep); float acc = dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]); acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]); acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]); acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]); acc = warp_reduce_sum(acc); if (lane == 0) { __nv_bfloat16 proj_bf = f32_to_bf16(acc); float h_new = bf16_to_f32(h_prev) + bf16_to_f32(proj_bf); __nv_bfloat16 h_bf = f32_to_bf16(h_new); g_hidden[n] = h_bf; float hf = bf16_to_f32(h_bf); smem_sq16[wid] = hf * hf; } } __syncthreads(); if (wid == 0) { float sq = (lane < n_rows) ? smem_sq16[lane] : 0.0f; sq = warp_reduce_sum(sq); if (lane == 0 && sq != 0.0f) atomicAdd(g_norm_sq, sq); } } // --------------------------------------------------------------------------- // MoE block: // - Fused M1+M2 (3-way K-split s_gate_up, red.release.gpu + ld.relaxed/fence.acquire spin, // redux.sync Top-8, balanced PRE=2 L2::256B BF16x2 routed gate+up) // - Overlapped M3: direct 12-13 rows/block, contiguous expert rotation, // 4 initial experts (e0..e3) + g_hidden[n] pre-issued before load_hh // --------------------------------------------------------------------------- __device__ __forceinline__ void run_moe_layer( cg::grid_group& grid, const MoeLayerPtrs& moe, __nv_bfloat16* g_hidden, __nv_bfloat16* final_out, float* g_norm_sq, float* g_router_logits, unsigned int* g_router_done, __nv_bfloat16* g_moe_hh, float routed_scaling, int n_experts, char* smem_raw, int bid, int nblocks, uint64_t pol_keep, uint64_t pol_stream ) { int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; uint4* smem_x_u4 = reinterpret_cast(smem_raw); float* smem_x_sum32 = reinterpret_cast(smem_x_u4 + 72 * X_U4_STRIDE); float* smem_red16 = smem_x_sum32 + 80; float* smem_sgu = smem_red16 + 16; int* smem_topk_idx = reinterpret_cast(smem_raw + 32768); float* smem_topk_w = reinterpret_cast(smem_raw + 32832); float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f); if (bid < n_experts) { // Fused load_rmsnorm + router: issues h8, w8, and router_w in parallel, dots in registers, 1 fewer barrier! float acc = 0.0f; if (tid < 288) { int shift = (bid % 72) * 4; int q = tid + shift; if (q >= 288) q -= 288; const uint4* rw_u4 = reinterpret_cast(moe.router_w) + static_cast(bid) * 288; uint4 h8 = ld_u4_ca(reinterpret_cast(g_hidden) + q); uint4 w8 = ld_u4_keep(reinterpret_cast(moe.moe_norm) + q, pol_keep); uint4 v = ld_u4_keep(rw_u4 + q, pol_keep); uint4 x4 = pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32); acc = fmaf(__uint_as_float(x4.x << 16), __uint_as_float(v.x << 16), acc); acc = fmaf(__uint_as_float(x4.x & 0xFFFF0000u), __uint_as_float(v.x & 0xFFFF0000u), acc); acc = fmaf(__uint_as_float(x4.y << 16), __uint_as_float(v.y << 16), acc); acc = fmaf(__uint_as_float(x4.y & 0xFFFF0000u), __uint_as_float(v.y & 0xFFFF0000u), acc); acc = fmaf(__uint_as_float(x4.z << 16), __uint_as_float(v.z << 16), acc); acc = fmaf(__uint_as_float(x4.z & 0xFFFF0000u), __uint_as_float(v.z & 0xFFFF0000u), acc); acc = fmaf(__uint_as_float(x4.w << 16), __uint_as_float(v.w << 16), acc); acc = fmaf(__uint_as_float(x4.w & 0xFFFF0000u), __uint_as_float(v.w & 0xFFFF0000u), acc); } acc = warp_reduce_sum(acc); if (lane == 0) smem_red16[wid] = acc; __syncthreads(); if (wid == 0) { float tot = (lane < 9) ? smem_red16[lane] : 0.0f; tot = warp_reduce_sum(tot); if (lane == 0) { g_router_logits[bid] = tot; asm volatile("red.release.gpu.global.add.u32 [%0], 1;" :: "l"(g_router_done) : "memory"); } } } else { int s_bid = bid - n_experts; int s_nblocks = nblocks - n_experts; int c0 = (s_bid * MOE_INTER) / s_nblocks; int c1 = ((s_bid + 1) * MOE_INTER) / s_nblocks; int n_pairs = c1 - c0; int n_jobs = n_pairs * 3; int sub_id = tid >> 3; int t8 = tid & 7; bool act = (sub_id < n_jobs); int p_loc = 0, ks = 0, k0 = 0, k1 = 0, k2 = 0; uint4 wg0 = {}, wu0 = {}, wg1 = {}, wu1 = {}, wg2 = {}, wu2 = {}; uint32_t sg0 = 0, su0 = 0, sg1 = 0, su1 = 0, sg2 = 0, su2 = 0; if (wid * 4 < n_jobs) { int j_safe = act ? sub_id : 0; p_loc = (j_safe * 21846) >> 16; ks = j_safe - p_loc * 3; int c = c0 + p_loc; const uint4* gw = reinterpret_cast(moe.s_gate_up_wq) + static_cast(c) * 144; const uint32_t* gsz = reinterpret_cast(moe.s_gate_up_sz) + static_cast(c) * 36; k0 = (ks * 3 + 0) * 8 + t8; k1 = (ks * 3 + 1) * 8 + t8; k2 = (ks * 3 + 2) * 8 + t8; wg0 = ld_u4_keep(gw + k0, pol_keep); wu0 = ld_u4_keep(gw + 72 + k0, pol_keep); wg1 = ld_u4_keep(gw + k1, pol_keep); wu1 = ld_u4_keep(gw + 72 + k1, pol_keep); wg2 = ld_u4_keep(gw + k2, pol_keep); wu2 = ld_u4_keep(gw + 72 + k2, pol_keep); sg0 = ld_u32_keep(gsz + (k0 >> 2), pol_keep); su0 = ld_u32_keep(gsz + 18 + (k0 >> 2), pol_keep); sg1 = ld_u32_keep(gsz + (k1 >> 2), pol_keep); su1 = ld_u32_keep(gsz + 18 + (k1 >> 2), pol_keep); sg2 = ld_u32_keep(gsz + (k2 >> 2), pol_keep); su2 = ld_u32_keep(gsz + 18 + (k2 >> 2), pol_keep); } load_rmsnorm_2304_to_smem_bf2( g_hidden, reinterpret_cast(moe.moe_norm), inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep ); if (wid * 4 < n_jobs) { float ga = 0.0f, ua = 0.0f; dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua); dot32_bf2_pair(wg1, sg1, wu1, su1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], ga, ua); dot32_bf2_pair(wg2, sg2, wu2, su2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2], ga, ua); ga = subwarp8_reduce_sum(ga); ua = subwarp8_reduce_sum(ua); if (act && t8 == 0) { smem_sgu[p_loc * 6 + ks] = ga; smem_sgu[p_loc * 6 + 3 + ks] = ua; } } __syncthreads(); if (tid < n_pairs) { float ga = smem_sgu[tid * 6 + 0] + smem_sgu[tid * 6 + 1] + smem_sgu[tid * 6 + 2]; float ua = smem_sgu[tid * 6 + 3] + smem_sgu[tid * 6 + 4] + smem_sgu[tid * 6 + 5]; g_moe_hh[8 * MOE_INTER + c0 + tid] = f32_to_bf16(silu_f32(ga) * ua); } } // Fused spin-wait + parallel-lookup redux.sync Top-8 in wid==0 (saves 1 __syncthreads + 7 shuffles per MoE!) if (wid == 0) { if (lane == 0) { unsigned int done; while (true) { asm volatile("ld.relaxed.gpu.global.u32 %0, [%1];" : "=r"(done) : "l"(g_router_done)); if (done >= static_cast(n_experts)) break; asm volatile("nanosleep.u32 64;"); } asm volatile("fence.acquire.gpu;" ::: "memory"); } __syncwarp(); float l0 = (lane < n_experts) ? g_router_logits[lane] : -1e30f; float l1 = (lane + 32 < n_experts) ? g_router_logits[lane + 32] : -1e30f; uint32_t k0 = (lane < n_experts) ? ((float_to_sortable_u32(l0) & 0xFFFFFFC0u) | (63u - lane)) : 0u; uint32_t k1 = (lane + 32 < n_experts) ? ((float_to_sortable_u32(l1) & 0xFFFFFFC0u) | (63u - (lane + 32))) : 0u; int my_bi = 0; #pragma unroll for (int k = 0; k < N_ACTIVE; ++k) { uint32_t km = (k0 >= k1) ? k0 : k1; uint32_t best; asm("redux.sync.max.u32 %0, %1, 0xffffffff;" : "=r"(best) : "r"(km)); int bi = 63 - int(best & 63u); if (lane == k) my_bi = bi; if (bi == lane) k0 = 0u; if (bi == lane + 32) k1 = 0u; } int src_lane = my_bi & 31; float v_lo = __shfl_sync(0xffffffffu, l0, src_lane); float v_hi = __shfl_sync(0xffffffffu, l1, src_lane); float my_val = (my_bi < 32) ? v_lo : v_hi; float m_top = __shfl_sync(0xffffffffu, my_val, 0); float e = (lane < N_ACTIVE) ? __expf(my_val - m_top) : 0.0f; float w_sum = warp_reduce_sum(e); if (lane < N_ACTIVE) { smem_topk_idx[lane] = my_bi; smem_topk_w[lane] = (e / (w_sum + 1e-9f)) * routed_scaling; } if (lane == 0) { smem_topk_idx[8] = n_experts; smem_topk_w[8] = 1.0f; if (bid == 0) *g_norm_sq = 0.0f; } } __syncthreads(); // Phase M2: Contiguous 1-wave 2-step lookahead L2::256B BF16x2 routed gate+up { int p0_idx, n_pairs; if (nblocks == 188 && n_experts == 64) { if (bid < 64) { p0_idx = bid * 45; n_pairs = 45; } else if (bid < 168) { p0_idx = 2880 + (bid - 64) * 43; n_pairs = 43; } else { p0_idx = 7352 + (bid - 168) * 42; n_pairs = 42; } } else { p0_idx = (bid * (N_ACTIVE * MOE_INTER)) / nblocks; n_pairs = ((bid + 1) * (N_ACTIVE * MOE_INTER)) / nblocks - p0_idx; } int sub_id = tid >> 3; int t8 = tid & 7; const uint4* all_wq = reinterpret_cast(moe.r_gate_up_wq); const uint32_t* all_sz = reinterpret_cast(moe.r_gate_up_sz); int n_warps = (n_pairs + 3) >> 2; if (wid < n_warps) { bool act = (sub_id < n_pairs); int pair = p0_idx + (act ? sub_id : 0); int j = pair >> 10; int c = pair & 1023; int e = smem_topk_idx[j]; size_t row_off = static_cast(e) * MOE_INTER + c; const uint4* gw = all_wq + row_off * 144; const uint32_t* gsz = all_sz + row_off * 36; float ga = 0.0f, ua = 0.0f; int k0 = t8, k1 = 8 + t8; uint4 wg0 = ld_u4_stream256(gw + k0, pol_stream), wu0 = ld_u4_stream256(gw + 72 + k0, pol_stream); uint4 wg1 = ld_u4_stream256(gw + k1, pol_stream), wu1 = ld_u4_stream256(gw + 72 + k1, pol_stream); uint32_t sg0 = ld_u32_stream(gsz + (k0 >> 2), pol_stream), su0 = ld_u32_stream(gsz + 18 + (k0 >> 2), pol_stream); uint32_t sg1 = ld_u32_stream(gsz + (k1 >> 2), pol_stream), su1 = ld_u32_stream(gsz + 18 + (k1 >> 2), pol_stream); #pragma unroll for (int it = 0; it < 7; ++it) { int kn = (it + 2) * 8 + t8; uint4 nwg = ld_u4_stream256(gw + kn, pol_stream), nwu = ld_u4_stream256(gw + 72 + kn, pol_stream); uint32_t nsg = ld_u32_stream(gsz + (kn >> 2), pol_stream), nsu = ld_u32_stream(gsz + 18 + (kn >> 2), pol_stream); dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua); wg0 = wg1; wu0 = wu1; sg0 = sg1; su0 = su1; k0 = k1; wg1 = nwg; wu1 = nwu; sg1 = nsg; su1 = nsu; k1 = kn; } dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua); dot32_bf2_pair(wg1, sg1, wu1, su1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], ga, ua); ga = subwarp8_reduce_sum(ga); ua = subwarp8_reduce_sum(ua); if (act && t8 == 0) g_moe_hh[pair] = f32_to_bf16(silu_f32(ga) * ua); } } grid.sync(); if (bid == 0 && tid == 0) *g_router_done = 0u; // Phase M3: direct 12-13 rows/block // Contiguous expert-group rotation (rot = bid*8/nblocks) + ALL 4 initial experts pre-issued BEFORE load_hh! uint4* smem_hh_u4 = reinterpret_cast(smem_raw); float* smem_hh_sum32 = reinterpret_cast(smem_hh_u4 + 288 * X_U4_STRIDE); float* smem_sq16 = smem_hh_sum32 + 288; int n0 = (bid * HIDDEN) / nblocks; int n_rows = ((bid + 1) * HIDDEN) / nblocks - n0; int rot = (bid * 8) / nblocks; const uint4* r_down_wq = reinterpret_cast(moe.r_down_wq); const uint32_t* r_down_sz = reinterpret_cast(moe.r_down_sz); const uint4* s_down_wq = reinterpret_cast(moe.s_down_wq); const uint32_t* s_down_sz = reinterpret_cast(moe.s_down_sz); uint4 w0 = make_uint4(0, 0, 0, 0), w1 = make_uint4(0, 0, 0, 0); uint4 w2 = make_uint4(0, 0, 0, 0), w3 = make_uint4(0, 0, 0, 0); uint32_t s0 = 0, s1 = 0, s2 = 0, s3 = 0; __nv_bfloat16 h_prev = f32_to_bf16(0.0f); if (wid < n_rows) { int n = n0 + wid; int e0 = smem_topk_idx[rot], e1 = smem_topk_idx[(rot + 1) & 7]; int e2 = smem_topk_idx[(rot + 2) & 7], e3 = smem_topk_idx[(rot + 3) & 7]; w0 = ld_u4_stream256(r_down_wq + (static_cast(e0) * HIDDEN + n) * 32 + lane, pol_stream); w1 = ld_u4_stream256(r_down_wq + (static_cast(e1) * HIDDEN + n) * 32 + lane, pol_stream); w2 = ld_u4_stream256(r_down_wq + (static_cast(e2) * HIDDEN + n) * 32 + lane, pol_stream); w3 = ld_u4_stream256(r_down_wq + (static_cast(e3) * HIDDEN + n) * 32 + lane, pol_stream); s0 = ld_u32_stream(r_down_sz + (static_cast(e0) * HIDDEN + n) * 8 + (lane >> 2), pol_stream); s1 = ld_u32_stream(r_down_sz + (static_cast(e1) * HIDDEN + n) * 8 + (lane >> 2), pol_stream); s2 = ld_u32_stream(r_down_sz + (static_cast(e2) * HIDDEN + n) * 8 + (lane >> 2), pol_stream); s3 = ld_u32_stream(r_down_sz + (static_cast(e3) * HIDDEN + n) * 8 + (lane >> 2), pol_stream); if (lane == 0) h_prev = g_hidden[n]; } { const uint4* hh_u4 = reinterpret_cast(g_moe_hh); int shift = (bid % 288) * 4; int q0 = tid + shift; if (q0 >= 1152) q0 -= 1152; int q1 = tid + 512 + shift; if (q1 >= 1152) q1 -= 1152; int q2 = tid + 1024 + shift; if (q2 >= 1152) q2 -= 1152; uint4 v0 = ld_u4_ca(hh_u4 + q0); uint4 v1 = ld_u4_ca(hh_u4 + q1); uint4 v2 = (tid < 128) ? ld_u4_ca(hh_u4 + q2) : make_uint4(0u, 0u, 0u, 0u); store_bf16_u4_to_smem_bf2(q0, v0, smem_hh_u4, smem_hh_sum32); store_bf16_u4_to_smem_bf2(q1, v1, smem_hh_u4, smem_hh_sum32); if (tid < 128) store_bf16_u4_to_smem_bf2(q2, v2, smem_hh_u4, smem_hh_sum32); } __syncthreads(); if (wid < n_rows) { int n = n0 + wid; float acc = 0.0f; #pragma unroll for (int j = 0; j < 5; ++j) { int jc = (j + rot) & 7; uint4 wn; uint32_t sn; if (j < 4) { int en = smem_topk_idx[(j + 4 + rot) & 7]; wn = ld_u4_stream256(r_down_wq + (static_cast(en) * HIDDEN + n) * 32 + lane, pol_stream); sn = ld_u32_stream(r_down_sz + (static_cast(en) * HIDDEN + n) * 8 + (lane >> 2), pol_stream); } else { wn = ld_u4_keep(s_down_wq + static_cast(n) * 32 + lane, pol_keep); sn = ld_u32_keep(s_down_sz + static_cast(n) * 8 + (lane >> 2), pol_keep); } acc = fmaf(smem_topk_w[jc], dot32_bf2(w0, s0, smem_hh_u4 + (jc * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[jc * 32 + lane]), acc); w0 = w1; s0 = s1; w1 = w2; s1 = s2; w2 = w3; s2 = s3; w3 = wn; s3 = sn; } int j5 = (5 + rot) & 7, j6 = (6 + rot) & 7, j7 = (7 + rot) & 7; acc = fmaf(smem_topk_w[j5], dot32_bf2(w0, s0, smem_hh_u4 + (j5 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j5 * 32 + lane]), acc); acc = fmaf(smem_topk_w[j6], dot32_bf2(w1, s1, smem_hh_u4 + (j6 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j6 * 32 + lane]), acc); acc = fmaf(smem_topk_w[j7], dot32_bf2(w2, s2, smem_hh_u4 + (j7 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j7 * 32 + lane]), acc); acc += dot32_bf2(w3, s3, smem_hh_u4 + (8 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[8 * 32 + lane]); acc = warp_reduce_sum(acc); if (lane == 0) { __nv_bfloat16 moe_bf = f32_to_bf16(acc); float h_new = bf16_to_f32(h_prev) + bf16_to_f32(moe_bf); __nv_bfloat16 h_bf = f32_to_bf16(h_new); if (final_out != nullptr) { final_out[n] = h_bf; } else { g_hidden[n] = h_bf; float hf = bf16_to_f32(h_bf); smem_sq16[wid] = hf * hf; } } } if (final_out == nullptr) { __syncthreads(); if (wid == 0) { float sq = (lane < n_rows) ? smem_sq16[lane] : 0.0f; sq = warp_reduce_sum(sq); if (lane == 0 && sq != 0.0f) atomicAdd(g_norm_sq, sq); } } } // --------------------------------------------------------------------------- // KDA Attention block: // - PRE=2 software-pipelined qkvg + overlapped coalesced conv preload (warps 12..14) // - K2 pre-issues all 8 S loads before smem_q/k/dec + tree warp reduction // - Input-first overlapped o_proj // --------------------------------------------------------------------------- __device__ __forceinline__ void run_kda_attn( cg::grid_group& grid, const KdaLayerPtrs& kda, float* S_state, __nv_bfloat16* cq, __nv_bfloat16* ck, __nv_bfloat16* cv, const __nv_bfloat16* in_hidden_or_null, __nv_bfloat16* g_hidden, float* g_norm_sq, float* g_q, float* g_k, float* g_v, float* g_decay, float* g_beta, __nv_bfloat16* g_attn_o, char* smem_raw, int bid, int nblocks, uint64_t pol_keep ) { int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; uint4* smem_x_u4 = reinterpret_cast(smem_raw); float* smem_x_sum32 = reinterpret_cast(smem_x_u4 + 128 * X_U4_STRIDE); float* smem_red16 = smem_x_sum32 + 128; float* smem_qkvg = smem_red16 + 16; // [96] floats float* smem_conv_pre = smem_qkvg + 96; // [96] floats (p0*w0 + p1*w1 + p2*w2) float* smem_conv_w3 = smem_conv_pre + 96; // [96] floats (w3) uint16_t* smem_p1 = reinterpret_cast(smem_conv_w3 + 96); // [96] u16 uint16_t* smem_p2 = smem_p1 + 96; // [96] u16 int r0 = (bid * 16384) / nblocks; int r1 = ((bid + 1) * 16384) / nblocks; int n_rows = r1 - r0; int sub_id = tid >> 3; int t8 = tid & 7; int n_sub = (n_rows + 1) >> 1; const uint4* qkvg_wq = reinterpret_cast(kda.qkvg_wq); const uint32_t* qkvg_sz = reinterpret_cast(kda.qkvg_sz); const uint2* conv_w_u2 = reinterpret_cast(kda.conv_w); bool has0 = false, has1 = false; const uint4* wr0 = nullptr; const uint4* wr1 = nullptr; const uint32_t* sr0 = nullptr; const uint32_t* sr1 = nullptr; uint4 wa0 = {}, wb0 = {}, wa1 = {}, wb1 = {}; uint32_t sa0 = 0, sb0 = 0, sa1 = 0, sb1 = 0; int k0 = t8, k1 = 8 + t8; if (wid * 4 < n_sub) { int r = r0 + sub_id * 2; has0 = (r < r1); has1 = (r + 1 < r1); int rr0 = has0 ? r : (r1 - 1); int rr1 = has1 ? (r + 1) : rr0; wr0 = qkvg_wq + static_cast(rr0) * 72; wr1 = qkvg_wq + static_cast(rr1) * 72; sr0 = qkvg_sz + static_cast(rr0) * 18; sr1 = qkvg_sz + static_cast(rr1) * 18; wa0 = ld_u4_keep(wr0 + k0, pol_keep); wb0 = ld_u4_keep(wr1 + k0, pol_keep); wa1 = ld_u4_keep(wr0 + k1, pol_keep); wb1 = ld_u4_keep(wr1 + k1, pol_keep); sa0 = ld_u32_keep(sr0 + (k0 >> 2), pol_keep); sb0 = ld_u32_keep(sr1 + (k0 >> 2), pol_keep); sa1 = ld_u32_keep(sr0 + (k1 >> 2), pol_keep); sb1 = ld_u32_keep(sr1 + (k1 >> 2), pol_keep); } else if (wid >= 12 && wid <= 14) { int loc = tid - 384; if (loc < n_rows) { int row = r0 + loc; int kind = row >> 12; int c = row & 4095; if (kind < 3) { const uint16_t* prev = reinterpret_cast( (kind == 0) ? cq : ((kind == 1) ? ck : cv) ); uint16_t p0 = ld_u16_keep(prev + c, pol_keep); uint16_t p1 = ld_u16_keep(prev + KDA_C + c, pol_keep); uint16_t p2 = ld_u16_keep(prev + 2 * KDA_C + c, pol_keep); uint2 cw = ld_u2_keep(conv_w_u2 + kind * KDA_C + c, pol_keep); float pre = __uint_as_float(uint32_t(p0) << 16) * __uint_as_float(cw.x << 16) + __uint_as_float(uint32_t(p1) << 16) * __uint_as_float(cw.x & 0xFFFF0000u) + __uint_as_float(uint32_t(p2) << 16) * __uint_as_float(cw.y << 16); smem_conv_pre[loc] = pre; smem_conv_w3[loc] = __uint_as_float(cw.y & 0xFFFF0000u); smem_p1[loc] = p1; smem_p2[loc] = p2; } } } if (in_hidden_or_null != nullptr) { load_input_and_rmsnorm_2304_to_smem_bf2( in_hidden_or_null, g_hidden, reinterpret_cast(kda.attn_norm), smem_x_u4, smem_x_sum32, smem_red16, bid, pol_keep ); } else { float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f); load_rmsnorm_2304_to_smem_bf2( g_hidden, reinterpret_cast(kda.attn_norm), inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep ); } // Phase K1: 1-wide 2-row BF16x2 GEMV (warps 0..10) + overlapped coalesced conv preload (warps 12..14) + beta (warp 15) { if (wid * 4 < n_sub) { float a0 = 0.0f, a1 = 0.0f; #pragma unroll for (int it = 0; it < 7; ++it) { int kn = (it + 2) * 8 + t8; uint4 nwa = ld_u4_keep(wr0 + kn, pol_keep), nwb = ld_u4_keep(wr1 + kn, pol_keep); uint32_t nsa = ld_u32_keep(sr0 + (kn >> 2), pol_keep), nsb = ld_u32_keep(sr1 + (kn >> 2), pol_keep); dot32_bf2_pair(wa0, sa0, wb0, sb0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], a0, a1); wa0 = wa1; wb0 = wb1; sa0 = sa1; sb0 = sb1; k0 = k1; wa1 = nwa; wb1 = nwb; sa1 = nsa; sb1 = nsb; k1 = kn; } dot32_bf2_pair(wa0, sa0, wb0, sb0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], a0, a1); dot32_bf2_pair(wa1, sa1, wb1, sb1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], a0, a1); a0 = subwarp8_reduce_sum(a0); a1 = subwarp8_reduce_sum(a1); if (t8 == 0) { if (has0) smem_qkvg[sub_id * 2] = a0; if (has1) smem_qkvg[sub_id * 2 + 1] = a1; } } else if (bid < KDA_H && wid == 15) { const uint4* bw_u4 = reinterpret_cast(kda.beta_w) + static_cast(bid) * 288; float acc = 0.0f; for (int q = lane; q < 288; q += 32) { uint4 v = ld_u4_keep(bw_u4 + q, pol_keep); uint4 x4 = smem_x_u4[(q >> 2) * X_U4_STRIDE + (q & 3)]; acc = fmaf(__uint_as_float(x4.x << 16), __uint_as_float(v.x << 16), acc); acc = fmaf(__uint_as_float(x4.x & 0xFFFF0000u), __uint_as_float(v.x & 0xFFFF0000u), acc); acc = fmaf(__uint_as_float(x4.y << 16), __uint_as_float(v.y << 16), acc); acc = fmaf(__uint_as_float(x4.y & 0xFFFF0000u), __uint_as_float(v.y & 0xFFFF0000u), acc); acc = fmaf(__uint_as_float(x4.z << 16), __uint_as_float(v.z << 16), acc); acc = fmaf(__uint_as_float(x4.z & 0xFFFF0000u), __uint_as_float(v.z & 0xFFFF0000u), acc); acc = fmaf(__uint_as_float(x4.w << 16), __uint_as_float(v.w << 16), acc); acc = fmaf(__uint_as_float(x4.w & 0xFFFF0000u), __uint_as_float(v.w & 0xFFFF0000u), acc); } acc = warp_reduce_sum(acc); if (lane == 0) g_beta[bid] = sigmoid_f32(acc); } __syncthreads(); // Coalesced 88-thread epilogue (1 FMA + SiLU + coalesced pol_keep stores!) if (tid < n_rows) { int row = r0 + tid; int kind = row >> 12; int c = row & 4095; __nv_bfloat16 val_bf = f32_to_bf16(smem_qkvg[tid]); float val_f = bf16_to_f32(val_bf); if (kind < 3) { float out = fmaf(val_f, smem_conv_w3[tid], smem_conv_pre[tid]); float s = bf16_to_f32(f32_to_bf16(silu_f32(out))); uint16_t* prev = reinterpret_cast( (kind == 0) ? cq : ((kind == 1) ? ck : cv) ); uint16_t val_u16 = *reinterpret_cast(&val_bf); st_u16_keep(prev + c, smem_p1[tid], pol_keep); st_u16_keep(prev + KDA_C + c, smem_p2[tid], pol_keep); st_u16_keep(prev + 2 * KDA_C + c, val_u16, pol_keep); if (kind == 0) g_q[c] = s * 0.08838834764831845f; else if (kind == 1) g_k[c] = s; else g_v[c] = s; } else { g_decay[c] = sigmoid_f32(-val_f); } } } if (bid == 0 && tid == 0) *g_norm_sq = 0.0f; grid.sync(); // Phase K2: KDA Recurrent State Update S[32, 128, 128] (pre-issued S loads + tree warp reduce!) if (bid < 128) { int h = bid >> 2; int col_base = (bid & 3) * 32; int j = col_base + lane; float* smem_q = reinterpret_cast(smem_raw); float* smem_k = smem_q + 128; float* smem_dec = smem_k + 128; float* smem_red = smem_dec + 128; float* smem_delta = smem_red + 512; float* S_head = S_state + static_cast(h) * 128 * 128; int r_base = wid * 8; float s_reg[8]; #pragma unroll for (int r = 0; r < 8; ++r) { s_reg[r] = ld_f32_keep(S_head + (r_base + r) * 128 + j, pol_keep); } if (tid < 128) { smem_q[tid] = g_q[h * 128 + tid]; smem_k[tid] = g_k[h * 128 + tid]; smem_dec[tid] = g_decay[h * 128 + tid]; } __syncthreads(); float pred_part = 0.0f; #pragma unroll for (int r = 0; r < 8; ++r) { int i = r_base + r; float s = s_reg[r] * smem_dec[i]; s_reg[r] = s; pred_part = fmaf(s, smem_k[i], pred_part); } smem_red[wid * 32 + lane] = pred_part; __syncthreads(); if (wid == 0) { float a0 = smem_red[ 0 * 32 + lane] + smem_red[ 1 * 32 + lane]; float a1 = smem_red[ 2 * 32 + lane] + smem_red[ 3 * 32 + lane]; float a2 = smem_red[ 4 * 32 + lane] + smem_red[ 5 * 32 + lane]; float a3 = smem_red[ 6 * 32 + lane] + smem_red[ 7 * 32 + lane]; float a4 = smem_red[ 8 * 32 + lane] + smem_red[ 9 * 32 + lane]; float a5 = smem_red[10 * 32 + lane] + smem_red[11 * 32 + lane]; float a6 = smem_red[12 * 32 + lane] + smem_red[13 * 32 + lane]; float a7 = smem_red[14 * 32 + lane] + smem_red[15 * 32 + lane]; float pred = ((a0 + a1) + (a2 + a3)) + ((a4 + a5) + (a6 + a7)); smem_delta[lane] = g_beta[h] * (g_v[h * 128 + j] - pred); } __syncthreads(); float delta = smem_delta[lane]; float o_part = 0.0f; #pragma unroll for (int r = 0; r < 8; ++r) { int i = r_base + r; float s_new = fmaf(smem_k[i], delta, s_reg[r]); st_f32_keep(S_head + i * 128 + j, s_new, pol_keep); o_part = fmaf(s_new, smem_q[i], o_part); } smem_red[wid * 32 + lane] = o_part; __syncthreads(); if (wid == 0) { float b0 = smem_red[ 0 * 32 + lane] + smem_red[ 1 * 32 + lane]; float b1 = smem_red[ 2 * 32 + lane] + smem_red[ 3 * 32 + lane]; float b2 = smem_red[ 4 * 32 + lane] + smem_red[ 5 * 32 + lane]; float b3 = smem_red[ 6 * 32 + lane] + smem_red[ 7 * 32 + lane]; float b4 = smem_red[ 8 * 32 + lane] + smem_red[ 9 * 32 + lane]; float b5 = smem_red[10 * 32 + lane] + smem_red[11 * 32 + lane]; float b6 = smem_red[12 * 32 + lane] + smem_red[13 * 32 + lane]; float b7 = smem_red[14 * 32 + lane] + smem_red[15 * 32 + lane]; float o_val = ((b0 + b1) + (b2 + b3)) + ((b4 + b5) + (b6 + b7)); g_attn_o[h * 128 + j] = f32_to_bf16(o_val); } } grid.sync(); run_load_and_oproj( g_attn_o, reinterpret_cast(kda.o_wq), reinterpret_cast(kda.o_sz), g_hidden, g_norm_sq, smem_x_u4, smem_x_sum32, smem_red16, bid, nblocks, pol_keep ); grid.sync(); } // --------------------------------------------------------------------------- // MLA Attention block: // - Spin-free fused A1+A2 (nibble-transposed int4 W_uk, 1 uint4/thread, 0 spin-wait) // - Warp-aligned FlashMLA A3 (async cp.async Step-0 copy, exp2f online softmax, streaming part_o) // - Slim overlapped A4 (4-split part_o prefetch + 1-warp exp2f softmax + overlapped W_uv) // - Input-first overlapped A5 o_proj // --------------------------------------------------------------------------- __device__ __forceinline__ void run_mla_attn( cg::grid_group& grid, const MlaLayerPtrs& mla, const __nv_bfloat16* src_c_kv, const __nv_bfloat16* src_k_rope, __nv_bfloat16* dst_c_kv, __nv_bfloat16* dst_k_rope, int ctx_len, int need_copy, float rope_theta, __nv_bfloat16* g_hidden, float* g_norm_sq, float* g_q_full_f32, __nv_bfloat16* g_q_rope, float2* g_mla_part_ml, __nv_bfloat16* g_mla_part_o, float* g_attn_o_f32, char* smem_raw, int bid, int nblocks, uint64_t pol_keep, uint64_t pol_stream ) { int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; uint4* smem_x_u4 = reinterpret_cast(smem_raw); float* smem_x_sum32 = reinterpret_cast(smem_x_u4 + 128 * X_U4_STRIDE); float* smem_red16 = smem_x_sum32 + 128; float* smem_cos = smem_red16 + 16; float* smem_sin = smem_cos + 32; float* smem_qpart = smem_sin + 32; // [64] floats for 2-way K-split q_nope! float* smem_qn = smem_qpart + 64; // [32] floats local q_nope[h, d_blk*32..+31]! float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f); load_rmsnorm_2304_to_smem_bf2( g_hidden, reinterpret_cast(mla.attn_norm), inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep ); const uint4* qkva_wq = reinterpret_cast(mla.qkva_wq); const uint32_t* qkva_sz = reinterpret_cast(mla.qkva_sz); if (bid < 128) { if (tid < 32) g_attn_o_f32[bid * 32 + tid] = 0.0f; int h = bid >> 2; int sub_id = tid >> 3; // 0..63 (ALL 16 warps active!) int t8 = tid & 7; // Step 1: 2-WAY K-SPLIT 32 rows of q_nope[h, d_blk*32..+31] straight into SMEM! { int row_loc = sub_id & 31; int ks = sub_id >> 5; // 0 or 1 int row = bid * 32 + row_loc; const uint4* w_row = qkva_wq + static_cast(row) * 72; const uint32_t* sz_row = qkva_sz + static_cast(row) * 18; float acc = 0.0f; if (ks == 0) { int k0 = 0 * 8 + t8, k1 = 1 * 8 + t8, k2 = 2 * 8 + t8, k3 = 3 * 8 + t8; uint4 w0 = ld_u4_keep(w_row + k0, pol_keep), w1 = ld_u4_keep(w_row + k1, pol_keep); uint4 w2 = ld_u4_keep(w_row + k2, pol_keep), w3 = ld_u4_keep(w_row + k3, pol_keep); uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep), s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep); uint32_t s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep), s3 = ld_u32_keep(sz_row + (k3 >> 2), pol_keep); acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]); acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]); acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]); acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]); } else { int k0 = 4 * 8 + t8, k1 = 5 * 8 + t8, k2 = 6 * 8 + t8; uint4 w0 = ld_u4_keep(w_row + k0, pol_keep), w1 = ld_u4_keep(w_row + k1, pol_keep), w2 = ld_u4_keep(w_row + k2, pol_keep); uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep), s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep), s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep); int k3 = 7 * 8 + t8, k4 = 8 * 8 + t8; uint4 w3 = ld_u4_keep(w_row + k3, pol_keep), w4 = ld_u4_keep(w_row + k4, pol_keep); uint32_t s3 = ld_u32_keep(sz_row + (k3 >> 2), pol_keep), s4 = ld_u32_keep(sz_row + (k4 >> 2), pol_keep); acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]); acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]); acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]); acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]); acc += dot32_bf2(w4, s4, smem_x_u4 + k4 * X_U4_STRIDE, smem_x_sum32[k4]); } acc = subwarp8_reduce_sum(acc); if (t8 == 0) smem_qpart[ks * 32 + row_loc] = acc; } // Pre-issue our single W_uk uint4 per thread BEFORE __syncthreads() (hides 100% of W_uk L2 latency!) const uint4* wuk_blk = reinterpret_cast(mla.wuk_wq) + static_cast(bid) * 512; uint4 w_uk = ld_u4_keep(wuk_blk + tid, pol_keep); __syncthreads(); if (tid < 32) { smem_qn[tid] = bf16_to_f32(f32_to_bf16(smem_qpart[tid] + smem_qpart[32 + tid])); } __syncthreads(); // Step 2: Spin-free W_uk (ZERO g_q_nope, ZERO g_mla_head_done spin-wait!) float* smem_qeff = reinterpret_cast(smem_raw); // [4 * 36] floats (reuses smem_x_u4, disjoint from smem_qn!) float* smem_bias = smem_qeff + 4 * X_F32_STRIDE; // [4] floats if (wid < 4) { const uint32_t* sz_ptr = reinterpret_cast(mla.wuk_sz) + static_cast(bid) * 128; uint32_t sz = ld_u32_keep(sz_ptr + tid, pol_keep); float scale = __uint_as_float(sz << 16); float zero = __uint_as_float(sz & 0xFFFF0000u); float qe = smem_qn[lane] * scale; smem_qeff[wid * X_F32_STRIDE + lane] = qe; float b = warp_reduce_sum(qe * (zero + 128.0f)); if (lane == 0) smem_bias[wid] = b; } __syncthreads(); // ALL 512 threads: 1 uint4 -> 1 dot32 -> atomicAdd into g_q_full_f32[h, tid]! { int g = tid >> 7; // 0..3 const float* xp = smem_qeff + g * X_F32_STRIDE; float dot = dot8_biased(w_uk.x, xp) + dot8_biased(w_uk.y, xp + 8) + dot8_biased(w_uk.z, xp + 16) + dot8_biased(w_uk.w, xp + 24); atomicAdd(&g_q_full_f32[h * KV_LORA + tid], dot - smem_bias[g]); } } else { int r_bid = bid - 128; int r_nblocks = nblocks - 128; int pair0 = (r_bid * 1312) / r_nblocks; int pair1 = ((r_bid + 1) * 1312) / r_nblocks; int r0 = 4096 + pair0 * 2; int n_pairs = pair1 - pair0; int sub_id = tid >> 3; int p_loc = sub_id >> 1; int t8 = tid & 7; int n_warps = (n_pairs + 1) >> 1; float val_f = 0.0f, partner = 0.0f; bool act = false; int row = r0; if (wid < n_warps) { act = (p_loc < n_pairs); row = r0 + (act ? sub_id : 0); const uint4* w_row = qkva_wq + static_cast(row) * 72; const uint32_t* sz_row = qkva_sz + static_cast(row) * 18; float acc = 0.0f; #pragma unroll for (int g3 = 0; g3 < 9; g3 += 3) { int k0 = (g3 + 0) * 8 + t8; int k1 = (g3 + 1) * 8 + t8; int k2 = (g3 + 2) * 8 + t8; uint4 w0 = ld_u4_keep(w_row + k0, pol_keep); uint4 w1 = ld_u4_keep(w_row + k1, pol_keep); uint4 w2 = ld_u4_keep(w_row + k2, pol_keep); uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep); uint32_t s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep); uint32_t s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep); acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]); acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]); acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]); } acc = subwarp8_reduce_sum(acc); val_f = bf16_to_f32(f32_to_bf16(acc)); partner = __shfl_xor_sync(0xffffffffu, val_f, 8); } else if (wid == 15) { float neg_log_theta = -__logf(rope_theta); float inv = __expf(float(2 * lane) * (1.0f / 64.0f) * neg_log_theta); float ang = float(ctx_len) * inv; float s, c; __sincosf(ang, &s, &c); smem_cos[lane] = c; smem_sin[lane] = s; } __syncthreads(); if (act && t8 == 0) { if (row < 6144) { int rr = row - 4096; int h = rr >> 6; int rd = rr & 63; int pair = rd >> 1; float c = smem_cos[pair]; float s = smem_sin[pair]; float out_r = (rd & 1) ? (val_f * c + partner * s) : (val_f * c - partner * s); g_q_rope[h * QK_ROPE + rd] = f32_to_bf16(out_r); } else if (row < 6656) { dst_c_kv[static_cast(ctx_len) * KV_LORA + (row - 6144)] = f32_to_bf16(val_f); } else { int rd = row - 6656; int pair = rd >> 1; float c = smem_cos[pair]; float s = smem_sin[pair]; float out_r = (rd & 1) ? (val_f * c + partner * s) : (val_f * c - partner * s); dst_k_rope[static_cast(ctx_len) * QK_ROPE + rd] = f32_to_bf16(out_r); } } } if (bid == 0 && tid == 0) *g_norm_sq = 0.0f; grid.sync(); // Phase A3: Zero-div72 Warp-Aligned FlashMLA + padded PART_O_STRIDE_U4 + pol_keep part_ml! int L = ctx_len + 1; int total_tiles = (L + 15) / 16; int cap_sp = (ctx_len <= 4096) ? 64 : (need_copy ? 128 : 96); int num_splits = (total_tiles < cap_sp) ? total_tiles : cap_sp; if (num_splits > nblocks) num_splits = nblocks; uint64_t kv_pol = (ctx_len <= 4096) ? pol_keep : pol_stream; if (bid < num_splits) { int tile_start = (bid * total_tiles) / num_splits; int tile_end = ((bid + 1) * total_tiles) / num_splits; uint4* smem_Q_u4 = reinterpret_cast(smem_raw); uint4* smem_KV_u4 = smem_Q_u4 + 32 * 72; float* smem_S = reinterpret_cast(smem_KV_u4 + 2 * 16 * 72); __nv_bfloat16* smem_P = reinterpret_cast<__nv_bfloat16*>(smem_S + 4 * 32 * 16); float* smem_m = reinterpret_cast(smem_P + 32 * 24); float* smem_l = smem_m + 32; float* smem_alpha = smem_l + 32; const uint4* src_ckv_u4 = reinterpret_cast(src_c_kv); const uint4* src_kr_u4 = reinterpret_cast(src_k_rope); uint4* dst_ckv_u4 = reinterpret_cast(dst_c_kv); uint4* dst_kr_u4 = reinterpret_cast(dst_k_rope); // Warp-aligned separate c_kv (512B bursts) + k_rope (tid<128) via cp.async auto issue_kv_load = [&](int buf, int tile_idx) { int t_base = tile_idx * 16; uint4* buf_ptr = smem_KV_u4 + buf * (16 * 72); int r0 = tid >> 6; // 0..7 int c8 = tid & 63; // 0..63 int r1 = r0 + 8; // 8..15 int t0 = t_base + r0; int t1 = t_base + r1; uint4* dst0 = buf_ptr + r0 * 72 + (c8 ^ (r0 & 7)); uint4* dst1 = buf_ptr + r1 * 72 + (c8 ^ (r1 & 7)); if (!need_copy) { if (t0 < L) cp_async_cg_16_pol(dst0, dst_ckv_u4 + static_cast(t0) * 64 + c8, kv_pol); else *dst0 = make_uint4(0u, 0u, 0u, 0u); if (t1 < L) cp_async_cg_16_pol(dst1, dst_ckv_u4 + static_cast(t1) * 64 + c8, kv_pol); else *dst1 = make_uint4(0u, 0u, 0u, 0u); if (tid < 128) { int rk = tid >> 3; int rc8 = tid & 7; int tk = t_base + rk; uint4* dstk = buf_ptr + rk * 72 + ((64 + rc8) ^ (rk & 7)); if (tk < L) cp_async_cg_16_pol(dstk, dst_kr_u4 + static_cast(tk) * 8 + rc8, kv_pol); else *dstk = make_uint4(0u, 0u, 0u, 0u); } } else { if (t0 < ctx_len) cp_async_cg_16_pol(dst0, src_ckv_u4 + static_cast(t0) * 64 + c8, kv_pol); else if (t0 < L) cp_async_cg_16_pol(dst0, dst_ckv_u4 + static_cast(t0) * 64 + c8, kv_pol); else *dst0 = make_uint4(0u, 0u, 0u, 0u); if (t1 < ctx_len) cp_async_cg_16_pol(dst1, src_ckv_u4 + static_cast(t1) * 64 + c8, kv_pol); else if (t1 < L) cp_async_cg_16_pol(dst1, dst_ckv_u4 + static_cast(t1) * 64 + c8, kv_pol); else *dst1 = make_uint4(0u, 0u, 0u, 0u); if (tid < 128) { int rk = tid >> 3, rc8 = tid & 7, tk = t_base + rk; uint4* dstk = buf_ptr + rk * 72 + ((64 + rc8) ^ (rk & 7)); if (tk < ctx_len) cp_async_cg_16_pol(dstk, src_kr_u4 + static_cast(tk) * 8 + rc8, kv_pol); else if (tk < L) cp_async_cg_16_pol(dstk, dst_kr_u4 + static_cast(tk) * 8 + rc8, kv_pol); else *dstk = make_uint4(0u, 0u, 0u, 0u); } } cp_async_commit(); }; issue_kv_load(0, tile_start); // Zero-div72 Q_ld via ld_u4_ca (leaves 100% of cp.async queue free for tile_start!) { const uint4* qf_u4 = reinterpret_cast(g_q_full_f32); int h_shift = bid & 31; uint4 ua[4], ub[4]; int hs[4], c8s[4]; #pragma unroll for (int pass = 0; pass < 4; ++pass) { int idx = pass * BLOCK_THREADS + tid; hs[pass] = ((idx >> 6) + h_shift) & 31; c8s[pass] = idx & 63; size_t off = (static_cast(hs[pass]) * 64 + c8s[pass]) * 2; ua[pass] = ld_u4_ca(qf_u4 + off + 0); ub[pass] = ld_u4_ca(qf_u4 + off + 1); } uint4 qrv = make_uint4(0u, 0u, 0u, 0u); int qrh = 0, qrc8 = 0; if (tid < 256) { qrh = ((tid >> 3) + h_shift) & 31; qrc8 = tid & 7; const uint4* qr_u4 = reinterpret_cast(g_q_rope); qrv = ld_u4_ca(qr_u4 + qrh * 8 + qrc8); } #pragma unroll for (int pass = 0; pass < 4; ++pass) { float4 a = *reinterpret_cast(&ua[pass]); float4 b = *reinterpret_cast(&ub[pass]); int h = hs[pass], c8 = c8s[pass]; smem_Q_u4[h * 72 + (c8 ^ (h & 7))] = make_uint4( pack_bf16x2(a.x, a.y), pack_bf16x2(a.z, a.w), pack_bf16x2(b.x, b.y), pack_bf16x2(b.z, b.w) ); } if (tid < 256) { smem_Q_u4[qrh * 72 + ((64 + qrc8) ^ (qrh & 7))] = qrv; } } if (tid < 32) { smem_m[tid] = -1e30f; smem_l[tid] = 0.0f; } float o_acc[8][4]; #pragma unroll for (int nt = 0; nt < 8; ++nt) { o_acc[nt][0] = 0.0f; o_acc[nt][1] = 0.0f; o_acc[nt][2] = 0.0f; o_acc[nt][3] = 0.0f; } int qm = wid >> 3; int qn = (wid >> 2) & 1; int qk = wid & 3; int pn = wid & 7; int buf = 0; for (int tile = tile_start; tile < tile_end; ++tile) { bool has_next = (tile + 1 < tile_end); if (has_next) { issue_kv_load(buf ^ 1, tile + 1); cp_async_wait<1>(); } else { cp_async_wait<0>(); } __syncthreads(); uint4* kv_buf = smem_KV_u4 + buf * (16 * 72); if (need_copy) { int t_base = tile * 16; int r0 = tid >> 6, c8 = tid & 63, r1 = r0 + 8; int t0 = t_base + r0, t1 = t_base + r1; if (t0 < ctx_len) st_u4_stream(dst_ckv_u4 + static_cast(t0) * 64 + c8, kv_buf[r0 * 72 + (c8 ^ (r0 & 7))], kv_pol); if (t1 < ctx_len) st_u4_stream(dst_ckv_u4 + static_cast(t1) * 64 + c8, kv_buf[r1 * 72 + (c8 ^ (r1 & 7))], kv_pol); if (tid < 128) { int rk = tid >> 3, rc8 = tid & 7, tk = t_base + rk; if (tk < ctx_len) st_u4_stream(dst_kr_u4 + static_cast(tk) * 8 + rc8, kv_buf[rk * 72 + ((64 + rc8) ^ (rk & 7))], kv_pol); } } float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f; int c8_start = qk * 18; #pragma unroll for (int ks = 0; ks < 9; ++ks) { int c8_b = c8_start + ks * 2; int ar = qm * 16 + (lane & 15); int ac8 = c8_b + ((lane >> 4) & 1); uint32_t a0, a1, a2, a3; ldmatrix_x4(a0, a1, a2, a3, smem_Q_u4 + ar * 72 + (ac8 ^ (ar & 7))); int br = qn * 8 + (lane & 7); int bc8 = c8_b + ((lane >> 3) & 1); uint32_t b0, b1; ldmatrix_x2(b0, b1, kv_buf + br * 72 + (bc8 ^ (br & 7))); mma_m16n8k16_bf16(s0, s1, s2, s3, a0, a1, a2, a3, b0, b1); } int s_r0 = qm * 16 + (lane >> 2); int s_r1 = s_r0 + 8; int s_c0 = qn * 8 + 2 * (lane & 3); float* s_dst = smem_S + qk * 512; s_dst[s_r0 * 16 + s_c0] = s0; s_dst[s_r0 * 16 + s_c0 + 1] = s1; s_dst[s_r1 * 16 + s_c0] = s2; s_dst[s_r1 * 16 + s_c0 + 1] = s3; __syncthreads(); { int sub = lane >> 4; int tl = lane & 15; int t_glob = tile * 16 + tl; int h = wid * 2 + sub; float sc = (t_glob < L) ? (smem_S[h * 16 + tl] + smem_S[512 + h * 16 + tl] + smem_S[1024 + h * 16 + tl] + smem_S[1536 + h * 16 + tl]) * 0.10411755264281874f : -1e30f; float m_tile = halfwarp16_reduce_max(sc); float m_old = smem_m[h]; float m_new = fmaxf(m_old, m_tile); float p = (t_glob < L) ? exp2f(sc - m_new) : 0.0f; float l_tile = halfwarp16_reduce_sum(p); float alpha = exp2f(m_old - m_new); smem_P[h * 24 + tl] = f32_to_bf16(p); if (tl == 0) { smem_alpha[h] = alpha; smem_m[h] = m_new; smem_l[h] = smem_l[h] * alpha + l_tile; } } __syncthreads(); { int row0 = qm * 16 + (lane >> 2); int row1 = row0 + 8; if (tile > tile_start) { float a0_s = smem_alpha[row0]; float a1_s = smem_alpha[row1]; #pragma unroll for (int nt = 0; nt < 8; ++nt) { o_acc[nt][0] *= a0_s; o_acc[nt][1] *= a0_s; o_acc[nt][2] *= a1_s; o_acc[nt][3] *= a1_s; } } int pr = qm * 16 + (lane & 15); int pc = (lane >> 4) ? 8 : 0; uint32_t pa0, pa1, pa2, pa3; ldmatrix_x4(pa0, pa1, pa2, pa3, smem_P + pr * 24 + pc); #pragma unroll for (int nt2 = 0; nt2 < 4; ++nt2) { int vr = lane & 15; int vc8 = pn * 8 + nt2 * 2 + ((lane >> 4) & 1); uint32_t b00, b10, b01, b11; ldmatrix_x4_trans(b00, b10, b01, b11, kv_buf + vr * 72 + (vc8 ^ (vr & 7))); mma_m16n8k16_bf16( o_acc[nt2 * 2][0], o_acc[nt2 * 2][1], o_acc[nt2 * 2][2], o_acc[nt2 * 2][3], pa0, pa1, pa2, pa3, b00, b10 ); mma_m16n8k16_bf16( o_acc[nt2 * 2 + 1][0], o_acc[nt2 * 2 + 1][1], o_acc[nt2 * 2 + 1][2], o_acc[nt2 * 2 + 1][3], pa0, pa1, pa2, pa3, b01, b11 ); } } if (has_next) __syncthreads(); buf ^= 1; } int row0 = qm * 16 + (lane >> 2); int row1 = row0 + 8; float inv_l0 = 1.0f / smem_l[row0]; float inv_l1 = 1.0f / smem_l[row1]; float2 my_ml = (tid < 32) ? make_float2(smem_m[tid], smem_l[tid]) : make_float2(0.0f, 0.0f); __syncthreads(); uint32_t* smem_O_u32 = reinterpret_cast(smem_raw); constexpr int O_U32_STRIDE = O_SMEM_STRIDE / 2; #pragma unroll for (int nt = 0; nt < 8; ++nt) { int col2 = (pn * 64 + nt * 8) / 2 + (lane & 3); smem_O_u32[row0 * O_U32_STRIDE + col2] = pack_bf16x2(o_acc[nt][0] * inv_l0, o_acc[nt][1] * inv_l0); smem_O_u32[row1 * O_U32_STRIDE + col2] = pack_bf16x2(o_acc[nt][2] * inv_l1, o_acc[nt][3] * inv_l1); } __syncthreads(); uint4* smem_O_u4 = reinterpret_cast(smem_raw); uint4* g_po_u4 = reinterpret_cast(g_mla_part_o) + static_cast(bid) * PART_O_STRIDE_U4; #pragma unroll for (int pass = 0; pass < 4; ++pass) { int idx = tid + pass * BLOCK_THREADS; int hh = idx >> 6; int cc = idx & 63; st_u4_stream(g_po_u4 + idx, smem_O_u4[hh * 65 + cc], pol_stream); } // Write part_ml LAST with pol_keep so it stays 100% L2-hot for A4! if (tid < 32) { st_f32x2_keep(g_mla_part_ml + tid * ML_STRIDE + bid, my_ml, pol_keep); } } grid.sync(); // Phase A4: OVERLAPPED part_o prefetch + 1-warp softmax + OVERLAPPED W_uv! if (bid < 128) { int h = bid >> 2; int g = bid & 3; if (tid < 32) reinterpret_cast(g_q_full_f32)[bid * 32 + tid] = make_uint4(0u, 0u, 0u, 0u); float* smem_part_lat = reinterpret_cast(smem_raw); float* smem_olat = smem_part_lat + 32 * LAT_STRIDE; float* smem_olat_sum32 = smem_olat + 4 * X_F32_STRIDE; float* smem_w = smem_olat_sum32 + 16; // Pre-issue up to 4 splits (<=128 total) of g_mla_part_o across all 16 warps before softmax int sg = tid >> 4; int d8 = tid & 15; const uint4* po_u4 = reinterpret_cast(g_mla_part_o) + (static_cast(h) * 64) + g * 16 + d8; int s0 = sg, s1 = sg + 32, s2 = sg + 64, s3 = sg + 96; uint4 v0 = (s0 < num_splits) ? ld_u4_stream(po_u4 + static_cast(s0) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u); uint4 v1 = (s1 < num_splits) ? ld_u4_stream(po_u4 + static_cast(s1) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u); uint4 v2 = (s2 < num_splits) ? ld_u4_stream(po_u4 + static_cast(s2) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u); uint4 v3 = (s3 < num_splits) ? ld_u4_stream(po_u4 + static_cast(s3) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u); if (wid == 0) { const float2* ml_head = g_mla_part_ml + h * ML_STRIDE; float2 ml_reg[4]; #pragma unroll for (int k = 0; k < 4; ++k) { int s = k * 32 + lane; ml_reg[k] = (s < num_splits) ? ld_f32x2_keep(ml_head + s, pol_keep) : make_float2(-1e30f, 0.0f); } float lm = fmaxf(fmaxf(ml_reg[0].x, ml_reg[1].x), fmaxf(ml_reg[2].x, ml_reg[3].x)); #pragma unroll for (int off = 16; off > 0; off >>= 1) lm = fmaxf(lm, __shfl_xor_sync(0xffffffffu, lm, off)); float lw = 0.0f; #pragma unroll for (int k = 0; k < 4; ++k) { int s = k * 32 + lane; float w = (s < num_splits) ? (ml_reg[k].y * exp2f(ml_reg[k].x - lm)) : 0.0f; ml_reg[k].y = w; lw += w; } float inv_l = 1.0f / warp_reduce_sum(lw); #pragma unroll for (int k = 0; k < 4; ++k) { int s = k * 32 + lane; if (s < num_splits) smem_w[s] = ml_reg[k].y * inv_l; } } __syncthreads(); { float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f; float a4 = 0.0f, a5 = 0.0f, a6 = 0.0f, a7 = 0.0f; auto acc_v = [&](int s, uint4 v) { if (s < num_splits) { float w = smem_w[s]; a0 = fmaf(w, __uint_as_float(v.x << 16), a0); a1 = fmaf(w, __uint_as_float(v.x & 0xFFFF0000u), a1); a2 = fmaf(w, __uint_as_float(v.y << 16), a2); a3 = fmaf(w, __uint_as_float(v.y & 0xFFFF0000u), a3); a4 = fmaf(w, __uint_as_float(v.z << 16), a4); a5 = fmaf(w, __uint_as_float(v.z & 0xFFFF0000u), a5); a6 = fmaf(w, __uint_as_float(v.w << 16), a6); a7 = fmaf(w, __uint_as_float(v.w & 0xFFFF0000u), a7); } }; acc_v(s0, v0); acc_v(s1, v1); acc_v(s2, v2); acc_v(s3, v3); float* dst = smem_part_lat + sg * LAT_STRIDE + d8 * 8; *reinterpret_cast(dst) = make_float4(a0, a1, a2, a3); *reinterpret_cast(dst + 4) = make_float4(a4, a5, a6, a7); } // Pre-issue W_uv load BEFORE olat reduction (hides W_uv L2 latency completely!) int d_out = tid >> 2; int t4 = tid & 3; int k32 = g * 4 + t4; const uint4* wuv_row = reinterpret_cast(mla.wuv_wq) + (static_cast(h) * V_HEAD + d_out) * 16; const uint32_t* sz_row = reinterpret_cast(mla.wuv_sz) + (static_cast(h) * V_HEAD + d_out) * 4; uint4 w_uv = ld_u4_keep(wuv_row + k32, pol_keep); uint32_t sz_uv = ld_u32_keep(sz_row + g, pol_keep); __syncthreads(); { int d = d_out; int q4 = t4; float s = 0.0f; #pragma unroll for (int r = 0; r < 8; ++r) { s += smem_part_lat[(q4 * 8 + r) * LAT_STRIDE + d]; } s = subwarp4_reduce_sum(s); if (q4 == 0) { smem_olat[(d >> 5) * X_F32_STRIDE + (d & 31)] = s; } } __syncthreads(); if (wid < 4) { float s32 = warp_reduce_sum(smem_olat[wid * X_F32_STRIDE + lane]); if (lane == 0) smem_olat_sum32[wid] = s32; } __syncthreads(); { float acc = subwarp4_reduce_sum(dot32_w4_f32(w_uv, sz_uv, smem_olat + t4 * X_F32_STRIDE, smem_olat_sum32[t4])); if (t4 == 0) atomicAdd(&g_attn_o_f32[h * V_HEAD + d_out], acc); } } grid.sync(); run_load_and_oproj( g_attn_o_f32, reinterpret_cast(mla.o_wq), reinterpret_cast(mla.o_sz), g_hidden, g_norm_sq, smem_x_u4, smem_x_sum32, smem_red16, bid, nblocks, pol_keep ); grid.sync(); } // --------------------------------------------------------------------------- // THE SINGLE MEGAKERNEL V11 // --------------------------------------------------------------------------- __global__ __launch_bounds__(BLOCK_THREADS, 1) void kimi_decode_megakernel(const MegaParams p) { extern __shared__ char smem_raw[]; cg::grid_group grid = cg::this_grid(); int bid = blockIdx.x; int nblocks = gridDim.x; uint64_t pol_keep, pol_stream; if (p.need_copy) asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 0.20;" : "=l"(pol_keep)); else asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;" : "=l"(pol_keep)); asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(pol_stream)); __nv_bfloat16* g_hidden = reinterpret_cast<__nv_bfloat16*>(p.g_hidden); __nv_bfloat16* g_attn_o = reinterpret_cast<__nv_bfloat16*>(p.g_attn_o); __nv_bfloat16* g_moe_hh = reinterpret_cast<__nv_bfloat16*>(p.g_moe_hh); #pragma unroll 1 for (int layer = 0; layer < 3; ++layer) { run_kda_attn( grid, p.kda[layer], p.kda_S[layer], reinterpret_cast<__nv_bfloat16*>(p.kda_cq[layer]), reinterpret_cast<__nv_bfloat16*>(p.kda_ck[layer]), reinterpret_cast<__nv_bfloat16*>(p.kda_cv[layer]), (layer == 0) ? reinterpret_cast(p.in_hidden) : nullptr, g_hidden, p.g_norm_sq, p.g_q, p.g_k, p.g_v, p.g_decay, p.g_beta, g_attn_o, smem_raw, bid, nblocks, pol_keep ); run_moe_layer( grid, p.moe[layer], g_hidden, nullptr, p.g_norm_sq, p.g_router_logits, &p.g_router_done[layer], g_moe_hh, p.routed_scaling, p.n_experts, smem_raw, bid, nblocks, pol_keep, pol_stream ); grid.sync(); } run_mla_attn( grid, p.mla, reinterpret_cast(p.src_c_kv), reinterpret_cast(p.src_k_rope), reinterpret_cast<__nv_bfloat16*>(p.dst_c_kv), reinterpret_cast<__nv_bfloat16*>(p.dst_k_rope), p.ctx_len, p.need_copy, p.rope_theta, g_hidden, p.g_norm_sq, p.g_q_full_f32, reinterpret_cast<__nv_bfloat16*>(p.g_q_rope), reinterpret_cast(p.g_mla_part_ml), reinterpret_cast<__nv_bfloat16*>(p.g_mla_part_o), p.g_attn_o_f32, smem_raw, bid, nblocks, pol_keep, pol_stream ); run_moe_layer( grid, p.moe[3], g_hidden, reinterpret_cast<__nv_bfloat16*>(p.out_hidden), p.g_norm_sq, p.g_router_logits, &p.g_router_done[3], g_moe_hh, p.routed_scaling, p.n_experts, smem_raw, bid, nblocks, pol_keep, pol_stream ); } extern "C" void launch_kimi_megakernel( const MegaParams* h_params, void* /*dense_arena_ptr*/, size_t /*dense_arena_bytes*/, cudaStream_t stream ) { static int s_sm_count = 0; static bool s_attr_set = false; if (!s_attr_set) { cudaDeviceProp prop; cudaGetDeviceProperties(&prop, 0); s_sm_count = prop.multiProcessorCount; cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, prop.persistingL2CacheMaxSize); cudaFuncSetAttribute(kimi_decode_megakernel, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_BYTES); s_attr_set = true; } MegaParams p = *h_params; void* args[] = { &p }; cudaLaunchCooperativeKernel( (void*)kimi_decode_megakernel, dim3(s_sm_count), dim3(BLOCK_THREADS), args, SMEM_BYTES, stream ); } """ _EXT = None def _get_ext(): global _EXT if _EXT is None: _EXT = load_inline( name="kimi_linear_megakernel_v11", cpp_sources=_CPP_SRC, cuda_sources=_CUDA_SRC, functions=["create_handle", "destroy_handle", "step_megakernel"], extra_cuda_cflags=[ "-O3", "-gencode=arch=compute_120,code=sm_120", "--expt-relaxed-constexpr", ], no_implicit_headers=True, ) return _EXT def _pack_sz(scales: torch.Tensor, zeros: torch.Tensor) -> torch.Tensor: s_u16 = scales.contiguous().view(torch.uint16).to(torch.int32) z_u16 = zeros.contiguous().view(torch.uint16).to(torch.int32) return (s_u16 | (z_u16 << 16)).contiguous() def _pack_linear(ql: QuantLinear): wq_t = ql.w_q.transpose(0, 1).contiguous() sz_t = _pack_sz(ql.scales.transpose(0, 1), ql.zeros.transpose(0, 1)) return wq_t, sz_t def _pack_linear_cat(qls: list[QuantLinear]): wqs = [q.w_q.transpose(0, 1).contiguous() for q in qls] szs = [_pack_sz(q.scales.transpose(0, 1), q.zeros.transpose(0, 1)) for q in qls] return torch.cat(wqs, dim=0).contiguous(), torch.cat(szs, dim=0).contiguous() def _pack_bf16_interleave8(w: torch.Tensor) -> torch.Tensor: orig_shape = w.shape return w.contiguous().view(*orig_shape[:-1], orig_shape[-1] // 8, 2, 4).transpose(-1, -2).contiguous().view(orig_shape) def _pack_into_arena(tensors: list[torch.Tensor], dev: torch.device) -> tuple[torch.Tensor, list[torch.Tensor]]: offsets = [] cur = 0 for t in tensors: nbytes = t.numel() * t.element_size() offsets.append((cur, nbytes, t.shape, t.dtype)) cur = (cur + nbytes + 127) & ~127 arena = torch.empty(cur, dtype=torch.uint8, device=dev) views = [] for t, (off, nbytes, shape, dtype) in zip(tensors, offsets): v = arena[off : off + nbytes].view(dtype).view(shape) v.copy_(t) views.append(v) return arena, views class Model(nn.Module): def __init__(self, cfg: Config): super().__init__() self.cfg = cfg self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern) self._handle = 0 self._packed_tensors = None self._dense_arena = None self._scratch = None self._packed_device = None self._last_kv_ptr = 0 self._last_kr_ptr = 0 self._last_kv_buf = None self._last_kr_buf = None def __del__(self): if getattr(self, "_handle", 0) != 0 and _EXT is not None: try: _EXT.destroy_handle(self._handle) except Exception: pass self._handle = 0 def _invalidate_packed(self): if self._handle != 0 and _EXT is not None: _EXT.destroy_handle(self._handle) self._handle = 0 self._packed_tensors = None self._dense_arena = None self._scratch = None self._packed_device = None def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False): res = super().load_state_dict(state_dict, strict=strict, assign=assign) self._invalidate_packed() if next(self.parameters()).is_cuda: self._build_packed() return res def _build_packed(self): dev = next(self.parameters()).device ext = _get_ext() cfg = self.cfg dense_list = [] for i in range(3): blk = self.blocks[i] kda = blk.attn qkvg_wq, qkvg_sz = _pack_linear_cat([kda.q_proj, kda.k_proj, kda.v_proj, kda.g_proj]) o_wq, o_sz = _pack_linear(kda.o_proj) dense_list.extend([ blk.attn_norm.contiguous(), qkvg_wq, qkvg_sz, _pack_bf16_interleave8(kda.beta_proj.weight), kda.conv_w.contiguous(), o_wq, o_sz, ]) blk3 = self.blocks[3] mla = blk3.attn MLA_H = cfg.mla_heads q_wq_t = mla.q_proj.w_q.transpose(0, 1).contiguous().view(MLA_H, cfg.qk_nope + cfg.qk_rope, -1) q_sz_t = _pack_sz(mla.q_proj.scales.transpose(0, 1), mla.q_proj.zeros.transpose(0, 1)).view(MLA_H, cfg.qk_nope + cfg.qk_rope, -1) kva_wq_t = mla.kv_a.w_q.transpose(0, 1).contiguous() kva_sz_t = _pack_sz(mla.kv_a.scales.transpose(0, 1), mla.kv_a.zeros.transpose(0, 1)) qkva_wq = torch.cat([ q_wq_t[:, : cfg.qk_nope, :].reshape(MLA_H * cfg.qk_nope, -1), q_wq_t[:, cfg.qk_nope :, :].reshape(MLA_H * cfg.qk_rope, -1), kva_wq_t, ], dim=0).contiguous() qkva_sz = torch.cat([ q_sz_t[:, : cfg.qk_nope, :].reshape(MLA_H * cfg.qk_nope, -1), q_sz_t[:, cfg.qk_nope :, :].reshape(MLA_H * cfg.qk_rope, -1), kva_sz_t, ], dim=0).contiguous() # Pure int4 nibble transpose of W_uk from packing-along-kv_lora to packing-along-q_nope w_raw = mla.kv_b.w_q.view(256, MLA_H, 256)[:, :, :128] u_r = torch.stack([w_raw & 0xF, w_raw >> 4], dim=1).reshape(512, MLA_H, 4, 16, 2) wuk_wq = (u_r[..., 0] | (u_r[..., 1] << 4)).permute(1, 2, 0, 3).contiguous() kvb_s = mla.kv_b.scales.view(4, MLA_H, 256) kvb_z = mla.kv_b.zeros.view(4, MLA_H, 256) wuk_sz = _pack_sz( kvb_s[:, :, :128].view(4, MLA_H, 4, 32).permute(1, 2, 0, 3), kvb_z[:, :, :128].view(4, MLA_H, 4, 32).permute(1, 2, 0, 3), ) wuv_wq = mla.kv_b.w_q.view(256, MLA_H, 256)[:, :, 128:].permute(1, 2, 0).contiguous() wuv_sz = _pack_sz( kvb_s[:, :, 128:].permute(1, 2, 0), kvb_z[:, :, 128:].permute(1, 2, 0), ) mla_o_wq, mla_o_sz = _pack_linear(mla.o_proj) dense_list.extend([ blk3.attn_norm.contiguous(), qkva_wq, qkva_sz, wuk_wq, wuk_sz, wuv_wq, wuv_sz, mla_o_wq, mla_o_sz, ]) routed_list = [] for i in range(4): blk = self.blocks[i] moe = blk.moe sg_wq = moe.s_gate.w_q[0].transpose(0, 1) su_wq = moe.s_up.w_q[0].transpose(0, 1) s_gu_wq = torch.stack([sg_wq, su_wq], dim=1).contiguous() s_gu_sz = torch.stack([ _pack_sz(moe.s_gate.scales[0].transpose(0, 1), moe.s_gate.zeros[0].transpose(0, 1)), _pack_sz(moe.s_up.scales[0].transpose(0, 1), moe.s_up.zeros[0].transpose(0, 1)), ], dim=1).contiguous() s_d_wq = moe.s_down.w_q[0].transpose(0, 1).contiguous() s_d_sz = _pack_sz(moe.s_down.scales[0].transpose(0, 1), moe.s_down.zeros[0].transpose(0, 1)) dense_list.extend([ blk.moe_norm.contiguous(), _pack_bf16_interleave8(moe.router.weight), s_gu_wq, s_gu_sz, s_d_wq, s_d_sz, ]) rg_wq = moe.gate.w_q.transpose(1, 2) ru_wq = moe.up.w_q.transpose(1, 2) r_gu_wq = torch.stack([rg_wq, ru_wq], dim=2).contiguous() r_gu_sz = torch.stack([ _pack_sz(moe.gate.scales.transpose(1, 2), moe.gate.zeros.transpose(1, 2)), _pack_sz(moe.up.scales.transpose(1, 2), moe.up.zeros.transpose(1, 2)), ], dim=2).contiguous() r_d_wq = moe.down.w_q.transpose(1, 2).contiguous() r_d_sz = _pack_sz(moe.down.scales.transpose(1, 2), moe.down.zeros.transpose(1, 2)) routed_list.extend([r_gu_wq, r_gu_sz, r_d_wq, r_d_sz]) arena, dense_views = _pack_into_arena(dense_list, dev) kda_tensors = dense_views[0:21] mla_tensors = dense_views[21:30] moe_tensors = [] for i in range(4): moe_tensors.extend(dense_views[30 + i * 6 : 30 + (i + 1) * 6]) moe_tensors.extend(routed_list[i * 4 : (i + 1) * 4]) scratch = [ torch.empty(cfg.hidden, dtype=torch.bfloat16, device=dev), # 0: g_hidden torch.zeros(1, dtype=torch.float32, device=dev), # 1: g_norm_sq torch.empty(4096, dtype=torch.float32, device=dev), # 2: g_q torch.empty(4096, dtype=torch.float32, device=dev), # 3: g_k torch.empty(4096, dtype=torch.float32, device=dev), # 4: g_v torch.empty(4096, dtype=torch.float32, device=dev), # 5: g_decay torch.empty(32, dtype=torch.float32, device=dev), # 6: g_beta torch.empty(4096, dtype=torch.bfloat16, device=dev), # 7: g_attn_o torch.empty(4096, dtype=torch.float32, device=dev), # 8: g_attn_o_f32 torch.zeros(32 * 512, dtype=torch.float32, device=dev), # 9: g_q_full_f32 torch.empty(32 * 64, dtype=torch.bfloat16, device=dev), # 10: g_q_rope torch.empty(32 * 192 * 2, dtype=torch.float32, device=dev), # 11: g_mla_part_ml (float2) torch.empty(188 * 2056 * 8, dtype=torch.bfloat16, device=dev), # 12: g_mla_part_o (padded stride!) torch.empty(cfg.n_experts, dtype=torch.float32, device=dev), # 13: g_router_logits torch.zeros(4, dtype=torch.int32, device=dev), # 14: g_router_done torch.empty(9 * cfg.moe_inter, dtype=torch.bfloat16, device=dev), # 15: g_moe_hh ] self._dense_arena = arena self._packed_tensors = dense_views + routed_list self._scratch = scratch self._packed_device = dev self._handle = ext.create_handle( kda_tensors, mla_tensors, moe_tensors, scratch, arena, float(cfg.routed_scaling), float(cfg.rope_theta), int(cfg.n_experts), ) def step(self, hidden: torch.Tensor, state: list): if self._handle == 0 or self._packed_device != hidden.device: self._build_packed() s0, s1, s2, s3 = state[0], state[1], state[2], state[3] src_ckv = s3["c_kv"] src_kr = s3["k_rope"] ctx_len = src_ckv.shape[0] if ( self._last_kv_buf is not None and src_ckv.data_ptr() == self._last_kv_ptr and src_kr.data_ptr() == self._last_kr_ptr and ctx_len < self._last_kv_buf.shape[0] ): dst_ckv = self._last_kv_buf dst_kr = self._last_kr_buf need_copy = 0 else: cap = ctx_len + 64 dst_ckv = torch.empty((cap, src_ckv.shape[1]), dtype=src_ckv.dtype, device=src_ckv.device) dst_kr = torch.empty((cap, src_kr.shape[1]), dtype=src_kr.dtype, device=src_kr.device) self._last_kv_buf = dst_ckv self._last_kr_buf = dst_kr self._last_kv_ptr = dst_ckv.data_ptr() self._last_kr_ptr = dst_kr.data_ptr() need_copy = 1 out = _EXT.step_megakernel( self._handle, hidden, s0["S"], s0["cq"], s0["ck"], s0["cv"], s1["S"], s1["cq"], s1["ck"], s1["cv"], s2["S"], s2["cq"], s2["ck"], s2["cv"], src_ckv, src_kr, dst_ckv, dst_kr, ctx_len, need_copy, ) s3["c_kv"] = dst_ckv[: ctx_len + 1] s3["k_rope"] = dst_kr[: ctx_len + 1] return out, state