KernelBench mega · RTX PRO 6000
Kimi-Linear Decode Gemini 4 Neon (XHigh)
Gemini 4 Neon (XHigh) wrote a single-launch cooperative SM120 CUDA megakernel (188 CTAs x 512 threads) that scores 36.655x on Kimi-Linear decode (0.157 / 0.166 / 0.176 ms/tok at 2k / 8k / 16k, geomean 6,010 tok/s), taking #1 overall ahead of Claude Opus 5.5 (35.34x). It unpacks centered [-8..+7] int4 weights directly into native SM120 bf16x2 FMA2 (lop3 + sub.rn.bf16x2 + fma.rn.bf16x2) with the zero point folded once per 32-element group, uses PTX release/acquire router barriers with redux.sync top-8 selection, runs FlashMLA on bf16 mma.sync tensor cores with cp.async KV pipelining and adaptive Blackwell persisting-L2 policies, and copies step-0 KV caches inside the single cooperative launch. Verdict is interesting for intra-group bf16x2 accumulation, an unguarded *g_norm_sq reset before grid.sync(), and load_state_dict-only weight repacking.
Kernel source (redacted)
"""Single-kernel W4A16 hybrid decode megakernel for Kimi-Linear (3x KDA + 1x MLA + 4x MoE) on SM120 Blackwell."""
from __future__ import annotations
from dataclasses import dataclass, field
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
OP_TYPE = "kimi_linear_w4a16_decode"
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
EPS = 1.0e-6
GROUP_SIZE = 128
@dataclass(frozen=True)
class Config:
hidden: int = 2304
kda_heads: int = 32
kda_head_dim: int = 128
short_conv: int = 4
mla_heads: int = 32
kv_lora: int = 512
qk_nope: int = 128
qk_rope: int = 64
v_head: int = 128
rope_theta: float = 10000.0
n_experts: int = 64
n_active: int = 8
n_shared: int = 1
moe_inter: int = 1024
routed_scaling: float = 2.446
group: int = 128
pattern: tuple = ("K", "K", "K", "M")
dtype: torch.dtype = field(default=torch.bfloat16)
def build_config(shape: dict) -> Config:
return Config(n_experts=int(shape.get("n_experts", 64)))
def _pack_int4(w_q: torch.Tensor) -> torch.Tensor:
lo = w_q[0::2] & 0xF
hi = w_q[1::2] & 0xF
return (lo | (hi << 4)).contiguous()
def quantize(w_io: torch.Tensor, group: int = GROUP_SIZE):
K, N = w_io.shape
ng = K // group
wg = w_io.view(ng, group, N).float()
wmin = wg.min(dim=1, keepdim=True).values
wmax = wg.max(dim=1, keepdim=True).values
scales = (wmax - wmin).clamp_min(1e-8) / 15.0
zeros = (-wmin / scales).round().clamp(0, 15)
w_q = ((wg / scales) + zeros).round().clamp(0, 15).to(torch.uint8).view(K, N)
return _pack_int4(w_q), scales.squeeze(1).to(torch.bfloat16), zeros.squeeze(1).to(torch.bfloat16)
class QuantLinear(nn.Module):
def __init__(self, in_f: int, out_f: int, group: int = GROUP_SIZE):
super().__init__()
assert in_f % group == 0 and in_f % 2 == 0
self.in_f, self.out_f, self.group = in_f, out_f, group
ng = in_f // group
self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8))
self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16))
class QuantExperts(nn.Module):
def __init__(self, n: int, in_f: int, out_f: int, group: int = GROUP_SIZE):
super().__init__()
self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group
ng = in_f // group
self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8))
self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
class KDA(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden
self.q_proj = QuantLinear(d, H * Dk, cfg.group)
self.k_proj = QuantLinear(d, H * Dk, cfg.group)
self.v_proj = QuantLinear(d, H * Dk, cfg.group)
self.g_proj = QuantLinear(d, H * Dk, cfg.group)
self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype)
self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype))
self.o_proj = QuantLinear(H * Dk, d, cfg.group)
self.scale = Dk ** -0.5
class MLA(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
H, d = cfg.mla_heads, cfg.hidden
self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group)
self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group)
self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group)
self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group)
self.scale = (cfg.qk_nope + cfg.qk_rope) ** -0.5
class MoE(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts
self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype)
self.gate = QuantExperts(E, d, m, cfg.group)
self.up = QuantExperts(E, d, m, cfg.group)
self.down = QuantExperts(E, m, d, cfg.group)
self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group)
self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group)
self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group)
class Block(nn.Module):
def __init__(self, cfg: Config, kind: str):
super().__init__()
self.kind = kind
self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
self.attn = KDA(cfg) if kind == "K" else MLA(cfg)
self.moe = MoE(cfg)
# --------------------------------------------------------------------------- #
# Single-Kernel CUDA Megakernel V11 (compiled via load_inline):
# - 188 x 512 cooperative grid (16 warps/SM, 128 regs, 0 stack, 0 spills, 0 MATCH.ANY)
# - 80 MiB L2 persisting pool (cudaLimitPersistingL2CacheSize) + adaptive Step-0/steady L2 policy
# - Native SM120 centered-int4 BF16x2 FMA2 GEMV with L2::256B prefetch hints
# - Input-first overlapped o_proj (12-13 rows/SM, zero extra atomics)
# - PRE=2 software-pipelined KDA qkvg + overlapped coalesced conv preload on warps 12..14
# - MoE fused router + shared gate/up + PTX release/acquire barrier + 4-expert pre-issued M3 down
# - Spin-free MLA A1+A2 (nibble-transposed int4 W_uk), async cp.async double-buffered Step-0 KV copy,
# exp2f FlashMLA A3 (64/96/128 splits + streaming part_o), and slim overlapped A4
# --------------------------------------------------------------------------- #
_CPP_SRC = r"""
#include <torch/extension.h>
#include <c10/cuda/CUDAStream.h>
#include <cstdint>
#include <vector>
struct KdaLayerPtrs {
const void* attn_norm;
const void* qkvg_wq;
const void* qkvg_sz;
const void* beta_w;
const void* conv_w;
const void* o_wq;
const void* o_sz;
};
struct MlaLayerPtrs {
const void* attn_norm;
const void* qkva_wq;
const void* qkva_sz;
const void* wuk_wq;
const void* wuk_sz;
const void* wuv_wq;
const void* wuv_sz;
const void* o_wq;
const void* o_sz;
};
struct MoeLayerPtrs {
const void* moe_norm;
const void* router_w;
const void* s_gate_up_wq;
const void* s_gate_up_sz;
const void* s_down_wq;
const void* s_down_sz;
const void* r_gate_up_wq;
const void* r_gate_up_sz;
const void* r_down_wq;
const void* r_down_sz;
};
struct MegaParams {
KdaLayerPtrs kda[3];
MlaLayerPtrs mla;
MoeLayerPtrs moe[4];
float* kda_S[3];
void* kda_cq[3];
void* kda_ck[3];
void* kda_cv[3];
const void* src_c_kv;
const void* src_k_rope;
void* dst_c_kv;
void* dst_k_rope;
int ctx_len;
int need_copy;
const void* in_hidden;
void* out_hidden;
void* g_hidden;
float* g_norm_sq;
float* g_q;
float* g_k;
float* g_v;
float* g_decay;
float* g_beta;
void* g_attn_o; // [4096] bf16 (KDA)
float* g_attn_o_f32; // [4096] f32 (MLA Phase A4 atomic accum)
float* g_q_full_f32; // [32*512] f32 (MLA spin-free W_uk accum)
void* g_q_rope; // [32*64] bf16 (MLA q_rope)
void* g_mla_part_ml; // [32 * 192] float2 (coalesced across splits!)
void* g_mla_part_o; // [188 * 2056 * 8] bf16 (padded stride!)
float* g_router_logits;
unsigned int* g_router_done; // [4] u32
void* g_moe_hh; // [9*1024] bf16
float routed_scaling;
float rope_theta;
int n_experts;
};
extern "C" void launch_kimi_megakernel(
const MegaParams* h_params,
void* dense_arena_ptr,
size_t dense_arena_bytes,
cudaStream_t stream
);
struct PackedHandle {
MegaParams base;
void* dense_arena_ptr;
size_t dense_arena_bytes;
};
int64_t create_handle(
std::vector<torch::Tensor> kda_tensors,
std::vector<torch::Tensor> mla_tensors,
std::vector<torch::Tensor> moe_tensors,
std::vector<torch::Tensor> scratch,
torch::Tensor dense_arena,
double routed_scaling,
double rope_theta,
int64_t n_experts
) {
auto* h = new PackedHandle();
h->dense_arena_ptr = dense_arena.data_ptr();
h->dense_arena_bytes = static_cast<size_t>(dense_arena.numel()) * dense_arena.element_size();
for (int i = 0; i < 3; ++i) {
h->base.kda[i].attn_norm = kda_tensors[i * 7 + 0].data_ptr();
h->base.kda[i].qkvg_wq = kda_tensors[i * 7 + 1].data_ptr();
h->base.kda[i].qkvg_sz = kda_tensors[i * 7 + 2].data_ptr();
h->base.kda[i].beta_w = kda_tensors[i * 7 + 3].data_ptr();
h->base.kda[i].conv_w = kda_tensors[i * 7 + 4].data_ptr();
h->base.kda[i].o_wq = kda_tensors[i * 7 + 5].data_ptr();
h->base.kda[i].o_sz = kda_tensors[i * 7 + 6].data_ptr();
}
h->base.mla.attn_norm = mla_tensors[0].data_ptr();
h->base.mla.qkva_wq = mla_tensors[1].data_ptr();
h->base.mla.qkva_sz = mla_tensors[2].data_ptr();
h->base.mla.wuk_wq = mla_tensors[3].data_ptr();
h->base.mla.wuk_sz = mla_tensors[4].data_ptr();
h->base.mla.wuv_wq = mla_tensors[5].data_ptr();
h->base.mla.wuv_sz = mla_tensors[6].data_ptr();
h->base.mla.o_wq = mla_tensors[7].data_ptr();
h->base.mla.o_sz = mla_tensors[8].data_ptr();
for (int i = 0; i < 4; ++i) {
h->base.moe[i].moe_norm = moe_tensors[i * 10 + 0].data_ptr();
h->base.moe[i].router_w = moe_tensors[i * 10 + 1].data_ptr();
h->base.moe[i].s_gate_up_wq = moe_tensors[i * 10 + 2].data_ptr();
h->base.moe[i].s_gate_up_sz = moe_tensors[i * 10 + 3].data_ptr();
h->base.moe[i].s_down_wq = moe_tensors[i * 10 + 4].data_ptr();
h->base.moe[i].s_down_sz = moe_tensors[i * 10 + 5].data_ptr();
h->base.moe[i].r_gate_up_wq = moe_tensors[i * 10 + 6].data_ptr();
h->base.moe[i].r_gate_up_sz = moe_tensors[i * 10 + 7].data_ptr();
h->base.moe[i].r_down_wq = moe_tensors[i * 10 + 8].data_ptr();
h->base.moe[i].r_down_sz = moe_tensors[i * 10 + 9].data_ptr();
}
h->base.g_hidden = scratch[0].data_ptr();
h->base.g_norm_sq = scratch[1].data_ptr<float>();
h->base.g_q = scratch[2].data_ptr<float>();
h->base.g_k = scratch[3].data_ptr<float>();
h->base.g_v = scratch[4].data_ptr<float>();
h->base.g_decay = scratch[5].data_ptr<float>();
h->base.g_beta = scratch[6].data_ptr<float>();
h->base.g_attn_o = scratch[7].data_ptr();
h->base.g_attn_o_f32 = scratch[8].data_ptr<float>();
h->base.g_q_full_f32 = scratch[9].data_ptr<float>();
h->base.g_q_rope = scratch[10].data_ptr();
h->base.g_mla_part_ml = scratch[11].data_ptr();
h->base.g_mla_part_o = scratch[12].data_ptr();
h->base.g_router_logits = scratch[13].data_ptr<float>();
h->base.g_router_done = reinterpret_cast<unsigned int*>(scratch[14].data_ptr<int32_t>());
h->base.g_moe_hh = scratch[15].data_ptr();
h->base.routed_scaling = static_cast<float>(routed_scaling);
h->base.rope_theta = static_cast<float>(rope_theta);
h->base.n_experts = static_cast<int>(n_experts);
return reinterpret_cast<int64_t>(h);
}
void destroy_handle(int64_t ptr) {
if (ptr != 0) delete reinterpret_cast<PackedHandle*>(ptr);
}
torch::Tensor step_megakernel(
int64_t handle_ptr,
torch::Tensor hidden,
torch::Tensor S0, torch::Tensor cq0, torch::Tensor ck0, torch::Tensor cv0,
torch::Tensor S1, torch::Tensor cq1, torch::Tensor ck1, torch::Tensor cv1,
torch::Tensor S2, torch::Tensor cq2, torch::Tensor ck2, torch::Tensor cv2,
torch::Tensor src_c_kv, torch::Tensor src_k_rope,
torch::Tensor dst_c_kv, torch::Tensor dst_k_rope,
int64_t ctx_len,
int64_t need_copy
) {
auto* h = reinterpret_cast<PackedHandle*>(handle_ptr);
auto out = torch::empty_like(hidden);
MegaParams p = h->base;
p.kda_S[0] = S0.data_ptr<float>();
p.kda_cq[0] = cq0.data_ptr();
p.kda_ck[0] = ck0.data_ptr();
p.kda_cv[0] = cv0.data_ptr();
p.kda_S[1] = S1.data_ptr<float>();
p.kda_cq[1] = cq1.data_ptr();
p.kda_ck[1] = ck1.data_ptr();
p.kda_cv[1] = cv1.data_ptr();
p.kda_S[2] = S2.data_ptr<float>();
p.kda_cq[2] = cq2.data_ptr();
p.kda_ck[2] = ck2.data_ptr();
p.kda_cv[2] = cv2.data_ptr();
p.src_c_kv = src_c_kv.data_ptr();
p.src_k_rope = src_k_rope.data_ptr();
p.dst_c_kv = dst_c_kv.data_ptr();
p.dst_k_rope = dst_k_rope.data_ptr();
p.ctx_len = static_cast<int>(ctx_len);
p.need_copy = static_cast<int>(need_copy);
p.in_hidden = hidden.data_ptr();
p.out_hidden = out.data_ptr();
launch_kimi_megakernel(&p, h->dense_arena_ptr, h->dense_arena_bytes, at::cuda::getCurrentCUDAStream().stream());
return out;
}
"""
_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cooperative_groups.h>
#include <cstdint>
#include <cmath>
namespace cg = cooperative_groups;
constexpr int HIDDEN = 2304;
constexpr int KDA_H = 32;
constexpr int KDA_C = 4096;
constexpr int KV_LORA = 512;
constexpr int QK_ROPE = 64;
constexpr int V_HEAD = 128;
constexpr int MOE_INTER = 1024;
constexpr int N_ACTIVE = 8;
constexpr int BLOCK_THREADS = 512;
constexpr int SMEM_BYTES = 86016; // 84 KB
constexpr int X_U4_STRIDE = 5; // 4 uint4s (32 bf16s as 16 bf16x2) + 1 uint4 pad -> 0 bank conflicts!
constexpr int X_F32_STRIDE = 36; // Used only for MLA W_uv (4*36 floats = 576 B)
constexpr int LAT_STRIDE = 132; // Padded 128+4 for MLA A4 smem_part_lat -> 0 bank conflicts!
constexpr int O_SMEM_STRIDE = 520; // Padded 512+8 bf16 (65 uint4s) for FlashMLA smem_O -> 0 bank conflicts!
constexpr int ML_STRIDE = 192; // Coalesced stride for float2 g_mla_part_ml[MLA_H][ML_STRIDE]
constexpr int PART_O_STRIDE_U4 = 2056; // Padded 32*64 + 8 uint4 (32896 B) -> breaks 32 KB L2 set aliasing!
struct KdaLayerPtrs {
const void* attn_norm;
const void* qkvg_wq;
const void* qkvg_sz;
const void* beta_w;
const void* conv_w;
const void* o_wq;
const void* o_sz;
};
struct MlaLayerPtrs {
const void* attn_norm;
const void* qkva_wq;
const void* qkva_sz;
const void* wuk_wq;
const void* wuk_sz;
const void* wuv_wq;
const void* wuv_sz;
const void* o_wq;
const void* o_sz;
};
struct MoeLayerPtrs {
const void* moe_norm;
const void* router_w;
const void* s_gate_up_wq;
const void* s_gate_up_sz;
const void* s_down_wq;
const void* s_down_sz;
const void* r_gate_up_wq;
const void* r_gate_up_sz;
const void* r_down_wq;
const void* r_down_sz;
};
struct MegaParams {
KdaLayerPtrs kda[3];
MlaLayerPtrs mla;
MoeLayerPtrs moe[4];
float* kda_S[3];
void* kda_cq[3];
void* kda_ck[3];
void* kda_cv[3];
const void* src_c_kv;
const void* src_k_rope;
void* dst_c_kv;
void* dst_k_rope;
int ctx_len;
int need_copy;
const void* in_hidden;
void* out_hidden;
void* g_hidden;
float* g_norm_sq;
float* g_q;
float* g_k;
float* g_v;
float* g_decay;
float* g_beta;
void* g_attn_o;
float* g_attn_o_f32;
float* g_q_full_f32;
void* g_q_rope;
void* g_mla_part_ml;
void* g_mla_part_o;
float* g_router_logits;
unsigned int* g_router_done;
void* g_moe_hh;
float routed_scaling;
float rope_theta;
int n_experts;
};
// ---------------------------------------------------------------------------
// Low-level PTX helpers & L2 cache policies
// ---------------------------------------------------------------------------
__device__ __forceinline__ float bf16_to_f32(__nv_bfloat16 v) {
return __bfloat162float(v);
}
__device__ __forceinline__ __nv_bfloat16 f32_to_bf16(float v) {
return __float2bfloat16(v);
}
__device__ __forceinline__ uint32_t pack_bf16x2(float lo, float hi) {
__nv_bfloat162 v = __floats2bfloat162_rn(lo, hi);
return *reinterpret_cast<uint32_t*>(&v);
}
__device__ __forceinline__ float silu_f32(float x) {
if (x >= 0.0f) return x / (1.0f + __expf(-x));
float e = __expf(x);
return x * (e / (1.0f + e));
}
__device__ __forceinline__ float sigmoid_f32(float x) {
if (x >= 0.0f) return 1.0f / (1.0f + __expf(-x));
float e = __expf(x);
return e / (1.0f + e);
}
__device__ __forceinline__ uint32_t float_to_sortable_u32(float f) {
uint32_t u = __float_as_uint(f);
uint32_t mask = static_cast<uint32_t>(-static_cast<int32_t>(u >> 31)) | 0x80000000u;
return u ^ mask;
}
__device__ __forceinline__ uint4 ld_u4_keep(const uint4* ptr, uint64_t pol) {
uint4 v;
asm volatile("ld.global.L2::256B.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ uint4 ld_u4_stream(const uint4* ptr, uint64_t pol) {
uint4 v;
asm volatile("ld.global.L1::evict_first.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ uint4 ld_u4_stream256(const uint4* ptr, uint64_t pol) {
uint4 v;
asm volatile("ld.global.L1::evict_first.L2::256B.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ uint4 ld_u4_ca(const uint4* ptr) {
uint4 v;
asm volatile("ld.global.ca.v4.u32 {%0, %1, %2, %3}, [%4];"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr));
return v;
}
__device__ __forceinline__ uint2 ld_u2_keep(const uint2* ptr, uint64_t pol) {
uint2 v;
asm volatile("ld.global.L2::cache_hint.v2.u32 {%0, %1}, [%2], %3;"
: "=r"(v.x), "=r"(v.y)
: "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ void st_u4_stream(uint4* ptr, uint4 v, uint64_t pol) {
asm volatile("st.global.cs.L2::cache_hint.v4.u32 [%0], {%1, %2, %3, %4}, %5;"
:: "l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w), "l"(pol));
}
__device__ __forceinline__ uint32_t ld_u32_keep(const uint32_t* ptr, uint64_t pol) {
uint32_t v;
asm volatile("ld.global.L2::cache_hint.u32 %0, [%1], %2;" : "=r"(v) : "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ uint32_t ld_u32_stream(const uint32_t* ptr, uint64_t pol) {
uint32_t v;
asm volatile("ld.global.L1::evict_first.L2::cache_hint.u32 %0, [%1], %2;" : "=r"(v) : "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ float ld_f32_keep(const float* ptr, uint64_t pol) {
float v;
asm volatile("ld.global.L2::cache_hint.f32 %0, [%1], %2;" : "=f"(v) : "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ void st_f32_keep(float* ptr, float val, uint64_t pol) {
asm volatile("st.global.L2::cache_hint.f32 [%0], %1, %2;" :: "l"(ptr), "f"(val), "l"(pol));
}
__device__ __forceinline__ void st_f32x2_keep(float2* ptr, float2 val, uint64_t pol) {
asm volatile("st.global.L2::cache_hint.v2.f32 [%0], {%1, %2}, %3;"
:: "l"(ptr), "f"(val.x), "f"(val.y), "l"(pol));
}
__device__ __forceinline__ float2 ld_f32x2_keep(const float2* ptr, uint64_t pol) {
float2 v;
asm volatile("ld.global.L2::cache_hint.v2.f32 {%0, %1}, [%2], %3;"
: "=f"(v.x), "=f"(v.y) : "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ uint16_t ld_u16_keep(const void* ptr, uint64_t pol) {
uint16_t v;
asm volatile("ld.global.L2::cache_hint.u16 %0, [%1], %2;" : "=h"(v) : "l"(ptr), "l"(pol));
return v;
}
__device__ __forceinline__ void st_u16_keep(void* ptr, uint16_t v, uint64_t pol) {
asm volatile("st.global.L2::cache_hint.u16 [%0], %1, %2;" :: "l"(ptr), "h"(v), "l"(pol));
}
// FP32 biased-128 unpack (for MLA W_uk and W_uv)
__device__ __forceinline__ float unpack_biased128(uint32_t v) {
uint32_t r;
asm("lop3.b32 %0, %1, 0x000F0000, 0x43000000, 0xEA;" : "=r"(r) : "r"(v));
return __uint_as_float(r);
}
__device__ __forceinline__ float dot8_biased(uint32_t p, const float* xp) {
float4 x0 = *reinterpret_cast<const float4*>(xp);
float4 x1 = *reinterpret_cast<const float4*>(xp + 4);
float acc = x0.x * unpack_biased128(p << 16);
acc = fmaf(x0.y, unpack_biased128(p << 12), acc);
acc = fmaf(x0.z, unpack_biased128(p << 8), acc);
acc = fmaf(x0.w, unpack_biased128(p << 4), acc);
acc = fmaf(x1.x, unpack_biased128(p), acc);
acc = fmaf(x1.y, unpack_biased128(p >> 4), acc);
acc = fmaf(x1.z, unpack_biased128(p >> 8), acc);
acc = fmaf(x1.w, unpack_biased128(p >> 12), acc);
return acc;
}
__device__ __forceinline__ float dot32_w4_f32(
uint4 w, uint32_t sz, const float* smem_x32, float x_sum32
) {
float dot = dot8_biased(w.x, smem_x32) + dot8_biased(w.y, smem_x32 + 8)
+ dot8_biased(w.z, smem_x32 + 16) + dot8_biased(w.w, smem_x32 + 24);
float scale = __uint_as_float(sz << 16);
float zero = __uint_as_float(sz & 0xFFFF0000u);
return scale * fmaf(-(zero + 128.0f), x_sum32, dot);
}
// Native SM120 BF16x2 centered [-8..+7] int4 unpack + FMA2 (8.33 TB/s L2, 0 overflow, 0 DC cancellation!)
__device__ __forceinline__ uint32_t unpack_bf2_s8(uint32_t v) {
uint32_t b, r;
asm("lop3.b32 %0, %1, 0x000F000F, 0x43004300, 0xEA;" : "=r"(b) : "r"(v));
asm("sub.rn.bf16x2 %0, %1, %2;" : "=r"(r) : "r"(b), "r"(0x43084308u));
return r;
}
__device__ __forceinline__ uint32_t fma_bf2_u32(uint32_t a, uint32_t b, uint32_t c) {
uint32_t r;
asm("fma.rn.bf16x2 %0, %1, %2, %3;" : "=r"(r) : "r"(a), "r"(b), "r"(c));
return r;
}
__device__ __forceinline__ float dot32_bf2(
uint4 w, uint32_t sz, const uint4* sx_u4, float x_sum32
) {
uint32_t acc = 0u;
#pragma unroll
for (int part = 0; part < 4; ++part) {
uint32_t p = (part == 0) ? w.x : ((part == 1) ? w.y : ((part == 2) ? w.z : w.w));
uint4 x4 = sx_u4[part];
acc = fma_bf2_u32(unpack_bf2_s8(p), x4.x, acc);
acc = fma_bf2_u32(unpack_bf2_s8(p >> 4), x4.y, acc);
acc = fma_bf2_u32(unpack_bf2_s8(p >> 8), x4.z, acc);
acc = fma_bf2_u32(unpack_bf2_s8(p >> 12), x4.w, acc);
}
float dot = __uint_as_float(acc << 16) + __uint_as_float(acc & 0xFFFF0000u);
float scale = __uint_as_float(sz << 16);
float zero = __uint_as_float(sz & 0xFFFF0000u);
return scale * fmaf(8.0f - zero, x_sum32, dot);
}
__device__ __forceinline__ void dot32_bf2_pair(
uint4 wa, uint32_t sza, uint4 wb, uint32_t szb,
const uint4* sx_u4, float x_sum32,
float& acc_a, float& acc_b
) {
uint32_t da = 0u, db = 0u;
#pragma unroll
for (int part = 0; part < 4; ++part) {
uint32_t pa = (part == 0) ? wa.x : ((part == 1) ? wa.y : ((part == 2) ? wa.z : wa.w));
uint32_t pb = (part == 0) ? wb.x : ((part == 1) ? wb.y : ((part == 2) ? wb.z : wb.w));
uint4 x4 = sx_u4[part];
da = fma_bf2_u32(unpack_bf2_s8(pa), x4.x, da); db = fma_bf2_u32(unpack_bf2_s8(pb), x4.x, db);
da = fma_bf2_u32(unpack_bf2_s8(pa >> 4), x4.y, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 4), x4.y, db);
da = fma_bf2_u32(unpack_bf2_s8(pa >> 8), x4.z, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 8), x4.z, db);
da = fma_bf2_u32(unpack_bf2_s8(pa >> 12), x4.w, da); db = fma_bf2_u32(unpack_bf2_s8(pb >> 12), x4.w, db);
}
float daf = __uint_as_float(da << 16) + __uint_as_float(da & 0xFFFF0000u);
float dbf = __uint_as_float(db << 16) + __uint_as_float(db & 0xFFFF0000u);
float sa = __uint_as_float(sza << 16), za = __uint_as_float(sza & 0xFFFF0000u);
float sb = __uint_as_float(szb << 16), zb = __uint_as_float(szb & 0xFFFF0000u);
acc_a += sa * fmaf(8.0f - za, x_sum32, daf);
acc_b += sb * fmaf(8.0f - zb, x_sum32, dbf);
}
// Uniform-mask warp/subwarp reductions (100% 0xffffffffu -> ZERO MATCH.ANY!)
__device__ __forceinline__ float warp_reduce_sum(float v) {
#pragma unroll
for (int off = 16; off > 0; off >>= 1) v += __shfl_xor_sync(0xffffffffu, v, off);
return v;
}
__device__ __forceinline__ float subwarp8_reduce_sum(float v) {
v += __shfl_xor_sync(0xffffffffu, v, 4);
v += __shfl_xor_sync(0xffffffffu, v, 2);
v += __shfl_xor_sync(0xffffffffu, v, 1);
return v;
}
__device__ __forceinline__ float subwarp4_reduce_sum(float v) {
v += __shfl_xor_sync(0xffffffffu, v, 2);
v += __shfl_xor_sync(0xffffffffu, v, 1);
return v;
}
__device__ __forceinline__ float halfwarp16_reduce_max(float v) {
v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 8));
v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 4));
v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 2));
v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, 1));
return v;
}
__device__ __forceinline__ float halfwarp16_reduce_sum(float v) {
v += __shfl_xor_sync(0xffffffffu, v, 8);
v += __shfl_xor_sync(0xffffffffu, v, 4);
v += __shfl_xor_sync(0xffffffffu, v, 2);
v += __shfl_xor_sync(0xffffffffu, v, 1);
return v;
}
__device__ __forceinline__ void cp_async_cg_16_pol(void* smem_ptr, const void* glob_ptr, uint64_t pol) {
uint32_t smem_addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
asm volatile("cp.async.cg.shared.global.L2::256B.L2::cache_hint [%0], [%1], 16, %2;\n"
:: "r"(smem_addr), "l"(glob_ptr), "l"(pol));
}
__device__ __forceinline__ void cp_async_commit() {
asm volatile("cp.async.commit_group;\n" ::);
}
template <int N>
__device__ __forceinline__ void cp_async_wait() {
asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}
__device__ __forceinline__ void ldmatrix_x4(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr) {
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
asm volatile("ldmatrix.sync.aligned.x4.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr));
}
__device__ __forceinline__ void ldmatrix_x2(uint32_t& r0, uint32_t& r1, const void* smem_ptr) {
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
asm volatile("ldmatrix.sync.aligned.x2.m8n8.shared.b16 {%0, %1}, [%2];\n"
: "=r"(r0), "=r"(r1) : "r"(addr));
}
__device__ __forceinline__ void ldmatrix_x4_trans(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr) {
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
asm volatile("ldmatrix.sync.aligned.x4.trans.m8n8.shared.b16 {%0, %1, %2, %3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr));
}
__device__ __forceinline__ void mma_m16n8k16_bf16(
float& d0, float& d1, float& d2, float& d3,
uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
uint32_t b0, uint32_t b1
) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
: "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
: "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)
);
}
// ---------------------------------------------------------------------------
// Vectorized BF16x2 SMEM activation loaders
// ---------------------------------------------------------------------------
__device__ __forceinline__ uint4 pack_and_store_rmsnorm8(
int q, uint4 h8, uint4 w8, float inv_rms, uint4* smem_x_u4, float* smem_x_sum32
) {
float r0 = __uint_as_float(h8.x << 16) * inv_rms * __uint_as_float(w8.x << 16);
float r1 = __uint_as_float(h8.x & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.x & 0xFFFF0000u);
float r2 = __uint_as_float(h8.y << 16) * inv_rms * __uint_as_float(w8.y << 16);
float r3 = __uint_as_float(h8.y & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.y & 0xFFFF0000u);
float r4 = __uint_as_float(h8.z << 16) * inv_rms * __uint_as_float(w8.z << 16);
float r5 = __uint_as_float(h8.z & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.z & 0xFFFF0000u);
float r6 = __uint_as_float(h8.w << 16) * inv_rms * __uint_as_float(w8.w << 16);
float r7 = __uint_as_float(h8.w & 0xFFFF0000u) * inv_rms * __uint_as_float(w8.w & 0xFFFF0000u);
uint32_t p0 = pack_bf16x2(r0, r4);
uint32_t p1 = pack_bf16x2(r1, r5);
uint32_t p2 = pack_bf16x2(r2, r6);
uint32_t p3 = pack_bf16x2(r3, r7);
int c = q >> 2;
uint4 x4 = make_uint4(p0, p1, p2, p3);
smem_x_u4[c * X_U4_STRIDE + (q & 3)] = x4;
float s = (__uint_as_float(p0 << 16) + __uint_as_float(p0 & 0xFFFF0000u))
+ (__uint_as_float(p1 << 16) + __uint_as_float(p1 & 0xFFFF0000u))
+ (__uint_as_float(p2 << 16) + __uint_as_float(p2 & 0xFFFF0000u))
+ (__uint_as_float(p3 << 16) + __uint_as_float(p3 & 0xFFFF0000u));
s = subwarp4_reduce_sum(s);
if ((threadIdx.x & 3) == 0) smem_x_sum32[c] = s;
return x4;
}
__device__ __forceinline__ void load_rmsnorm_2304_to_smem_bf2(
const __nv_bfloat16* g_in,
const __nv_bfloat16* norm_w,
float inv_rms,
uint4* smem_x_u4,
float* smem_x_sum32,
int bid,
uint64_t pol_keep
) {
int tid = threadIdx.x;
if (tid < 288) {
int shift = (bid % 72) * 4;
int q = tid + shift;
if (q >= 288) q -= 288;
uint4 h8 = ld_u4_ca(reinterpret_cast<const uint4*>(g_in) + q);
uint4 w8 = ld_u4_keep(reinterpret_cast<const uint4*>(norm_w) + q, pol_keep);
pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32);
}
__syncthreads();
}
__device__ __forceinline__ void load_input_and_rmsnorm_2304_to_smem_bf2(
const __nv_bfloat16* in_hidden,
__nv_bfloat16* g_hidden,
const __nv_bfloat16* norm_w,
uint4* smem_x_u4,
float* smem_x_sum32,
float* smem_red16,
int bid,
uint64_t pol_keep
) {
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
uint4 h8 = make_uint4(0, 0, 0, 0);
uint4 w8 = make_uint4(0, 0, 0, 0);
int q = 0;
float local_sq = 0.0f;
if (tid < 288) {
int shift = (bid % 72) * 4;
q = tid + shift;
if (q >= 288) q -= 288;
h8 = ld_u4_ca(reinterpret_cast<const uint4*>(in_hidden) + q);
w8 = ld_u4_keep(reinterpret_cast<const uint4*>(norm_w) + q, pol_keep);
if (bid == 0) reinterpret_cast<uint4*>(g_hidden)[q] = h8;
float h0 = __uint_as_float(h8.x << 16), h1 = __uint_as_float(h8.x & 0xFFFF0000u);
float h2 = __uint_as_float(h8.y << 16), h3 = __uint_as_float(h8.y & 0xFFFF0000u);
float h4 = __uint_as_float(h8.z << 16), h5 = __uint_as_float(h8.z & 0xFFFF0000u);
float h6 = __uint_as_float(h8.w << 16), h7 = __uint_as_float(h8.w & 0xFFFF0000u);
local_sq = h0*h0 + h1*h1 + h2*h2 + h3*h3 + h4*h4 + h5*h5 + h6*h6 + h7*h7;
}
local_sq = warp_reduce_sum(local_sq);
if (lane == 0) smem_red16[wid] = local_sq;
__syncthreads();
if (wid == 0) {
float tot = (lane < 9) ? smem_red16[lane] : 0.0f;
tot = warp_reduce_sum(tot);
if (lane == 0) smem_red16[0] = rsqrtf(tot * (1.0f / float(HIDDEN)) + 1.0e-6f);
}
__syncthreads();
float inv_rms = smem_red16[0];
if (tid < 288) {
pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32);
}
__syncthreads();
}
__device__ __forceinline__ void store_bf16_u4_to_smem_bf2(
int q, uint4 v, uint4* smem_u4, float* smem_sum32
) {
uint32_t p0 = __byte_perm(v.x, v.z, 0x5410u);
uint32_t p1 = __byte_perm(v.x, v.z, 0x7632u);
uint32_t p2 = __byte_perm(v.y, v.w, 0x5410u);
uint32_t p3 = __byte_perm(v.y, v.w, 0x7632u);
int c = q >> 2;
smem_u4[c * X_U4_STRIDE + (q & 3)] = make_uint4(p0, p1, p2, p3);
float s = (__uint_as_float(v.x << 16) + __uint_as_float(v.x & 0xFFFF0000u))
+ (__uint_as_float(v.y << 16) + __uint_as_float(v.y & 0xFFFF0000u))
+ (__uint_as_float(v.z << 16) + __uint_as_float(v.z & 0xFFFF0000u))
+ (__uint_as_float(v.w << 16) + __uint_as_float(v.w & 0xFFFF0000u));
s = subwarp4_reduce_sum(s);
if ((threadIdx.x & 3) == 0) smem_sum32[c] = s;
}
// ---------------------------------------------------------------------------
// Overlapped o_proj (4096 -> 2304):
// - Direct 12-13 rows/block across all 188 SMs (zero remainder atomics)
// - Issues g_attn_o_in L2 loads first, overlapped with o_wq[0..1] + g_hidden[n]
// ---------------------------------------------------------------------------
template <bool INPUT_IS_F32>
__device__ __forceinline__ void run_load_and_oproj(
const void* g_attn_o_in,
const uint4* o_wq,
const uint32_t* o_sz,
__nv_bfloat16* g_hidden,
float* g_norm_sq,
uint4* smem_x_u4,
float* smem_x_sum32,
float* smem_sq16,
int bid,
int nblocks,
uint64_t pol_keep
) {
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
int n0 = (bid * HIDDEN) / nblocks;
int n_rows = ((bid + 1) * HIDDEN) / nblocks - n0;
uint4 w0 = make_uint4(0, 0, 0, 0), w1 = make_uint4(0, 0, 0, 0);
uint32_t s0 = 0, s1 = 0;
const uint4* wr = nullptr;
const uint32_t* sr = nullptr;
int n = 0;
__nv_bfloat16 h_prev = f32_to_bf16(0.0f);
int q = (tid + (bid & 127) * 4) & 511;
uint4 vo_a = {}, vo_b = {};
if constexpr (!INPUT_IS_F32) {
vo_a = ld_u4_ca(reinterpret_cast<const uint4*>(g_attn_o_in) + q);
} else {
const uint4* src = reinterpret_cast<const uint4*>(g_attn_o_in);
vo_a = ld_u4_ca(src + q * 2 + 0);
vo_b = ld_u4_ca(src + q * 2 + 1);
}
if (wid < n_rows) {
n = n0 + wid;
wr = o_wq + static_cast<size_t>(n) * 128;
sr = o_sz + static_cast<size_t>(n) * 32;
w0 = ld_u4_keep(wr + lane, pol_keep);
w1 = ld_u4_keep(wr + lane + 32, pol_keep);
s0 = ld_u32_keep(sr + (lane >> 2), pol_keep);
s1 = ld_u32_keep(sr + ((lane + 32) >> 2), pol_keep);
if (lane == 0) h_prev = g_hidden[n];
}
if constexpr (!INPUT_IS_F32) {
store_bf16_u4_to_smem_bf2(q, vo_a, smem_x_u4, smem_x_sum32);
} else {
float4 a = *reinterpret_cast<float4*>(&vo_a);
float4 b = *reinterpret_cast<float4*>(&vo_b);
uint32_t p0 = pack_bf16x2(a.x, b.x);
uint32_t p1 = pack_bf16x2(a.y, b.y);
uint32_t p2 = pack_bf16x2(a.z, b.z);
uint32_t p3 = pack_bf16x2(a.w, b.w);
int c = q >> 2;
smem_x_u4[c * X_U4_STRIDE + (q & 3)] = make_uint4(p0, p1, p2, p3);
float s = (__uint_as_float(p0 << 16) + __uint_as_float(p0 & 0xFFFF0000u))
+ (__uint_as_float(p1 << 16) + __uint_as_float(p1 & 0xFFFF0000u))
+ (__uint_as_float(p2 << 16) + __uint_as_float(p2 & 0xFFFF0000u))
+ (__uint_as_float(p3 << 16) + __uint_as_float(p3 & 0xFFFF0000u));
s = subwarp4_reduce_sum(s);
if ((tid & 3) == 0) smem_x_sum32[c] = s;
}
__syncthreads();
if (wid < n_rows) {
int k0 = lane, k1 = lane + 32, k2 = lane + 64, k3 = lane + 96;
uint4 w2 = ld_u4_keep(wr + k2, pol_keep);
uint4 w3 = ld_u4_keep(wr + k3, pol_keep);
uint32_t s2 = ld_u32_keep(sr + (k2 >> 2), pol_keep);
uint32_t s3 = ld_u32_keep(sr + (k3 >> 2), pol_keep);
float acc = dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]);
acc = warp_reduce_sum(acc);
if (lane == 0) {
__nv_bfloat16 proj_bf = f32_to_bf16(acc);
float h_new = bf16_to_f32(h_prev) + bf16_to_f32(proj_bf);
__nv_bfloat16 h_bf = f32_to_bf16(h_new);
g_hidden[n] = h_bf;
float hf = bf16_to_f32(h_bf);
smem_sq16[wid] = hf * hf;
}
}
__syncthreads();
if (wid == 0) {
float sq = (lane < n_rows) ? smem_sq16[lane] : 0.0f;
sq = warp_reduce_sum(sq);
if (lane == 0 && sq != 0.0f) atomicAdd(g_norm_sq, sq);
}
}
// ---------------------------------------------------------------------------
// MoE block:
// - Fused M1+M2 (3-way K-split s_gate_up, red.release.gpu + ld.relaxed/fence.acquire spin,
// redux.sync Top-8, balanced PRE=2 L2::256B BF16x2 routed gate+up)
// - Overlapped M3: direct 12-13 rows/block, contiguous expert rotation,
// 4 initial experts (e0..e3) + g_hidden[n] pre-issued before load_hh
// ---------------------------------------------------------------------------
__device__ __forceinline__ void run_moe_layer(
cg::grid_group& grid,
const MoeLayerPtrs& moe,
__nv_bfloat16* g_hidden,
__nv_bfloat16* final_out,
float* g_norm_sq,
float* g_router_logits,
unsigned int* g_router_done,
__nv_bfloat16* g_moe_hh,
float routed_scaling,
int n_experts,
char* smem_raw,
int bid,
int nblocks,
uint64_t pol_keep,
uint64_t pol_stream
) {
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
uint4* smem_x_u4 = reinterpret_cast<uint4*>(smem_raw);
float* smem_x_sum32 = reinterpret_cast<float*>(smem_x_u4 + 72 * X_U4_STRIDE);
float* smem_red16 = smem_x_sum32 + 80;
float* smem_sgu = smem_red16 + 16;
int* smem_topk_idx = reinterpret_cast<int*>(smem_raw + 32768);
float* smem_topk_w = reinterpret_cast<float*>(smem_raw + 32832);
float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f);
if (bid < n_experts) {
// Fused load_rmsnorm + router: issues h8, w8, and router_w in parallel, dots in registers, 1 fewer barrier!
float acc = 0.0f;
if (tid < 288) {
int shift = (bid % 72) * 4;
int q = tid + shift;
if (q >= 288) q -= 288;
const uint4* rw_u4 = reinterpret_cast<const uint4*>(moe.router_w) + static_cast<size_t>(bid) * 288;
uint4 h8 = ld_u4_ca(reinterpret_cast<const uint4*>(g_hidden) + q);
uint4 w8 = ld_u4_keep(reinterpret_cast<const uint4*>(moe.moe_norm) + q, pol_keep);
uint4 v = ld_u4_keep(rw_u4 + q, pol_keep);
uint4 x4 = pack_and_store_rmsnorm8(q, h8, w8, inv_rms, smem_x_u4, smem_x_sum32);
acc = fmaf(__uint_as_float(x4.x << 16), __uint_as_float(v.x << 16), acc);
acc = fmaf(__uint_as_float(x4.x & 0xFFFF0000u), __uint_as_float(v.x & 0xFFFF0000u), acc);
acc = fmaf(__uint_as_float(x4.y << 16), __uint_as_float(v.y << 16), acc);
acc = fmaf(__uint_as_float(x4.y & 0xFFFF0000u), __uint_as_float(v.y & 0xFFFF0000u), acc);
acc = fmaf(__uint_as_float(x4.z << 16), __uint_as_float(v.z << 16), acc);
acc = fmaf(__uint_as_float(x4.z & 0xFFFF0000u), __uint_as_float(v.z & 0xFFFF0000u), acc);
acc = fmaf(__uint_as_float(x4.w << 16), __uint_as_float(v.w << 16), acc);
acc = fmaf(__uint_as_float(x4.w & 0xFFFF0000u), __uint_as_float(v.w & 0xFFFF0000u), acc);
}
acc = warp_reduce_sum(acc);
if (lane == 0) smem_red16[wid] = acc;
__syncthreads();
if (wid == 0) {
float tot = (lane < 9) ? smem_red16[lane] : 0.0f;
tot = warp_reduce_sum(tot);
if (lane == 0) {
g_router_logits[bid] = tot;
asm volatile("red.release.gpu.global.add.u32 [%0], 1;" :: "l"(g_router_done) : "memory");
}
}
} else {
int s_bid = bid - n_experts;
int s_nblocks = nblocks - n_experts;
int c0 = (s_bid * MOE_INTER) / s_nblocks;
int c1 = ((s_bid + 1) * MOE_INTER) / s_nblocks;
int n_pairs = c1 - c0;
int n_jobs = n_pairs * 3;
int sub_id = tid >> 3;
int t8 = tid & 7;
bool act = (sub_id < n_jobs);
int p_loc = 0, ks = 0, k0 = 0, k1 = 0, k2 = 0;
uint4 wg0 = {}, wu0 = {}, wg1 = {}, wu1 = {}, wg2 = {}, wu2 = {};
uint32_t sg0 = 0, su0 = 0, sg1 = 0, su1 = 0, sg2 = 0, su2 = 0;
if (wid * 4 < n_jobs) {
int j_safe = act ? sub_id : 0;
p_loc = (j_safe * 21846) >> 16;
ks = j_safe - p_loc * 3;
int c = c0 + p_loc;
const uint4* gw = reinterpret_cast<const uint4*>(moe.s_gate_up_wq) + static_cast<size_t>(c) * 144;
const uint32_t* gsz = reinterpret_cast<const uint32_t*>(moe.s_gate_up_sz) + static_cast<size_t>(c) * 36;
k0 = (ks * 3 + 0) * 8 + t8;
k1 = (ks * 3 + 1) * 8 + t8;
k2 = (ks * 3 + 2) * 8 + t8;
wg0 = ld_u4_keep(gw + k0, pol_keep); wu0 = ld_u4_keep(gw + 72 + k0, pol_keep);
wg1 = ld_u4_keep(gw + k1, pol_keep); wu1 = ld_u4_keep(gw + 72 + k1, pol_keep);
wg2 = ld_u4_keep(gw + k2, pol_keep); wu2 = ld_u4_keep(gw + 72 + k2, pol_keep);
sg0 = ld_u32_keep(gsz + (k0 >> 2), pol_keep); su0 = ld_u32_keep(gsz + 18 + (k0 >> 2), pol_keep);
sg1 = ld_u32_keep(gsz + (k1 >> 2), pol_keep); su1 = ld_u32_keep(gsz + 18 + (k1 >> 2), pol_keep);
sg2 = ld_u32_keep(gsz + (k2 >> 2), pol_keep); su2 = ld_u32_keep(gsz + 18 + (k2 >> 2), pol_keep);
}
load_rmsnorm_2304_to_smem_bf2(
g_hidden, reinterpret_cast<const __nv_bfloat16*>(moe.moe_norm),
inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep
);
if (wid * 4 < n_jobs) {
float ga = 0.0f, ua = 0.0f;
dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua);
dot32_bf2_pair(wg1, sg1, wu1, su1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], ga, ua);
dot32_bf2_pair(wg2, sg2, wu2, su2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2], ga, ua);
ga = subwarp8_reduce_sum(ga);
ua = subwarp8_reduce_sum(ua);
if (act && t8 == 0) {
smem_sgu[p_loc * 6 + ks] = ga;
smem_sgu[p_loc * 6 + 3 + ks] = ua;
}
}
__syncthreads();
if (tid < n_pairs) {
float ga = smem_sgu[tid * 6 + 0] + smem_sgu[tid * 6 + 1] + smem_sgu[tid * 6 + 2];
float ua = smem_sgu[tid * 6 + 3] + smem_sgu[tid * 6 + 4] + smem_sgu[tid * 6 + 5];
g_moe_hh[8 * MOE_INTER + c0 + tid] = f32_to_bf16(silu_f32(ga) * ua);
}
}
// Fused spin-wait + parallel-lookup redux.sync Top-8 in wid==0 (saves 1 __syncthreads + 7 shuffles per MoE!)
if (wid == 0) {
if (lane == 0) {
unsigned int done;
while (true) {
asm volatile("ld.relaxed.gpu.global.u32 %0, [%1];" : "=r"(done) : "l"(g_router_done));
if (done >= static_cast<unsigned int>(n_experts)) break;
asm volatile("nanosleep.u32 64;");
}
asm volatile("fence.acquire.gpu;" ::: "memory");
}
__syncwarp();
float l0 = (lane < n_experts) ? g_router_logits[lane] : -1e30f;
float l1 = (lane + 32 < n_experts) ? g_router_logits[lane + 32] : -1e30f;
uint32_t k0 = (lane < n_experts) ? ((float_to_sortable_u32(l0) & 0xFFFFFFC0u) | (63u - lane)) : 0u;
uint32_t k1 = (lane + 32 < n_experts) ? ((float_to_sortable_u32(l1) & 0xFFFFFFC0u) | (63u - (lane + 32))) : 0u;
int my_bi = 0;
#pragma unroll
for (int k = 0; k < N_ACTIVE; ++k) {
uint32_t km = (k0 >= k1) ? k0 : k1;
uint32_t best;
asm("redux.sync.max.u32 %0, %1, 0xffffffff;" : "=r"(best) : "r"(km));
int bi = 63 - int(best & 63u);
if (lane == k) my_bi = bi;
if (bi == lane) k0 = 0u;
if (bi == lane + 32) k1 = 0u;
}
int src_lane = my_bi & 31;
float v_lo = __shfl_sync(0xffffffffu, l0, src_lane);
float v_hi = __shfl_sync(0xffffffffu, l1, src_lane);
float my_val = (my_bi < 32) ? v_lo : v_hi;
float m_top = __shfl_sync(0xffffffffu, my_val, 0);
float e = (lane < N_ACTIVE) ? __expf(my_val - m_top) : 0.0f;
float w_sum = warp_reduce_sum(e);
if (lane < N_ACTIVE) {
smem_topk_idx[lane] = my_bi;
smem_topk_w[lane] = (e / (w_sum + 1e-9f)) * routed_scaling;
}
if (lane == 0) {
smem_topk_idx[8] = n_experts;
smem_topk_w[8] = 1.0f;
if (bid == 0) *g_norm_sq = 0.0f;
}
}
__syncthreads();
// Phase M2: Contiguous 1-wave 2-step lookahead L2::256B BF16x2 routed gate+up
{
int p0_idx, n_pairs;
if (nblocks == 188 && n_experts == 64) {
if (bid < 64) {
p0_idx = bid * 45; n_pairs = 45;
} else if (bid < 168) {
p0_idx = 2880 + (bid - 64) * 43; n_pairs = 43;
} else {
p0_idx = 7352 + (bid - 168) * 42; n_pairs = 42;
}
} else {
p0_idx = (bid * (N_ACTIVE * MOE_INTER)) / nblocks;
n_pairs = ((bid + 1) * (N_ACTIVE * MOE_INTER)) / nblocks - p0_idx;
}
int sub_id = tid >> 3;
int t8 = tid & 7;
const uint4* all_wq = reinterpret_cast<const uint4*>(moe.r_gate_up_wq);
const uint32_t* all_sz = reinterpret_cast<const uint32_t*>(moe.r_gate_up_sz);
int n_warps = (n_pairs + 3) >> 2;
if (wid < n_warps) {
bool act = (sub_id < n_pairs);
int pair = p0_idx + (act ? sub_id : 0);
int j = pair >> 10;
int c = pair & 1023;
int e = smem_topk_idx[j];
size_t row_off = static_cast<size_t>(e) * MOE_INTER + c;
const uint4* gw = all_wq + row_off * 144;
const uint32_t* gsz = all_sz + row_off * 36;
float ga = 0.0f, ua = 0.0f;
int k0 = t8, k1 = 8 + t8;
uint4 wg0 = ld_u4_stream256(gw + k0, pol_stream), wu0 = ld_u4_stream256(gw + 72 + k0, pol_stream);
uint4 wg1 = ld_u4_stream256(gw + k1, pol_stream), wu1 = ld_u4_stream256(gw + 72 + k1, pol_stream);
uint32_t sg0 = ld_u32_stream(gsz + (k0 >> 2), pol_stream), su0 = ld_u32_stream(gsz + 18 + (k0 >> 2), pol_stream);
uint32_t sg1 = ld_u32_stream(gsz + (k1 >> 2), pol_stream), su1 = ld_u32_stream(gsz + 18 + (k1 >> 2), pol_stream);
#pragma unroll
for (int it = 0; it < 7; ++it) {
int kn = (it + 2) * 8 + t8;
uint4 nwg = ld_u4_stream256(gw + kn, pol_stream), nwu = ld_u4_stream256(gw + 72 + kn, pol_stream);
uint32_t nsg = ld_u32_stream(gsz + (kn >> 2), pol_stream), nsu = ld_u32_stream(gsz + 18 + (kn >> 2), pol_stream);
dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua);
wg0 = wg1; wu0 = wu1; sg0 = sg1; su0 = su1; k0 = k1;
wg1 = nwg; wu1 = nwu; sg1 = nsg; su1 = nsu; k1 = kn;
}
dot32_bf2_pair(wg0, sg0, wu0, su0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], ga, ua);
dot32_bf2_pair(wg1, sg1, wu1, su1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], ga, ua);
ga = subwarp8_reduce_sum(ga);
ua = subwarp8_reduce_sum(ua);
if (act && t8 == 0) g_moe_hh[pair] = f32_to_bf16(silu_f32(ga) * ua);
}
}
grid.sync();
if (bid == 0 && tid == 0) *g_router_done = 0u;
// Phase M3: direct 12-13 rows/block
// Contiguous expert-group rotation (rot = bid*8/nblocks) + ALL 4 initial experts pre-issued BEFORE load_hh!
uint4* smem_hh_u4 = reinterpret_cast<uint4*>(smem_raw);
float* smem_hh_sum32 = reinterpret_cast<float*>(smem_hh_u4 + 288 * X_U4_STRIDE);
float* smem_sq16 = smem_hh_sum32 + 288;
int n0 = (bid * HIDDEN) / nblocks;
int n_rows = ((bid + 1) * HIDDEN) / nblocks - n0;
int rot = (bid * 8) / nblocks;
const uint4* r_down_wq = reinterpret_cast<const uint4*>(moe.r_down_wq);
const uint32_t* r_down_sz = reinterpret_cast<const uint32_t*>(moe.r_down_sz);
const uint4* s_down_wq = reinterpret_cast<const uint4*>(moe.s_down_wq);
const uint32_t* s_down_sz = reinterpret_cast<const uint32_t*>(moe.s_down_sz);
uint4 w0 = make_uint4(0, 0, 0, 0), w1 = make_uint4(0, 0, 0, 0);
uint4 w2 = make_uint4(0, 0, 0, 0), w3 = make_uint4(0, 0, 0, 0);
uint32_t s0 = 0, s1 = 0, s2 = 0, s3 = 0;
__nv_bfloat16 h_prev = f32_to_bf16(0.0f);
if (wid < n_rows) {
int n = n0 + wid;
int e0 = smem_topk_idx[rot], e1 = smem_topk_idx[(rot + 1) & 7];
int e2 = smem_topk_idx[(rot + 2) & 7], e3 = smem_topk_idx[(rot + 3) & 7];
w0 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e0) * HIDDEN + n) * 32 + lane, pol_stream);
w1 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e1) * HIDDEN + n) * 32 + lane, pol_stream);
w2 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e2) * HIDDEN + n) * 32 + lane, pol_stream);
w3 = ld_u4_stream256(r_down_wq + (static_cast<size_t>(e3) * HIDDEN + n) * 32 + lane, pol_stream);
s0 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e0) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
s1 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e1) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
s2 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e2) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
s3 = ld_u32_stream(r_down_sz + (static_cast<size_t>(e3) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
if (lane == 0) h_prev = g_hidden[n];
}
{
const uint4* hh_u4 = reinterpret_cast<const uint4*>(g_moe_hh);
int shift = (bid % 288) * 4;
int q0 = tid + shift; if (q0 >= 1152) q0 -= 1152;
int q1 = tid + 512 + shift; if (q1 >= 1152) q1 -= 1152;
int q2 = tid + 1024 + shift; if (q2 >= 1152) q2 -= 1152;
uint4 v0 = ld_u4_ca(hh_u4 + q0);
uint4 v1 = ld_u4_ca(hh_u4 + q1);
uint4 v2 = (tid < 128) ? ld_u4_ca(hh_u4 + q2) : make_uint4(0u, 0u, 0u, 0u);
store_bf16_u4_to_smem_bf2(q0, v0, smem_hh_u4, smem_hh_sum32);
store_bf16_u4_to_smem_bf2(q1, v1, smem_hh_u4, smem_hh_sum32);
if (tid < 128) store_bf16_u4_to_smem_bf2(q2, v2, smem_hh_u4, smem_hh_sum32);
}
__syncthreads();
if (wid < n_rows) {
int n = n0 + wid;
float acc = 0.0f;
#pragma unroll
for (int j = 0; j < 5; ++j) {
int jc = (j + rot) & 7;
uint4 wn; uint32_t sn;
if (j < 4) {
int en = smem_topk_idx[(j + 4 + rot) & 7];
wn = ld_u4_stream256(r_down_wq + (static_cast<size_t>(en) * HIDDEN + n) * 32 + lane, pol_stream);
sn = ld_u32_stream(r_down_sz + (static_cast<size_t>(en) * HIDDEN + n) * 8 + (lane >> 2), pol_stream);
} else {
wn = ld_u4_keep(s_down_wq + static_cast<size_t>(n) * 32 + lane, pol_keep);
sn = ld_u32_keep(s_down_sz + static_cast<size_t>(n) * 8 + (lane >> 2), pol_keep);
}
acc = fmaf(smem_topk_w[jc], dot32_bf2(w0, s0, smem_hh_u4 + (jc * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[jc * 32 + lane]), acc);
w0 = w1; s0 = s1;
w1 = w2; s1 = s2;
w2 = w3; s2 = s3;
w3 = wn; s3 = sn;
}
int j5 = (5 + rot) & 7, j6 = (6 + rot) & 7, j7 = (7 + rot) & 7;
acc = fmaf(smem_topk_w[j5], dot32_bf2(w0, s0, smem_hh_u4 + (j5 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j5 * 32 + lane]), acc);
acc = fmaf(smem_topk_w[j6], dot32_bf2(w1, s1, smem_hh_u4 + (j6 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j6 * 32 + lane]), acc);
acc = fmaf(smem_topk_w[j7], dot32_bf2(w2, s2, smem_hh_u4 + (j7 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[j7 * 32 + lane]), acc);
acc += dot32_bf2(w3, s3, smem_hh_u4 + (8 * 32 + lane) * X_U4_STRIDE, smem_hh_sum32[8 * 32 + lane]);
acc = warp_reduce_sum(acc);
if (lane == 0) {
__nv_bfloat16 moe_bf = f32_to_bf16(acc);
float h_new = bf16_to_f32(h_prev) + bf16_to_f32(moe_bf);
__nv_bfloat16 h_bf = f32_to_bf16(h_new);
if (final_out != nullptr) {
final_out[n] = h_bf;
} else {
g_hidden[n] = h_bf;
float hf = bf16_to_f32(h_bf);
smem_sq16[wid] = hf * hf;
}
}
}
if (final_out == nullptr) {
__syncthreads();
if (wid == 0) {
float sq = (lane < n_rows) ? smem_sq16[lane] : 0.0f;
sq = warp_reduce_sum(sq);
if (lane == 0 && sq != 0.0f) atomicAdd(g_norm_sq, sq);
}
}
}
// ---------------------------------------------------------------------------
// KDA Attention block:
// - PRE=2 software-pipelined qkvg + overlapped coalesced conv preload (warps 12..14)
// - K2 pre-issues all 8 S loads before smem_q/k/dec + tree warp reduction
// - Input-first overlapped o_proj
// ---------------------------------------------------------------------------
__device__ __forceinline__ void run_kda_attn(
cg::grid_group& grid,
const KdaLayerPtrs& kda,
float* S_state,
__nv_bfloat16* cq,
__nv_bfloat16* ck,
__nv_bfloat16* cv,
const __nv_bfloat16* in_hidden_or_null,
__nv_bfloat16* g_hidden,
float* g_norm_sq,
float* g_q, float* g_k, float* g_v, float* g_decay, float* g_beta,
__nv_bfloat16* g_attn_o,
char* smem_raw,
int bid,
int nblocks,
uint64_t pol_keep
) {
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
uint4* smem_x_u4 = reinterpret_cast<uint4*>(smem_raw);
float* smem_x_sum32 = reinterpret_cast<float*>(smem_x_u4 + 128 * X_U4_STRIDE);
float* smem_red16 = smem_x_sum32 + 128;
float* smem_qkvg = smem_red16 + 16; // [96] floats
float* smem_conv_pre = smem_qkvg + 96; // [96] floats (p0*w0 + p1*w1 + p2*w2)
float* smem_conv_w3 = smem_conv_pre + 96; // [96] floats (w3)
uint16_t* smem_p1 = reinterpret_cast<uint16_t*>(smem_conv_w3 + 96); // [96] u16
uint16_t* smem_p2 = smem_p1 + 96; // [96] u16
int r0 = (bid * 16384) / nblocks;
int r1 = ((bid + 1) * 16384) / nblocks;
int n_rows = r1 - r0;
int sub_id = tid >> 3;
int t8 = tid & 7;
int n_sub = (n_rows + 1) >> 1;
const uint4* qkvg_wq = reinterpret_cast<const uint4*>(kda.qkvg_wq);
const uint32_t* qkvg_sz = reinterpret_cast<const uint32_t*>(kda.qkvg_sz);
const uint2* conv_w_u2 = reinterpret_cast<const uint2*>(kda.conv_w);
bool has0 = false, has1 = false;
const uint4* wr0 = nullptr; const uint4* wr1 = nullptr;
const uint32_t* sr0 = nullptr; const uint32_t* sr1 = nullptr;
uint4 wa0 = {}, wb0 = {}, wa1 = {}, wb1 = {};
uint32_t sa0 = 0, sb0 = 0, sa1 = 0, sb1 = 0;
int k0 = t8, k1 = 8 + t8;
if (wid * 4 < n_sub) {
int r = r0 + sub_id * 2;
has0 = (r < r1);
has1 = (r + 1 < r1);
int rr0 = has0 ? r : (r1 - 1);
int rr1 = has1 ? (r + 1) : rr0;
wr0 = qkvg_wq + static_cast<size_t>(rr0) * 72;
wr1 = qkvg_wq + static_cast<size_t>(rr1) * 72;
sr0 = qkvg_sz + static_cast<size_t>(rr0) * 18;
sr1 = qkvg_sz + static_cast<size_t>(rr1) * 18;
wa0 = ld_u4_keep(wr0 + k0, pol_keep); wb0 = ld_u4_keep(wr1 + k0, pol_keep);
wa1 = ld_u4_keep(wr0 + k1, pol_keep); wb1 = ld_u4_keep(wr1 + k1, pol_keep);
sa0 = ld_u32_keep(sr0 + (k0 >> 2), pol_keep); sb0 = ld_u32_keep(sr1 + (k0 >> 2), pol_keep);
sa1 = ld_u32_keep(sr0 + (k1 >> 2), pol_keep); sb1 = ld_u32_keep(sr1 + (k1 >> 2), pol_keep);
} else if (wid >= 12 && wid <= 14) {
int loc = tid - 384;
if (loc < n_rows) {
int row = r0 + loc;
int kind = row >> 12;
int c = row & 4095;
if (kind < 3) {
const uint16_t* prev = reinterpret_cast<const uint16_t*>(
(kind == 0) ? cq : ((kind == 1) ? ck : cv)
);
uint16_t p0 = ld_u16_keep(prev + c, pol_keep);
uint16_t p1 = ld_u16_keep(prev + KDA_C + c, pol_keep);
uint16_t p2 = ld_u16_keep(prev + 2 * KDA_C + c, pol_keep);
uint2 cw = ld_u2_keep(conv_w_u2 + kind * KDA_C + c, pol_keep);
float pre = __uint_as_float(uint32_t(p0) << 16) * __uint_as_float(cw.x << 16)
+ __uint_as_float(uint32_t(p1) << 16) * __uint_as_float(cw.x & 0xFFFF0000u)
+ __uint_as_float(uint32_t(p2) << 16) * __uint_as_float(cw.y << 16);
smem_conv_pre[loc] = pre;
smem_conv_w3[loc] = __uint_as_float(cw.y & 0xFFFF0000u);
smem_p1[loc] = p1;
smem_p2[loc] = p2;
}
}
}
if (in_hidden_or_null != nullptr) {
load_input_and_rmsnorm_2304_to_smem_bf2(
in_hidden_or_null, g_hidden,
reinterpret_cast<const __nv_bfloat16*>(kda.attn_norm),
smem_x_u4, smem_x_sum32, smem_red16, bid, pol_keep
);
} else {
float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f);
load_rmsnorm_2304_to_smem_bf2(
g_hidden, reinterpret_cast<const __nv_bfloat16*>(kda.attn_norm),
inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep
);
}
// Phase K1: 1-wide 2-row BF16x2 GEMV (warps 0..10) + overlapped coalesced conv preload (warps 12..14) + beta (warp 15)
{
if (wid * 4 < n_sub) {
float a0 = 0.0f, a1 = 0.0f;
#pragma unroll
for (int it = 0; it < 7; ++it) {
int kn = (it + 2) * 8 + t8;
uint4 nwa = ld_u4_keep(wr0 + kn, pol_keep), nwb = ld_u4_keep(wr1 + kn, pol_keep);
uint32_t nsa = ld_u32_keep(sr0 + (kn >> 2), pol_keep), nsb = ld_u32_keep(sr1 + (kn >> 2), pol_keep);
dot32_bf2_pair(wa0, sa0, wb0, sb0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], a0, a1);
wa0 = wa1; wb0 = wb1; sa0 = sa1; sb0 = sb1; k0 = k1;
wa1 = nwa; wb1 = nwb; sa1 = nsa; sb1 = nsb; k1 = kn;
}
dot32_bf2_pair(wa0, sa0, wb0, sb0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0], a0, a1);
dot32_bf2_pair(wa1, sa1, wb1, sb1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1], a0, a1);
a0 = subwarp8_reduce_sum(a0);
a1 = subwarp8_reduce_sum(a1);
if (t8 == 0) {
if (has0) smem_qkvg[sub_id * 2] = a0;
if (has1) smem_qkvg[sub_id * 2 + 1] = a1;
}
} else if (bid < KDA_H && wid == 15) {
const uint4* bw_u4 = reinterpret_cast<const uint4*>(kda.beta_w) + static_cast<size_t>(bid) * 288;
float acc = 0.0f;
for (int q = lane; q < 288; q += 32) {
uint4 v = ld_u4_keep(bw_u4 + q, pol_keep);
uint4 x4 = smem_x_u4[(q >> 2) * X_U4_STRIDE + (q & 3)];
acc = fmaf(__uint_as_float(x4.x << 16), __uint_as_float(v.x << 16), acc);
acc = fmaf(__uint_as_float(x4.x & 0xFFFF0000u), __uint_as_float(v.x & 0xFFFF0000u), acc);
acc = fmaf(__uint_as_float(x4.y << 16), __uint_as_float(v.y << 16), acc);
acc = fmaf(__uint_as_float(x4.y & 0xFFFF0000u), __uint_as_float(v.y & 0xFFFF0000u), acc);
acc = fmaf(__uint_as_float(x4.z << 16), __uint_as_float(v.z << 16), acc);
acc = fmaf(__uint_as_float(x4.z & 0xFFFF0000u), __uint_as_float(v.z & 0xFFFF0000u), acc);
acc = fmaf(__uint_as_float(x4.w << 16), __uint_as_float(v.w << 16), acc);
acc = fmaf(__uint_as_float(x4.w & 0xFFFF0000u), __uint_as_float(v.w & 0xFFFF0000u), acc);
}
acc = warp_reduce_sum(acc);
if (lane == 0) g_beta[bid] = sigmoid_f32(acc);
}
__syncthreads();
// Coalesced 88-thread epilogue (1 FMA + SiLU + coalesced pol_keep stores!)
if (tid < n_rows) {
int row = r0 + tid;
int kind = row >> 12;
int c = row & 4095;
__nv_bfloat16 val_bf = f32_to_bf16(smem_qkvg[tid]);
float val_f = bf16_to_f32(val_bf);
if (kind < 3) {
float out = fmaf(val_f, smem_conv_w3[tid], smem_conv_pre[tid]);
float s = bf16_to_f32(f32_to_bf16(silu_f32(out)));
uint16_t* prev = reinterpret_cast<uint16_t*>(
(kind == 0) ? cq : ((kind == 1) ? ck : cv)
);
uint16_t val_u16 = *reinterpret_cast<uint16_t*>(&val_bf);
st_u16_keep(prev + c, smem_p1[tid], pol_keep);
st_u16_keep(prev + KDA_C + c, smem_p2[tid], pol_keep);
st_u16_keep(prev + 2 * KDA_C + c, val_u16, pol_keep);
if (kind == 0) g_q[c] = s * 0.08838834764831845f;
else if (kind == 1) g_k[c] = s;
else g_v[c] = s;
} else {
g_decay[c] = sigmoid_f32(-val_f);
}
}
}
if (bid == 0 && tid == 0) *g_norm_sq = 0.0f;
grid.sync();
// Phase K2: KDA Recurrent State Update S[32, 128, 128] (pre-issued S loads + tree warp reduce!)
if (bid < 128) {
int h = bid >> 2;
int col_base = (bid & 3) * 32;
int j = col_base + lane;
float* smem_q = reinterpret_cast<float*>(smem_raw);
float* smem_k = smem_q + 128;
float* smem_dec = smem_k + 128;
float* smem_red = smem_dec + 128;
float* smem_delta = smem_red + 512;
float* S_head = S_state + static_cast<size_t>(h) * 128 * 128;
int r_base = wid * 8;
float s_reg[8];
#pragma unroll
for (int r = 0; r < 8; ++r) {
s_reg[r] = ld_f32_keep(S_head + (r_base + r) * 128 + j, pol_keep);
}
if (tid < 128) {
smem_q[tid] = g_q[h * 128 + tid];
smem_k[tid] = g_k[h * 128 + tid];
smem_dec[tid] = g_decay[h * 128 + tid];
}
__syncthreads();
float pred_part = 0.0f;
#pragma unroll
for (int r = 0; r < 8; ++r) {
int i = r_base + r;
float s = s_reg[r] * smem_dec[i];
s_reg[r] = s;
pred_part = fmaf(s, smem_k[i], pred_part);
}
smem_red[wid * 32 + lane] = pred_part;
__syncthreads();
if (wid == 0) {
float a0 = smem_red[ 0 * 32 + lane] + smem_red[ 1 * 32 + lane];
float a1 = smem_red[ 2 * 32 + lane] + smem_red[ 3 * 32 + lane];
float a2 = smem_red[ 4 * 32 + lane] + smem_red[ 5 * 32 + lane];
float a3 = smem_red[ 6 * 32 + lane] + smem_red[ 7 * 32 + lane];
float a4 = smem_red[ 8 * 32 + lane] + smem_red[ 9 * 32 + lane];
float a5 = smem_red[10 * 32 + lane] + smem_red[11 * 32 + lane];
float a6 = smem_red[12 * 32 + lane] + smem_red[13 * 32 + lane];
float a7 = smem_red[14 * 32 + lane] + smem_red[15 * 32 + lane];
float pred = ((a0 + a1) + (a2 + a3)) + ((a4 + a5) + (a6 + a7));
smem_delta[lane] = g_beta[h] * (g_v[h * 128 + j] - pred);
}
__syncthreads();
float delta = smem_delta[lane];
float o_part = 0.0f;
#pragma unroll
for (int r = 0; r < 8; ++r) {
int i = r_base + r;
float s_new = fmaf(smem_k[i], delta, s_reg[r]);
st_f32_keep(S_head + i * 128 + j, s_new, pol_keep);
o_part = fmaf(s_new, smem_q[i], o_part);
}
smem_red[wid * 32 + lane] = o_part;
__syncthreads();
if (wid == 0) {
float b0 = smem_red[ 0 * 32 + lane] + smem_red[ 1 * 32 + lane];
float b1 = smem_red[ 2 * 32 + lane] + smem_red[ 3 * 32 + lane];
float b2 = smem_red[ 4 * 32 + lane] + smem_red[ 5 * 32 + lane];
float b3 = smem_red[ 6 * 32 + lane] + smem_red[ 7 * 32 + lane];
float b4 = smem_red[ 8 * 32 + lane] + smem_red[ 9 * 32 + lane];
float b5 = smem_red[10 * 32 + lane] + smem_red[11 * 32 + lane];
float b6 = smem_red[12 * 32 + lane] + smem_red[13 * 32 + lane];
float b7 = smem_red[14 * 32 + lane] + smem_red[15 * 32 + lane];
float o_val = ((b0 + b1) + (b2 + b3)) + ((b4 + b5) + (b6 + b7));
g_attn_o[h * 128 + j] = f32_to_bf16(o_val);
}
}
grid.sync();
run_load_and_oproj<false>(
g_attn_o,
reinterpret_cast<const uint4*>(kda.o_wq),
reinterpret_cast<const uint32_t*>(kda.o_sz),
g_hidden, g_norm_sq,
smem_x_u4, smem_x_sum32, smem_red16, bid, nblocks, pol_keep
);
grid.sync();
}
// ---------------------------------------------------------------------------
// MLA Attention block:
// - Spin-free fused A1+A2 (nibble-transposed int4 W_uk, 1 uint4/thread, 0 spin-wait)
// - Warp-aligned FlashMLA A3 (async cp.async Step-0 copy, exp2f online softmax, streaming part_o)
// - Slim overlapped A4 (4-split part_o prefetch + 1-warp exp2f softmax + overlapped W_uv)
// - Input-first overlapped A5 o_proj
// ---------------------------------------------------------------------------
__device__ __forceinline__ void run_mla_attn(
cg::grid_group& grid,
const MlaLayerPtrs& mla,
const __nv_bfloat16* src_c_kv,
const __nv_bfloat16* src_k_rope,
__nv_bfloat16* dst_c_kv,
__nv_bfloat16* dst_k_rope,
int ctx_len,
int need_copy,
float rope_theta,
__nv_bfloat16* g_hidden,
float* g_norm_sq,
float* g_q_full_f32,
__nv_bfloat16* g_q_rope,
float2* g_mla_part_ml,
__nv_bfloat16* g_mla_part_o,
float* g_attn_o_f32,
char* smem_raw,
int bid,
int nblocks,
uint64_t pol_keep,
uint64_t pol_stream
) {
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
uint4* smem_x_u4 = reinterpret_cast<uint4*>(smem_raw);
float* smem_x_sum32 = reinterpret_cast<float*>(smem_x_u4 + 128 * X_U4_STRIDE);
float* smem_red16 = smem_x_sum32 + 128;
float* smem_cos = smem_red16 + 16;
float* smem_sin = smem_cos + 32;
float* smem_qpart = smem_sin + 32; // [64] floats for 2-way K-split q_nope!
float* smem_qn = smem_qpart + 64; // [32] floats local q_nope[h, d_blk*32..+31]!
float inv_rms = rsqrtf((*g_norm_sq) * (1.0f / float(HIDDEN)) + 1.0e-6f);
load_rmsnorm_2304_to_smem_bf2(
g_hidden, reinterpret_cast<const __nv_bfloat16*>(mla.attn_norm),
inv_rms, smem_x_u4, smem_x_sum32, bid, pol_keep
);
const uint4* qkva_wq = reinterpret_cast<const uint4*>(mla.qkva_wq);
const uint32_t* qkva_sz = reinterpret_cast<const uint32_t*>(mla.qkva_sz);
if (bid < 128) {
if (tid < 32) g_attn_o_f32[bid * 32 + tid] = 0.0f;
int h = bid >> 2;
int sub_id = tid >> 3; // 0..63 (ALL 16 warps active!)
int t8 = tid & 7;
// Step 1: 2-WAY K-SPLIT 32 rows of q_nope[h, d_blk*32..+31] straight into SMEM!
{
int row_loc = sub_id & 31;
int ks = sub_id >> 5; // 0 or 1
int row = bid * 32 + row_loc;
const uint4* w_row = qkva_wq + static_cast<size_t>(row) * 72;
const uint32_t* sz_row = qkva_sz + static_cast<size_t>(row) * 18;
float acc = 0.0f;
if (ks == 0) {
int k0 = 0 * 8 + t8, k1 = 1 * 8 + t8, k2 = 2 * 8 + t8, k3 = 3 * 8 + t8;
uint4 w0 = ld_u4_keep(w_row + k0, pol_keep), w1 = ld_u4_keep(w_row + k1, pol_keep);
uint4 w2 = ld_u4_keep(w_row + k2, pol_keep), w3 = ld_u4_keep(w_row + k3, pol_keep);
uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep), s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep);
uint32_t s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep), s3 = ld_u32_keep(sz_row + (k3 >> 2), pol_keep);
acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]);
} else {
int k0 = 4 * 8 + t8, k1 = 5 * 8 + t8, k2 = 6 * 8 + t8;
uint4 w0 = ld_u4_keep(w_row + k0, pol_keep), w1 = ld_u4_keep(w_row + k1, pol_keep), w2 = ld_u4_keep(w_row + k2, pol_keep);
uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep), s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep), s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep);
int k3 = 7 * 8 + t8, k4 = 8 * 8 + t8;
uint4 w3 = ld_u4_keep(w_row + k3, pol_keep), w4 = ld_u4_keep(w_row + k4, pol_keep);
uint32_t s3 = ld_u32_keep(sz_row + (k3 >> 2), pol_keep), s4 = ld_u32_keep(sz_row + (k4 >> 2), pol_keep);
acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
acc += dot32_bf2(w3, s3, smem_x_u4 + k3 * X_U4_STRIDE, smem_x_sum32[k3]);
acc += dot32_bf2(w4, s4, smem_x_u4 + k4 * X_U4_STRIDE, smem_x_sum32[k4]);
}
acc = subwarp8_reduce_sum(acc);
if (t8 == 0) smem_qpart[ks * 32 + row_loc] = acc;
}
// Pre-issue our single W_uk uint4 per thread BEFORE __syncthreads() (hides 100% of W_uk L2 latency!)
const uint4* wuk_blk = reinterpret_cast<const uint4*>(mla.wuk_wq) + static_cast<size_t>(bid) * 512;
uint4 w_uk = ld_u4_keep(wuk_blk + tid, pol_keep);
__syncthreads();
if (tid < 32) {
smem_qn[tid] = bf16_to_f32(f32_to_bf16(smem_qpart[tid] + smem_qpart[32 + tid]));
}
__syncthreads();
// Step 2: Spin-free W_uk (ZERO g_q_nope, ZERO g_mla_head_done spin-wait!)
float* smem_qeff = reinterpret_cast<float*>(smem_raw); // [4 * 36] floats (reuses smem_x_u4, disjoint from smem_qn!)
float* smem_bias = smem_qeff + 4 * X_F32_STRIDE; // [4] floats
if (wid < 4) {
const uint32_t* sz_ptr = reinterpret_cast<const uint32_t*>(mla.wuk_sz) + static_cast<size_t>(bid) * 128;
uint32_t sz = ld_u32_keep(sz_ptr + tid, pol_keep);
float scale = __uint_as_float(sz << 16);
float zero = __uint_as_float(sz & 0xFFFF0000u);
float qe = smem_qn[lane] * scale;
smem_qeff[wid * X_F32_STRIDE + lane] = qe;
float b = warp_reduce_sum(qe * (zero + 128.0f));
if (lane == 0) smem_bias[wid] = b;
}
__syncthreads();
// ALL 512 threads: 1 uint4 -> 1 dot32 -> atomicAdd into g_q_full_f32[h, tid]!
{
int g = tid >> 7; // 0..3
const float* xp = smem_qeff + g * X_F32_STRIDE;
float dot = dot8_biased(w_uk.x, xp) + dot8_biased(w_uk.y, xp + 8)
+ dot8_biased(w_uk.z, xp + 16) + dot8_biased(w_uk.w, xp + 24);
atomicAdd(&g_q_full_f32[h * KV_LORA + tid], dot - smem_bias[g]);
}
} else {
int r_bid = bid - 128;
int r_nblocks = nblocks - 128;
int pair0 = (r_bid * 1312) / r_nblocks;
int pair1 = ((r_bid + 1) * 1312) / r_nblocks;
int r0 = 4096 + pair0 * 2;
int n_pairs = pair1 - pair0;
int sub_id = tid >> 3;
int p_loc = sub_id >> 1;
int t8 = tid & 7;
int n_warps = (n_pairs + 1) >> 1;
float val_f = 0.0f, partner = 0.0f;
bool act = false;
int row = r0;
if (wid < n_warps) {
act = (p_loc < n_pairs);
row = r0 + (act ? sub_id : 0);
const uint4* w_row = qkva_wq + static_cast<size_t>(row) * 72;
const uint32_t* sz_row = qkva_sz + static_cast<size_t>(row) * 18;
float acc = 0.0f;
#pragma unroll
for (int g3 = 0; g3 < 9; g3 += 3) {
int k0 = (g3 + 0) * 8 + t8;
int k1 = (g3 + 1) * 8 + t8;
int k2 = (g3 + 2) * 8 + t8;
uint4 w0 = ld_u4_keep(w_row + k0, pol_keep);
uint4 w1 = ld_u4_keep(w_row + k1, pol_keep);
uint4 w2 = ld_u4_keep(w_row + k2, pol_keep);
uint32_t s0 = ld_u32_keep(sz_row + (k0 >> 2), pol_keep);
uint32_t s1 = ld_u32_keep(sz_row + (k1 >> 2), pol_keep);
uint32_t s2 = ld_u32_keep(sz_row + (k2 >> 2), pol_keep);
acc += dot32_bf2(w0, s0, smem_x_u4 + k0 * X_U4_STRIDE, smem_x_sum32[k0]);
acc += dot32_bf2(w1, s1, smem_x_u4 + k1 * X_U4_STRIDE, smem_x_sum32[k1]);
acc += dot32_bf2(w2, s2, smem_x_u4 + k2 * X_U4_STRIDE, smem_x_sum32[k2]);
}
acc = subwarp8_reduce_sum(acc);
val_f = bf16_to_f32(f32_to_bf16(acc));
partner = __shfl_xor_sync(0xffffffffu, val_f, 8);
} else if (wid == 15) {
float neg_log_theta = -__logf(rope_theta);
float inv = __expf(float(2 * lane) * (1.0f / 64.0f) * neg_log_theta);
float ang = float(ctx_len) * inv;
float s, c;
__sincosf(ang, &s, &c);
smem_cos[lane] = c;
smem_sin[lane] = s;
}
__syncthreads();
if (act && t8 == 0) {
if (row < 6144) {
int rr = row - 4096;
int h = rr >> 6;
int rd = rr & 63;
int pair = rd >> 1;
float c = smem_cos[pair];
float s = smem_sin[pair];
float out_r = (rd & 1) ? (val_f * c + partner * s) : (val_f * c - partner * s);
g_q_rope[h * QK_ROPE + rd] = f32_to_bf16(out_r);
} else if (row < 6656) {
dst_c_kv[static_cast<size_t>(ctx_len) * KV_LORA + (row - 6144)] = f32_to_bf16(val_f);
} else {
int rd = row - 6656;
int pair = rd >> 1;
float c = smem_cos[pair];
float s = smem_sin[pair];
float out_r = (rd & 1) ? (val_f * c + partner * s) : (val_f * c - partner * s);
dst_k_rope[static_cast<size_t>(ctx_len) * QK_ROPE + rd] = f32_to_bf16(out_r);
}
}
}
if (bid == 0 && tid == 0) *g_norm_sq = 0.0f;
grid.sync();
// Phase A3: Zero-div72 Warp-Aligned FlashMLA + padded PART_O_STRIDE_U4 + pol_keep part_ml!
int L = ctx_len + 1;
int total_tiles = (L + 15) / 16;
int cap_sp = (ctx_len <= 4096) ? 64 : (need_copy ? 128 : 96);
int num_splits = (total_tiles < cap_sp) ? total_tiles : cap_sp;
if (num_splits > nblocks) num_splits = nblocks;
uint64_t kv_pol = (ctx_len <= 4096) ? pol_keep : pol_stream;
if (bid < num_splits) {
int tile_start = (bid * total_tiles) / num_splits;
int tile_end = ((bid + 1) * total_tiles) / num_splits;
uint4* smem_Q_u4 = reinterpret_cast<uint4*>(smem_raw);
uint4* smem_KV_u4 = smem_Q_u4 + 32 * 72;
float* smem_S = reinterpret_cast<float*>(smem_KV_u4 + 2 * 16 * 72);
__nv_bfloat16* smem_P = reinterpret_cast<__nv_bfloat16*>(smem_S + 4 * 32 * 16);
float* smem_m = reinterpret_cast<float*>(smem_P + 32 * 24);
float* smem_l = smem_m + 32;
float* smem_alpha = smem_l + 32;
const uint4* src_ckv_u4 = reinterpret_cast<const uint4*>(src_c_kv);
const uint4* src_kr_u4 = reinterpret_cast<const uint4*>(src_k_rope);
uint4* dst_ckv_u4 = reinterpret_cast<uint4*>(dst_c_kv);
uint4* dst_kr_u4 = reinterpret_cast<uint4*>(dst_k_rope);
// Warp-aligned separate c_kv (512B bursts) + k_rope (tid<128) via cp.async
auto issue_kv_load = [&](int buf, int tile_idx) {
int t_base = tile_idx * 16;
uint4* buf_ptr = smem_KV_u4 + buf * (16 * 72);
int r0 = tid >> 6; // 0..7
int c8 = tid & 63; // 0..63
int r1 = r0 + 8; // 8..15
int t0 = t_base + r0;
int t1 = t_base + r1;
uint4* dst0 = buf_ptr + r0 * 72 + (c8 ^ (r0 & 7));
uint4* dst1 = buf_ptr + r1 * 72 + (c8 ^ (r1 & 7));
if (!need_copy) {
if (t0 < L) cp_async_cg_16_pol(dst0, dst_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_pol);
else *dst0 = make_uint4(0u, 0u, 0u, 0u);
if (t1 < L) cp_async_cg_16_pol(dst1, dst_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_pol);
else *dst1 = make_uint4(0u, 0u, 0u, 0u);
if (tid < 128) {
int rk = tid >> 3;
int rc8 = tid & 7;
int tk = t_base + rk;
uint4* dstk = buf_ptr + rk * 72 + ((64 + rc8) ^ (rk & 7));
if (tk < L) cp_async_cg_16_pol(dstk, dst_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_pol);
else *dstk = make_uint4(0u, 0u, 0u, 0u);
}
} else {
if (t0 < ctx_len) cp_async_cg_16_pol(dst0, src_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_pol);
else if (t0 < L) cp_async_cg_16_pol(dst0, dst_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_pol);
else *dst0 = make_uint4(0u, 0u, 0u, 0u);
if (t1 < ctx_len) cp_async_cg_16_pol(dst1, src_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_pol);
else if (t1 < L) cp_async_cg_16_pol(dst1, dst_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_pol);
else *dst1 = make_uint4(0u, 0u, 0u, 0u);
if (tid < 128) {
int rk = tid >> 3, rc8 = tid & 7, tk = t_base + rk;
uint4* dstk = buf_ptr + rk * 72 + ((64 + rc8) ^ (rk & 7));
if (tk < ctx_len) cp_async_cg_16_pol(dstk, src_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_pol);
else if (tk < L) cp_async_cg_16_pol(dstk, dst_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_pol);
else *dstk = make_uint4(0u, 0u, 0u, 0u);
}
}
cp_async_commit();
};
issue_kv_load(0, tile_start);
// Zero-div72 Q_ld via ld_u4_ca (leaves 100% of cp.async queue free for tile_start!)
{
const uint4* qf_u4 = reinterpret_cast<const uint4*>(g_q_full_f32);
int h_shift = bid & 31;
uint4 ua[4], ub[4];
int hs[4], c8s[4];
#pragma unroll
for (int pass = 0; pass < 4; ++pass) {
int idx = pass * BLOCK_THREADS + tid;
hs[pass] = ((idx >> 6) + h_shift) & 31;
c8s[pass] = idx & 63;
size_t off = (static_cast<size_t>(hs[pass]) * 64 + c8s[pass]) * 2;
ua[pass] = ld_u4_ca(qf_u4 + off + 0);
ub[pass] = ld_u4_ca(qf_u4 + off + 1);
}
uint4 qrv = make_uint4(0u, 0u, 0u, 0u);
int qrh = 0, qrc8 = 0;
if (tid < 256) {
qrh = ((tid >> 3) + h_shift) & 31;
qrc8 = tid & 7;
const uint4* qr_u4 = reinterpret_cast<const uint4*>(g_q_rope);
qrv = ld_u4_ca(qr_u4 + qrh * 8 + qrc8);
}
#pragma unroll
for (int pass = 0; pass < 4; ++pass) {
float4 a = *reinterpret_cast<float4*>(&ua[pass]);
float4 b = *reinterpret_cast<float4*>(&ub[pass]);
int h = hs[pass], c8 = c8s[pass];
smem_Q_u4[h * 72 + (c8 ^ (h & 7))] = make_uint4(
pack_bf16x2(a.x, a.y), pack_bf16x2(a.z, a.w),
pack_bf16x2(b.x, b.y), pack_bf16x2(b.z, b.w)
);
}
if (tid < 256) {
smem_Q_u4[qrh * 72 + ((64 + qrc8) ^ (qrh & 7))] = qrv;
}
}
if (tid < 32) {
smem_m[tid] = -1e30f;
smem_l[tid] = 0.0f;
}
float o_acc[8][4];
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
o_acc[nt][0] = 0.0f; o_acc[nt][1] = 0.0f;
o_acc[nt][2] = 0.0f; o_acc[nt][3] = 0.0f;
}
int qm = wid >> 3;
int qn = (wid >> 2) & 1;
int qk = wid & 3;
int pn = wid & 7;
int buf = 0;
for (int tile = tile_start; tile < tile_end; ++tile) {
bool has_next = (tile + 1 < tile_end);
if (has_next) {
issue_kv_load(buf ^ 1, tile + 1);
cp_async_wait<1>();
} else {
cp_async_wait<0>();
}
__syncthreads();
uint4* kv_buf = smem_KV_u4 + buf * (16 * 72);
if (need_copy) {
int t_base = tile * 16;
int r0 = tid >> 6, c8 = tid & 63, r1 = r0 + 8;
int t0 = t_base + r0, t1 = t_base + r1;
if (t0 < ctx_len) st_u4_stream(dst_ckv_u4 + static_cast<size_t>(t0) * 64 + c8, kv_buf[r0 * 72 + (c8 ^ (r0 & 7))], kv_pol);
if (t1 < ctx_len) st_u4_stream(dst_ckv_u4 + static_cast<size_t>(t1) * 64 + c8, kv_buf[r1 * 72 + (c8 ^ (r1 & 7))], kv_pol);
if (tid < 128) {
int rk = tid >> 3, rc8 = tid & 7, tk = t_base + rk;
if (tk < ctx_len) st_u4_stream(dst_kr_u4 + static_cast<size_t>(tk) * 8 + rc8, kv_buf[rk * 72 + ((64 + rc8) ^ (rk & 7))], kv_pol);
}
}
float s0 = 0.0f, s1 = 0.0f, s2 = 0.0f, s3 = 0.0f;
int c8_start = qk * 18;
#pragma unroll
for (int ks = 0; ks < 9; ++ks) {
int c8_b = c8_start + ks * 2;
int ar = qm * 16 + (lane & 15);
int ac8 = c8_b + ((lane >> 4) & 1);
uint32_t a0, a1, a2, a3;
ldmatrix_x4(a0, a1, a2, a3, smem_Q_u4 + ar * 72 + (ac8 ^ (ar & 7)));
int br = qn * 8 + (lane & 7);
int bc8 = c8_b + ((lane >> 3) & 1);
uint32_t b0, b1;
ldmatrix_x2(b0, b1, kv_buf + br * 72 + (bc8 ^ (br & 7)));
mma_m16n8k16_bf16(s0, s1, s2, s3, a0, a1, a2, a3, b0, b1);
}
int s_r0 = qm * 16 + (lane >> 2);
int s_r1 = s_r0 + 8;
int s_c0 = qn * 8 + 2 * (lane & 3);
float* s_dst = smem_S + qk * 512;
s_dst[s_r0 * 16 + s_c0] = s0;
s_dst[s_r0 * 16 + s_c0 + 1] = s1;
s_dst[s_r1 * 16 + s_c0] = s2;
s_dst[s_r1 * 16 + s_c0 + 1] = s3;
__syncthreads();
{
int sub = lane >> 4;
int tl = lane & 15;
int t_glob = tile * 16 + tl;
int h = wid * 2 + sub;
float sc = (t_glob < L)
? (smem_S[h * 16 + tl] + smem_S[512 + h * 16 + tl]
+ smem_S[1024 + h * 16 + tl] + smem_S[1536 + h * 16 + tl]) * 0.10411755264281874f
: -1e30f;
float m_tile = halfwarp16_reduce_max(sc);
float m_old = smem_m[h];
float m_new = fmaxf(m_old, m_tile);
float p = (t_glob < L) ? exp2f(sc - m_new) : 0.0f;
float l_tile = halfwarp16_reduce_sum(p);
float alpha = exp2f(m_old - m_new);
smem_P[h * 24 + tl] = f32_to_bf16(p);
if (tl == 0) {
smem_alpha[h] = alpha;
smem_m[h] = m_new;
smem_l[h] = smem_l[h] * alpha + l_tile;
}
}
__syncthreads();
{
int row0 = qm * 16 + (lane >> 2);
int row1 = row0 + 8;
if (tile > tile_start) {
float a0_s = smem_alpha[row0];
float a1_s = smem_alpha[row1];
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
o_acc[nt][0] *= a0_s; o_acc[nt][1] *= a0_s;
o_acc[nt][2] *= a1_s; o_acc[nt][3] *= a1_s;
}
}
int pr = qm * 16 + (lane & 15);
int pc = (lane >> 4) ? 8 : 0;
uint32_t pa0, pa1, pa2, pa3;
ldmatrix_x4(pa0, pa1, pa2, pa3, smem_P + pr * 24 + pc);
#pragma unroll
for (int nt2 = 0; nt2 < 4; ++nt2) {
int vr = lane & 15;
int vc8 = pn * 8 + nt2 * 2 + ((lane >> 4) & 1);
uint32_t b00, b10, b01, b11;
ldmatrix_x4_trans(b00, b10, b01, b11, kv_buf + vr * 72 + (vc8 ^ (vr & 7)));
mma_m16n8k16_bf16(
o_acc[nt2 * 2][0], o_acc[nt2 * 2][1], o_acc[nt2 * 2][2], o_acc[nt2 * 2][3],
pa0, pa1, pa2, pa3, b00, b10
);
mma_m16n8k16_bf16(
o_acc[nt2 * 2 + 1][0], o_acc[nt2 * 2 + 1][1], o_acc[nt2 * 2 + 1][2], o_acc[nt2 * 2 + 1][3],
pa0, pa1, pa2, pa3, b01, b11
);
}
}
if (has_next) __syncthreads();
buf ^= 1;
}
int row0 = qm * 16 + (lane >> 2);
int row1 = row0 + 8;
float inv_l0 = 1.0f / smem_l[row0];
float inv_l1 = 1.0f / smem_l[row1];
float2 my_ml = (tid < 32) ? make_float2(smem_m[tid], smem_l[tid]) : make_float2(0.0f, 0.0f);
__syncthreads();
uint32_t* smem_O_u32 = reinterpret_cast<uint32_t*>(smem_raw);
constexpr int O_U32_STRIDE = O_SMEM_STRIDE / 2;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
int col2 = (pn * 64 + nt * 8) / 2 + (lane & 3);
smem_O_u32[row0 * O_U32_STRIDE + col2] = pack_bf16x2(o_acc[nt][0] * inv_l0, o_acc[nt][1] * inv_l0);
smem_O_u32[row1 * O_U32_STRIDE + col2] = pack_bf16x2(o_acc[nt][2] * inv_l1, o_acc[nt][3] * inv_l1);
}
__syncthreads();
uint4* smem_O_u4 = reinterpret_cast<uint4*>(smem_raw);
uint4* g_po_u4 = reinterpret_cast<uint4*>(g_mla_part_o) + static_cast<size_t>(bid) * PART_O_STRIDE_U4;
#pragma unroll
for (int pass = 0; pass < 4; ++pass) {
int idx = tid + pass * BLOCK_THREADS;
int hh = idx >> 6;
int cc = idx & 63;
st_u4_stream(g_po_u4 + idx, smem_O_u4[hh * 65 + cc], pol_stream);
}
// Write part_ml LAST with pol_keep so it stays 100% L2-hot for A4!
if (tid < 32) {
st_f32x2_keep(g_mla_part_ml + tid * ML_STRIDE + bid, my_ml, pol_keep);
}
}
grid.sync();
// Phase A4: OVERLAPPED part_o prefetch + 1-warp softmax + OVERLAPPED W_uv!
if (bid < 128) {
int h = bid >> 2;
int g = bid & 3;
if (tid < 32) reinterpret_cast<uint4*>(g_q_full_f32)[bid * 32 + tid] = make_uint4(0u, 0u, 0u, 0u);
float* smem_part_lat = reinterpret_cast<float*>(smem_raw);
float* smem_olat = smem_part_lat + 32 * LAT_STRIDE;
float* smem_olat_sum32 = smem_olat + 4 * X_F32_STRIDE;
float* smem_w = smem_olat_sum32 + 16;
// Pre-issue up to 4 splits (<=128 total) of g_mla_part_o across all 16 warps before softmax
int sg = tid >> 4;
int d8 = tid & 15;
const uint4* po_u4 = reinterpret_cast<const uint4*>(g_mla_part_o)
+ (static_cast<size_t>(h) * 64) + g * 16 + d8;
int s0 = sg, s1 = sg + 32, s2 = sg + 64, s3 = sg + 96;
uint4 v0 = (s0 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s0) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
uint4 v1 = (s1 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s1) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
uint4 v2 = (s2 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s2) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
uint4 v3 = (s3 < num_splits) ? ld_u4_stream(po_u4 + static_cast<size_t>(s3) * PART_O_STRIDE_U4, pol_stream) : make_uint4(0u, 0u, 0u, 0u);
if (wid == 0) {
const float2* ml_head = g_mla_part_ml + h * ML_STRIDE;
float2 ml_reg[4];
#pragma unroll
for (int k = 0; k < 4; ++k) {
int s = k * 32 + lane;
ml_reg[k] = (s < num_splits) ? ld_f32x2_keep(ml_head + s, pol_keep) : make_float2(-1e30f, 0.0f);
}
float lm = fmaxf(fmaxf(ml_reg[0].x, ml_reg[1].x), fmaxf(ml_reg[2].x, ml_reg[3].x));
#pragma unroll
for (int off = 16; off > 0; off >>= 1) lm = fmaxf(lm, __shfl_xor_sync(0xffffffffu, lm, off));
float lw = 0.0f;
#pragma unroll
for (int k = 0; k < 4; ++k) {
int s = k * 32 + lane;
float w = (s < num_splits) ? (ml_reg[k].y * exp2f(ml_reg[k].x - lm)) : 0.0f;
ml_reg[k].y = w;
lw += w;
}
float inv_l = 1.0f / warp_reduce_sum(lw);
#pragma unroll
for (int k = 0; k < 4; ++k) {
int s = k * 32 + lane;
if (s < num_splits) smem_w[s] = ml_reg[k].y * inv_l;
}
}
__syncthreads();
{
float a0 = 0.0f, a1 = 0.0f, a2 = 0.0f, a3 = 0.0f;
float a4 = 0.0f, a5 = 0.0f, a6 = 0.0f, a7 = 0.0f;
auto acc_v = [&](int s, uint4 v) {
if (s < num_splits) {
float w = smem_w[s];
a0 = fmaf(w, __uint_as_float(v.x << 16), a0);
a1 = fmaf(w, __uint_as_float(v.x & 0xFFFF0000u), a1);
a2 = fmaf(w, __uint_as_float(v.y << 16), a2);
a3 = fmaf(w, __uint_as_float(v.y & 0xFFFF0000u), a3);
a4 = fmaf(w, __uint_as_float(v.z << 16), a4);
a5 = fmaf(w, __uint_as_float(v.z & 0xFFFF0000u), a5);
a6 = fmaf(w, __uint_as_float(v.w << 16), a6);
a7 = fmaf(w, __uint_as_float(v.w & 0xFFFF0000u), a7);
}
};
acc_v(s0, v0); acc_v(s1, v1); acc_v(s2, v2); acc_v(s3, v3);
float* dst = smem_part_lat + sg * LAT_STRIDE + d8 * 8;
*reinterpret_cast<float4*>(dst) = make_float4(a0, a1, a2, a3);
*reinterpret_cast<float4*>(dst + 4) = make_float4(a4, a5, a6, a7);
}
// Pre-issue W_uv load BEFORE olat reduction (hides W_uv L2 latency completely!)
int d_out = tid >> 2;
int t4 = tid & 3;
int k32 = g * 4 + t4;
const uint4* wuv_row = reinterpret_cast<const uint4*>(mla.wuv_wq) + (static_cast<size_t>(h) * V_HEAD + d_out) * 16;
const uint32_t* sz_row = reinterpret_cast<const uint32_t*>(mla.wuv_sz) + (static_cast<size_t>(h) * V_HEAD + d_out) * 4;
uint4 w_uv = ld_u4_keep(wuv_row + k32, pol_keep);
uint32_t sz_uv = ld_u32_keep(sz_row + g, pol_keep);
__syncthreads();
{
int d = d_out;
int q4 = t4;
float s = 0.0f;
#pragma unroll
for (int r = 0; r < 8; ++r) {
s += smem_part_lat[(q4 * 8 + r) * LAT_STRIDE + d];
}
s = subwarp4_reduce_sum(s);
if (q4 == 0) {
smem_olat[(d >> 5) * X_F32_STRIDE + (d & 31)] = s;
}
}
__syncthreads();
if (wid < 4) {
float s32 = warp_reduce_sum(smem_olat[wid * X_F32_STRIDE + lane]);
if (lane == 0) smem_olat_sum32[wid] = s32;
}
__syncthreads();
{
float acc = subwarp4_reduce_sum(dot32_w4_f32(w_uv, sz_uv, smem_olat + t4 * X_F32_STRIDE, smem_olat_sum32[t4]));
if (t4 == 0) atomicAdd(&g_attn_o_f32[h * V_HEAD + d_out], acc);
}
}
grid.sync();
run_load_and_oproj<true>(
g_attn_o_f32,
reinterpret_cast<const uint4*>(mla.o_wq),
reinterpret_cast<const uint32_t*>(mla.o_sz),
g_hidden, g_norm_sq,
smem_x_u4, smem_x_sum32, smem_red16, bid, nblocks, pol_keep
);
grid.sync();
}
// ---------------------------------------------------------------------------
// THE SINGLE MEGAKERNEL V11
// ---------------------------------------------------------------------------
__global__ __launch_bounds__(BLOCK_THREADS, 1)
void kimi_decode_megakernel(const MegaParams p) {
extern __shared__ char smem_raw[];
cg::grid_group grid = cg::this_grid();
int bid = blockIdx.x;
int nblocks = gridDim.x;
uint64_t pol_keep, pol_stream;
if (p.need_copy) asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 0.20;" : "=l"(pol_keep));
else asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;" : "=l"(pol_keep));
asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(pol_stream));
__nv_bfloat16* g_hidden = reinterpret_cast<__nv_bfloat16*>(p.g_hidden);
__nv_bfloat16* g_attn_o = reinterpret_cast<__nv_bfloat16*>(p.g_attn_o);
__nv_bfloat16* g_moe_hh = reinterpret_cast<__nv_bfloat16*>(p.g_moe_hh);
#pragma unroll 1
for (int layer = 0; layer < 3; ++layer) {
run_kda_attn(
grid,
p.kda[layer],
p.kda_S[layer],
reinterpret_cast<__nv_bfloat16*>(p.kda_cq[layer]),
reinterpret_cast<__nv_bfloat16*>(p.kda_ck[layer]),
reinterpret_cast<__nv_bfloat16*>(p.kda_cv[layer]),
(layer == 0) ? reinterpret_cast<const __nv_bfloat16*>(p.in_hidden) : nullptr,
g_hidden, p.g_norm_sq,
p.g_q, p.g_k, p.g_v, p.g_decay, p.g_beta, g_attn_o,
smem_raw, bid, nblocks, pol_keep
);
run_moe_layer(
grid,
p.moe[layer],
g_hidden, nullptr, p.g_norm_sq,
p.g_router_logits, &p.g_router_done[layer], g_moe_hh,
p.routed_scaling, p.n_experts,
smem_raw, bid, nblocks, pol_keep, pol_stream
);
grid.sync();
}
run_mla_attn(
grid,
p.mla,
reinterpret_cast<const __nv_bfloat16*>(p.src_c_kv),
reinterpret_cast<const __nv_bfloat16*>(p.src_k_rope),
reinterpret_cast<__nv_bfloat16*>(p.dst_c_kv),
reinterpret_cast<__nv_bfloat16*>(p.dst_k_rope),
p.ctx_len, p.need_copy, p.rope_theta,
g_hidden, p.g_norm_sq,
p.g_q_full_f32, reinterpret_cast<__nv_bfloat16*>(p.g_q_rope),
reinterpret_cast<float2*>(p.g_mla_part_ml), reinterpret_cast<__nv_bfloat16*>(p.g_mla_part_o),
p.g_attn_o_f32, smem_raw, bid, nblocks, pol_keep, pol_stream
);
run_moe_layer(
grid,
p.moe[3],
g_hidden, reinterpret_cast<__nv_bfloat16*>(p.out_hidden),
p.g_norm_sq,
p.g_router_logits, &p.g_router_done[3], g_moe_hh,
p.routed_scaling, p.n_experts,
smem_raw, bid, nblocks, pol_keep, pol_stream
);
}
extern "C" void launch_kimi_megakernel(
const MegaParams* h_params,
void* /*dense_arena_ptr*/,
size_t /*dense_arena_bytes*/,
cudaStream_t stream
) {
static int s_sm_count = 0;
static bool s_attr_set = false;
if (!s_attr_set) {
cudaDeviceProp prop;
cudaGetDeviceProperties(&prop, 0);
s_sm_count = prop.multiProcessorCount;
cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, prop.persistingL2CacheMaxSize);
cudaFuncSetAttribute(kimi_decode_megakernel, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_BYTES);
s_attr_set = true;
}
MegaParams p = *h_params;
void* args[] = { &p };
cudaLaunchCooperativeKernel(
(void*)kimi_decode_megakernel,
dim3(s_sm_count),
dim3(BLOCK_THREADS),
args,
SMEM_BYTES,
stream
);
}
"""
_EXT = None
def _get_ext():
global _EXT
if _EXT is None:
_EXT = load_inline(
name="kimi_linear_megakernel_v11",
cpp_sources=_CPP_SRC,
cuda_sources=_CUDA_SRC,
functions=["create_handle", "destroy_handle", "step_megakernel"],
extra_cuda_cflags=[
"-O3",
"-gencode=arch=compute_120,code=sm_120",
"--expt-relaxed-constexpr",
],
no_implicit_headers=True,
)
return _EXT
def _pack_sz(scales: torch.Tensor, zeros: torch.Tensor) -> torch.Tensor:
s_u16 = scales.contiguous().view(torch.uint16).to(torch.int32)
z_u16 = zeros.contiguous().view(torch.uint16).to(torch.int32)
return (s_u16 | (z_u16 << 16)).contiguous()
def _pack_linear(ql: QuantLinear):
wq_t = ql.w_q.transpose(0, 1).contiguous()
sz_t = _pack_sz(ql.scales.transpose(0, 1), ql.zeros.transpose(0, 1))
return wq_t, sz_t
def _pack_linear_cat(qls: list[QuantLinear]):
wqs = [q.w_q.transpose(0, 1).contiguous() for q in qls]
szs = [_pack_sz(q.scales.transpose(0, 1), q.zeros.transpose(0, 1)) for q in qls]
return torch.cat(wqs, dim=0).contiguous(), torch.cat(szs, dim=0).contiguous()
def _pack_bf16_interleave8(w: torch.Tensor) -> torch.Tensor:
orig_shape = w.shape
return w.contiguous().view(*orig_shape[:-1], orig_shape[-1] // 8, 2, 4).transpose(-1, -2).contiguous().view(orig_shape)
def _pack_into_arena(tensors: list[torch.Tensor], dev: torch.device) -> tuple[torch.Tensor, list[torch.Tensor]]:
offsets = []
cur = 0
for t in tensors:
nbytes = t.numel() * t.element_size()
offsets.append((cur, nbytes, t.shape, t.dtype))
cur = (cur + nbytes + 127) & ~127
arena = torch.empty(cur, dtype=torch.uint8, device=dev)
views = []
for t, (off, nbytes, shape, dtype) in zip(tensors, offsets):
v = arena[off : off + nbytes].view(dtype).view(shape)
v.copy_(t)
views.append(v)
return arena, views
class Model(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern)
self._handle = 0
self._packed_tensors = None
self._dense_arena = None
self._scratch = None
self._packed_device = None
self._last_kv_ptr = 0
self._last_kr_ptr = 0
self._last_kv_buf = None
self._last_kr_buf = None
def __del__(self):
if getattr(self, "_handle", 0) != 0 and _EXT is not None:
try:
_EXT.destroy_handle(self._handle)
except Exception:
pass
self._handle = 0
def _invalidate_packed(self):
if self._handle != 0 and _EXT is not None:
_EXT.destroy_handle(self._handle)
self._handle = 0
self._packed_tensors = None
self._dense_arena = None
self._scratch = None
self._packed_device = None
def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
res = super().load_state_dict(state_dict, strict=strict, assign=assign)
self._invalidate_packed()
if next(self.parameters()).is_cuda:
self._build_packed()
return res
def _build_packed(self):
dev = next(self.parameters()).device
ext = _get_ext()
cfg = self.cfg
dense_list = []
for i in range(3):
blk = self.blocks[i]
kda = blk.attn
qkvg_wq, qkvg_sz = _pack_linear_cat([kda.q_proj, kda.k_proj, kda.v_proj, kda.g_proj])
o_wq, o_sz = _pack_linear(kda.o_proj)
dense_list.extend([
blk.attn_norm.contiguous(),
qkvg_wq, qkvg_sz,
_pack_bf16_interleave8(kda.beta_proj.weight),
kda.conv_w.contiguous(),
o_wq, o_sz,
])
blk3 = self.blocks[3]
mla = blk3.attn
MLA_H = cfg.mla_heads
q_wq_t = mla.q_proj.w_q.transpose(0, 1).contiguous().view(MLA_H, cfg.qk_nope + cfg.qk_rope, -1)
q_sz_t = _pack_sz(mla.q_proj.scales.transpose(0, 1), mla.q_proj.zeros.transpose(0, 1)).view(MLA_H, cfg.qk_nope + cfg.qk_rope, -1)
kva_wq_t = mla.kv_a.w_q.transpose(0, 1).contiguous()
kva_sz_t = _pack_sz(mla.kv_a.scales.transpose(0, 1), mla.kv_a.zeros.transpose(0, 1))
qkva_wq = torch.cat([
q_wq_t[:, : cfg.qk_nope, :].reshape(MLA_H * cfg.qk_nope, -1),
q_wq_t[:, cfg.qk_nope :, :].reshape(MLA_H * cfg.qk_rope, -1),
kva_wq_t,
], dim=0).contiguous()
qkva_sz = torch.cat([
q_sz_t[:, : cfg.qk_nope, :].reshape(MLA_H * cfg.qk_nope, -1),
q_sz_t[:, cfg.qk_nope :, :].reshape(MLA_H * cfg.qk_rope, -1),
kva_sz_t,
], dim=0).contiguous()
# Pure int4 nibble transpose of W_uk from packing-along-kv_lora to packing-along-q_nope
w_raw = mla.kv_b.w_q.view(256, MLA_H, 256)[:, :, :128]
u_r = torch.stack([w_raw & 0xF, w_raw >> 4], dim=1).reshape(512, MLA_H, 4, 16, 2)
wuk_wq = (u_r[..., 0] | (u_r[..., 1] << 4)).permute(1, 2, 0, 3).contiguous()
kvb_s = mla.kv_b.scales.view(4, MLA_H, 256)
kvb_z = mla.kv_b.zeros.view(4, MLA_H, 256)
wuk_sz = _pack_sz(
kvb_s[:, :, :128].view(4, MLA_H, 4, 32).permute(1, 2, 0, 3),
kvb_z[:, :, :128].view(4, MLA_H, 4, 32).permute(1, 2, 0, 3),
)
wuv_wq = mla.kv_b.w_q.view(256, MLA_H, 256)[:, :, 128:].permute(1, 2, 0).contiguous()
wuv_sz = _pack_sz(
kvb_s[:, :, 128:].permute(1, 2, 0),
kvb_z[:, :, 128:].permute(1, 2, 0),
)
mla_o_wq, mla_o_sz = _pack_linear(mla.o_proj)
dense_list.extend([
blk3.attn_norm.contiguous(),
qkva_wq, qkva_sz,
wuk_wq, wuk_sz,
wuv_wq, wuv_sz,
mla_o_wq, mla_o_sz,
])
routed_list = []
for i in range(4):
blk = self.blocks[i]
moe = blk.moe
sg_wq = moe.s_gate.w_q[0].transpose(0, 1)
su_wq = moe.s_up.w_q[0].transpose(0, 1)
s_gu_wq = torch.stack([sg_wq, su_wq], dim=1).contiguous()
s_gu_sz = torch.stack([
_pack_sz(moe.s_gate.scales[0].transpose(0, 1), moe.s_gate.zeros[0].transpose(0, 1)),
_pack_sz(moe.s_up.scales[0].transpose(0, 1), moe.s_up.zeros[0].transpose(0, 1)),
], dim=1).contiguous()
s_d_wq = moe.s_down.w_q[0].transpose(0, 1).contiguous()
s_d_sz = _pack_sz(moe.s_down.scales[0].transpose(0, 1), moe.s_down.zeros[0].transpose(0, 1))
dense_list.extend([
blk.moe_norm.contiguous(),
_pack_bf16_interleave8(moe.router.weight),
s_gu_wq, s_gu_sz,
s_d_wq, s_d_sz,
])
rg_wq = moe.gate.w_q.transpose(1, 2)
ru_wq = moe.up.w_q.transpose(1, 2)
r_gu_wq = torch.stack([rg_wq, ru_wq], dim=2).contiguous()
r_gu_sz = torch.stack([
_pack_sz(moe.gate.scales.transpose(1, 2), moe.gate.zeros.transpose(1, 2)),
_pack_sz(moe.up.scales.transpose(1, 2), moe.up.zeros.transpose(1, 2)),
], dim=2).contiguous()
r_d_wq = moe.down.w_q.transpose(1, 2).contiguous()
r_d_sz = _pack_sz(moe.down.scales.transpose(1, 2), moe.down.zeros.transpose(1, 2))
routed_list.extend([r_gu_wq, r_gu_sz, r_d_wq, r_d_sz])
arena, dense_views = _pack_into_arena(dense_list, dev)
kda_tensors = dense_views[0:21]
mla_tensors = dense_views[21:30]
moe_tensors = []
for i in range(4):
moe_tensors.extend(dense_views[30 + i * 6 : 30 + (i + 1) * 6])
moe_tensors.extend(routed_list[i * 4 : (i + 1) * 4])
scratch = [
torch.empty(cfg.hidden, dtype=torch.bfloat16, device=dev), # 0: g_hidden
torch.zeros(1, dtype=torch.float32, device=dev), # 1: g_norm_sq
torch.empty(4096, dtype=torch.float32, device=dev), # 2: g_q
torch.empty(4096, dtype=torch.float32, device=dev), # 3: g_k
torch.empty(4096, dtype=torch.float32, device=dev), # 4: g_v
torch.empty(4096, dtype=torch.float32, device=dev), # 5: g_decay
torch.empty(32, dtype=torch.float32, device=dev), # 6: g_beta
torch.empty(4096, dtype=torch.bfloat16, device=dev), # 7: g_attn_o
torch.empty(4096, dtype=torch.float32, device=dev), # 8: g_attn_o_f32
torch.zeros(32 * 512, dtype=torch.float32, device=dev), # 9: g_q_full_f32
torch.empty(32 * 64, dtype=torch.bfloat16, device=dev), # 10: g_q_rope
torch.empty(32 * 192 * 2, dtype=torch.float32, device=dev), # 11: g_mla_part_ml (float2)
torch.empty(188 * 2056 * 8, dtype=torch.bfloat16, device=dev), # 12: g_mla_part_o (padded stride!)
torch.empty(cfg.n_experts, dtype=torch.float32, device=dev), # 13: g_router_logits
torch.zeros(4, dtype=torch.int32, device=dev), # 14: g_router_done
torch.empty(9 * cfg.moe_inter, dtype=torch.bfloat16, device=dev), # 15: g_moe_hh
]
self._dense_arena = arena
self._packed_tensors = dense_views + routed_list
self._scratch = scratch
self._packed_device = dev
self._handle = ext.create_handle(
kda_tensors, mla_tensors, moe_tensors, scratch, arena,
float(cfg.routed_scaling), float(cfg.rope_theta), int(cfg.n_experts),
)
def step(self, hidden: torch.Tensor, state: list):
if self._handle == 0 or self._packed_device != hidden.device:
self._build_packed()
s0, s1, s2, s3 = state[0], state[1], state[2], state[3]
src_ckv = s3["c_kv"]
src_kr = s3["k_rope"]
ctx_len = src_ckv.shape[0]
if (
self._last_kv_buf is not None
and src_ckv.data_ptr() == self._last_kv_ptr
and src_kr.data_ptr() == self._last_kr_ptr
and ctx_len < self._last_kv_buf.shape[0]
):
dst_ckv = self._last_kv_buf
dst_kr = self._last_kr_buf
need_copy = 0
else:
cap = ctx_len + 64
dst_ckv = torch.empty((cap, src_ckv.shape[1]), dtype=src_ckv.dtype, device=src_ckv.device)
dst_kr = torch.empty((cap, src_kr.shape[1]), dtype=src_kr.dtype, device=src_kr.device)
self._last_kv_buf = dst_ckv
self._last_kr_buf = dst_kr
self._last_kv_ptr = dst_ckv.data_ptr()
self._last_kr_ptr = dst_kr.data_ptr()
need_copy = 1
out = _EXT.step_megakernel(
self._handle,
hidden,
s0["S"], s0["cq"], s0["ck"], s0["cv"],
s1["S"], s1["cq"], s1["ck"], s1["cv"],
s2["S"], s2["cq"], s2["ck"], s2["cv"],
src_ckv, src_kr,
dst_ckv, dst_kr,
ctx_len, need_copy,
)
s3["c_kv"] = dst_ckv[: ctx_len + 1]
s3["k_rope"] = dst_kr[: ctx_len + 1]
return out, state
20261118_150415_agy_gemini-4-neon-xhigh_02_kimi_linear_decode