KernelBench mega · RTX PRO 6000
Kimi-Linear Decode Gemini 4 Carbon (High)
manually audited: clean
Re-graded 2026-11-16 under grader v3 on tetra: 24.68x (23.76x before; see regrade_v3). Original audit: Gemini 4 Carbon wrote one persistent CUDA kernel, 376 CTAs (two per SM) held together by a single-word release/acquire grid barrier, that runs the whole decode token: three KDA layers, the absorbed MLA layer and a 64-expert top-8 plus shared-expert MoE after each. The int4 GEMVs build each weight as a bf16 "128 + nibble" with one OR (0x43004300) and fold the zero point in once per group against precomputed 32-element activation sums; activations stay fp32 from the bf16 RMSNorm output, so it is a true W4A16 GEMV. MLA attention runs in the 512-dim latent space on bf16 mma.sync tensor cores with a two-stage cp.async pipeline over up to 128 KV splits, and dense weights are pinned in L2 with evict_last hints while routed experts stream evict_first.
Kernel source (redacted)
"""Single-launch CUDA megakernel for Kimi-Linear W4A16 hybrid decode."""
from __future__ import annotations
from dataclasses import dataclass, field
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
OP_TYPE = "kimi_linear_w4a16_decode"
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
EPS = 1.0e-6
GROUP_SIZE = 128
@dataclass(frozen=True)
class Config:
hidden: int = 2304
kda_heads: int = 32
kda_head_dim: int = 128
short_conv: int = 4
mla_heads: int = 32
kv_lora: int = 512
qk_nope: int = 128
qk_rope: int = 64
v_head: int = 128
rope_theta: float = 10000.0
n_experts: int = 64
n_active: int = 8
n_shared: int = 1
moe_inter: int = 1024
routed_scaling: float = 2.446
group: int = 128
pattern: tuple = ("K", "K", "K", "M")
dtype: torch.dtype = field(default=torch.bfloat16)
def build_config(shape: dict) -> Config:
return Config(n_experts=int(shape.get("n_experts", 64)))
def _pack_int4(w_q: torch.Tensor) -> torch.Tensor:
lo = w_q[0::2] & 0xF
hi = w_q[1::2] & 0xF
return (lo | (hi << 4)).contiguous()
def quantize(w_io: torch.Tensor, group: int = GROUP_SIZE):
K, N = w_io.shape
ng = K // group
wg = w_io.view(ng, group, N).float()
wmin = wg.min(dim=1, keepdim=True).values
wmax = wg.max(dim=1, keepdim=True).values
scales = (wmax - wmin).clamp_min(1e-8) / 15.0
zeros = (-wmin / scales).round().clamp(0, 15)
w_q = ((wg / scales) + zeros).round().clamp(0, 15).to(torch.uint8).view(K, N)
return (
_pack_int4(w_q),
scales.squeeze(1).to(torch.bfloat16),
zeros.squeeze(1).to(torch.bfloat16),
)
class QuantLinear(nn.Module):
def __init__(self, in_f: int, out_f: int, group: int = GROUP_SIZE):
super().__init__()
assert in_f % group == 0 and in_f % 2 == 0
self.in_f, self.out_f, self.group = in_f, out_f, group
ng = in_f // group
self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8))
self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16))
def init_random(self, gen: torch.Generator, std: float = 0.02) -> None:
w = torch.randn(self.in_f, self.out_f, generator=gen) * std
wq, s, z = quantize(w, self.group)
self.w_q.copy_(wq)
self.scales.copy_(s)
self.zeros.copy_(z)
class QuantExperts(nn.Module):
def __init__(self, n: int, in_f: int, out_f: int, group: int = GROUP_SIZE):
super().__init__()
self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group
ng = in_f // group
self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8))
self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
def init_random(self, gen: torch.Generator, std: float = 0.02) -> None:
for e in range(self.n):
w = torch.randn(self.in_f, self.out_f, generator=gen) * std
wq, s, z = quantize(w, self.group)
self.w_q[e].copy_(wq)
self.scales[e].copy_(s)
self.zeros[e].copy_(z)
class KDA(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden
self.q_proj = QuantLinear(d, H * Dk, cfg.group)
self.k_proj = QuantLinear(d, H * Dk, cfg.group)
self.v_proj = QuantLinear(d, H * Dk, cfg.group)
self.g_proj = QuantLinear(d, H * Dk, cfg.group)
self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype)
self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype))
self.o_proj = QuantLinear(H * Dk, d, cfg.group)
class MLA(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
H, d = cfg.mla_heads, cfg.hidden
self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group)
self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group)
self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group)
self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group)
class MoE(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts
self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype)
self.gate = QuantExperts(E, d, m, cfg.group)
self.up = QuantExperts(E, d, m, cfg.group)
self.down = QuantExperts(E, m, d, cfg.group)
self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group)
self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group)
self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group)
class Block(nn.Module):
def __init__(self, cfg: Config, kind: str):
super().__init__()
self.kind = kind
self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
self.attn = KDA(cfg) if kind == "K" else MLA(cfg)
self.moe = MoE(cfg)
_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
#include <string.h>
#include <math.h>
constexpr int HIDDEN = 2304;
constexpr int KDA_DK = 128;
constexpr int KDA_C = 4096;
constexpr int KV_LORA = 512;
constexpr int QK_NOPE = 128;
constexpr int QK_ROPE = 64;
constexpr int V_HEAD = 128;
constexpr int MOE_INTER = 1024;
constexpr int GROUP = 128;
constexpr int QGROUP = 32;
constexpr int BLOCK_THREADS = 256;
constexpr int MAX_KV_SPLITS = 128;
constexpr int KV_SPLIT_STRIDE = 128;
// Sense-flipping single-word GPU release-acquire barrier across all resident CTAs
struct FastGridBarrier {
uint32_t* arrived;
__device__ __forceinline__ void sync() {
__barrier_sync(0);
if (threadIdx.x == 0) {
uint32_t nb = (blockIdx.x == 0) ? (0x80000000u - (gridDim.x - 1u)) : 1u;
uint32_t old_arrive;
asm volatile("atom.add.release.gpu.u32 %0, [%1], %2;\n"
: "=r"(old_arrive)
: "l"(arrived), "r"(nb)
: "memory");
uint32_t cur_arrive;
do {
asm volatile("ld.acquire.gpu.u32 %0, [%1];\n"
: "=r"(cur_arrive)
: "l"(arrived)
: "memory");
} while (((old_arrive ^ cur_arrive) & 0x80000000u) == 0u);
}
__barrier_sync(0);
}
};
struct QuantLinearPtrs {
const uint8_t* w_q;
const __nv_bfloat16* scales;
const __nv_bfloat16* zeros;
};
struct KDAPtrs {
QuantLinearPtrs q_proj;
QuantLinearPtrs k_proj;
QuantLinearPtrs v_proj;
QuantLinearPtrs g_proj;
const __nv_bfloat16* beta_w; // [32, 2304]
const __nv_bfloat16* conv_w; // [3, 4096, 4]
QuantLinearPtrs o_proj; // [4096, 2304]
};
struct MLAPtrs {
QuantLinearPtrs q_proj; // [2304, 6144]
QuantLinearPtrs kv_a; // [2304, 576]
QuantLinearPtrs kv_b; // [512, 8192]
QuantLinearPtrs o_proj; // [4096, 2304]
};
struct MoEPtrs {
const __nv_bfloat16* router_w; // [64, 2304]
QuantLinearPtrs gate; // [64, 1152, 1024]
QuantLinearPtrs up; // [64, 1152, 1024]
QuantLinearPtrs down; // [64, 512, 2304]
QuantLinearPtrs s_gate; // [1, 1152, 1024]
QuantLinearPtrs s_up; // [1, 1152, 1024]
QuantLinearPtrs s_down; // [1, 512, 2304]
};
struct KDAStatePtrs {
float* S; // [32, 128, 128]
__nv_bfloat16* cq; // [3, 4096]
__nv_bfloat16* ck; // [3, 4096]
__nv_bfloat16* cv; // [3, 4096]
};
struct MegakernelParams {
const __nv_bfloat16* attn_norm[4];
const __nv_bfloat16* moe_norm[4];
KDAPtrs kda[3];
MLAPtrs mla;
MoEPtrs moe[4];
KDAStatePtrs kda_st[3];
const __nv_bfloat16* c_kv_in;
__nv_bfloat16* c_kv_out;
const __nv_bfloat16* k_rope_in;
__nv_bfloat16* k_rope_out;
int pos;
float routed_scaling;
const __nv_bfloat16* hidden_in;
__nv_bfloat16* hidden_out;
void* workspace;
};
struct alignas(16) Workspace {
// Dedicated per-layer split atomicAdd buffers & arrival counters (zeroed once at start of step!)
alignas(16) float kda_qkvg[3][2][16384]; // 384 KB (2-way split!)
alignas(16) float mla_qkv[4][6720]; // 105 KB (4-way split!)
alignas(16) float mla_o_raw[4][4096]; // 64 KB (4-way split!)
alignas(16) float attn_o_accum[4][4][2304]; // 144 KB (4-way split!)
alignas(16) float moe_gu_accum[4][2][9 * 2048]; // 576 KB (2-way split!)
alignas(16) float moe_down_accum[4][4][2304]; // 144 KB (4-way split!)
alignas(16) uint32_t router_done[4]; // 16 B
alignas(16) uint32_t moe_down_done[4]; // 16 B
// End of zeroed section (~1.4 MB -> ~1.4 us across 376 CTAs!)
alignas(16) __nv_bfloat16 h[2304];
alignas(16) __nv_bfloat16 h_after_attn[2304];
alignas(16) __nv_bfloat16 x_norm[2304];
alignas(16) float x_qgsum[80];
alignas(16) float beta[32];
alignas(16) __nv_bfloat16 attn_o_bf16[4096];
alignas(16) float router_logits[64];
alignas(16) int topk_idx[8];
alignas(16) float topk_w[12];
alignas(16) __nv_bfloat16 q_latent_bf16[32 * 512];
alignas(16) __nv_bfloat16 q_rope_bf16[32 * 64];
alignas(16) float o_latent[32 * 512];
alignas(16) float attn_m[32 * KV_SPLIT_STRIDE];
alignas(16) float attn_l[32 * KV_SPLIT_STRIDE];
alignas(16) __nv_bfloat16 attn_o_splits[MAX_KV_SPLITS * 32 * 512];
alignas(16) uint32_t grid_bar[4];
};
__device__ __forceinline__ float silu_f(float x) {
return x / (1.0f + __expf(-x));
}
__device__ __forceinline__ float warp_sum(float v) {
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
v += __shfl_xor_sync(0xffffffff, v, offset);
}
return v;
}
__device__ __forceinline__ uint64_t make_evict_first_policy() {
uint64_t policy;
asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;\n" : "=l"(policy));
return policy;
}
__device__ __forceinline__ uint64_t make_evict_last_policy() {
uint64_t policy;
asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;\n" : "=l"(policy));
return policy;
}
__device__ __forceinline__ uint32_t load_u32_policy(const void* ptr, uint64_t policy) {
uint32_t ret;
asm volatile("ld.global.L1::no_allocate.L2::cache_hint.u32 %0, [%1], %2;\n"
: "=r"(ret)
: "l"(ptr), "l"(policy));
return ret;
}
__device__ __forceinline__ uint2 load_u2_policy(const void* ptr, uint64_t policy) {
uint2 ret;
asm volatile("ld.global.L1::no_allocate.L2::cache_hint.v2.u32 {%0, %1}, [%2], %3;\n"
: "=r"(ret.x), "=r"(ret.y)
: "l"(ptr), "l"(policy));
return ret;
}
__device__ __forceinline__ uint4 load_u4_policy(const void* ptr, uint64_t policy) {
uint4 ret;
asm volatile("ld.global.L1::no_allocate.L2::cache_hint.v4.u32 {%0, %1, %2, %3}, [%4], %5;\n"
: "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w)
: "l"(ptr), "l"(policy));
return ret;
}
__device__ __forceinline__ void cp_async_16_evict_first(void* smem_ptr, const void* glob_ptr, bool pred, uint64_t policy) {
uint32_t smem_addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
int src_bytes = pred ? 16 : 0;
asm volatile(
"cp.async.cg.shared.global.L2::cache_hint [%0], [%1], 16, %2, %3;\n"
:: "r"(smem_addr), "l"(glob_ptr), "r"(src_bytes), "l"(policy)
);
}
__device__ __forceinline__ void cp_async_commit() {
asm volatile("cp.async.commit_group;\n" ::: "memory");
}
template <int N>
__device__ __forceinline__ void cp_async_wait() {
asm volatile("cp.async.wait_group %0;\n" :: "n"(N) : "memory");
}
__device__ __forceinline__ uint32_t load_u32_streaming(const void* ptr) {
uint32_t ret;
asm volatile("ld.global.cs.u32 %0, [%1];\n" : "=r"(ret) : "l"(ptr));
return ret;
}
__device__ __forceinline__ void store_u32_streaming(void* ptr, uint32_t v) {
asm volatile("st.global.cs.u32 [%0], %1;\n" :: "l"(ptr), "r"(v) : "memory");
}
__device__ __forceinline__ uint2 load_u2_streaming(const void* ptr) {
uint2 ret;
asm volatile("ld.global.cs.v2.u32 {%0, %1}, [%2];\n"
: "=r"(ret.x), "=r"(ret.y)
: "l"(ptr));
return ret;
}
__device__ __forceinline__ uint4 load_u4_streaming(const void* ptr) {
uint4 ret;
asm volatile("ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n"
: "=r"(ret.x), "=r"(ret.y), "=r"(ret.z), "=r"(ret.w)
: "l"(ptr));
return ret;
}
__device__ __forceinline__ void store_u4_streaming(void* ptr, uint4 v) {
asm volatile("st.global.cs.v4.u32 [%0], {%1, %2, %3, %4};\n"
:
: "l"(ptr), "r"(v.x), "r"(v.y), "r"(v.z), "r"(v.w)
: "memory");
}
__device__ __forceinline__ void mma_m16n8k16_bf16(
float& d0, float& d1, float& d2, float& d3,
uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
uint32_t b0, uint32_t b1
) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
: "=f"(d0), "=f"(d1), "=f"(d2), "=f"(d3)
: "r"(a0), "r"(a1), "r"(a2), "r"(a3),
"r"(b0), "r"(b1),
"f"(d0), "f"(d1), "f"(d2), "f"(d3)
);
}
__device__ __forceinline__ void ldmatrix_x2_trans(uint32_t& b0, uint32_t& b1, const void* smem_ptr) {
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
asm volatile(
"ldmatrix.sync.aligned.m8n8.x2.trans.shared.b16 {%0, %1}, [%2];\n"
: "=r"(b0), "=r"(b1)
: "r"(addr)
);
}
__device__ __forceinline__ void load_x_norm_to_smem(
const Workspace* ws,
float* __restrict__ sx,
float* __restrict__ sx_qgsum
) {
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 xv = *reinterpret_cast<const uint4*>(&ws->x_norm[base]);
const __nv_bfloat16* xb = reinterpret_cast<const __nv_bfloat16*>(&xv);
#pragma unroll
for (int m = 0; m < 8; ++m) {
sx[base + m] = __bfloat162float(xb[m]);
}
}
}
if (threadIdx.x < (HIDDEN / QGROUP)) {
sx_qgsum[threadIdx.x] = ws->x_qgsum[threadIdx.x];
}
__syncthreads();
}
__device__ __forceinline__ void cta_rmsnorm_from_h_2304(
const __nv_bfloat16* __restrict__ h_in,
const __nv_bfloat16* __restrict__ norm_w,
float* __restrict__ sx,
float* __restrict__ sx_qgsum, // nullptr if qgsum not needed
float* __restrict__ s_red
) {
float reg_h[16];
float sq_sum = 0.0f;
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 hv = *reinterpret_cast<const uint4*>(&h_in[base]);
const __nv_bfloat16* hb = reinterpret_cast<const __nv_bfloat16*>(&hv);
#pragma unroll
for (int m = 0; m < 8; ++m) {
float vf = __bfloat162float(hb[m]);
reg_h[pass * 8 + m] = vf;
sq_sum += vf * vf;
}
}
}
int wid = threadIdx.x >> 5;
int lane = threadIdx.x & 31;
sq_sum = warp_sum(sq_sum);
if (lane == 0) s_red[wid] = sq_sum;
__syncthreads();
if (wid == 0) {
float v = (lane < 8) ? s_red[lane] : 0.0f;
v = warp_sum(v);
if (lane == 0) s_red[0] = rsqrtf(v * (1.0f / float(HIDDEN)) + 1.0e-6f);
}
__syncthreads();
float inv_rms = s_red[0];
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 wv = *reinterpret_cast<const uint4*>(&norm_w[base]);
const __nv_bfloat16* wb = reinterpret_cast<const __nv_bfloat16*>(&wv);
#pragma unroll
for (int m = 0; m < 8; ++m) {
__nv_bfloat16 b = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m]));
sx[base + m] = __bfloat162float(b);
}
}
}
__syncthreads();
if (sx_qgsum != nullptr) {
for (int qg = wid; qg < (HIDDEN / QGROUP); qg += 8) {
float s = warp_sum(sx[qg * 32 + lane]);
if (lane == 0) sx_qgsum[qg] = s;
}
__syncthreads();
}
}
__device__ __forceinline__ void reduce4_only_2304(
const __nv_bfloat16* __restrict__ res_in,
const float (&accum)[4][HIDDEN],
__nv_bfloat16* __restrict__ res_out
) {
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 hv = *reinterpret_cast<const uint4*>(&res_in[base]);
const __nv_bfloat16* hb = reinterpret_cast<const __nv_bfloat16*>(&hv);
float4 d0a = *reinterpret_cast<const float4*>(&accum[0][base]);
float4 d0b = *reinterpret_cast<const float4*>(&accum[0][base + 4]);
float4 d1a = *reinterpret_cast<const float4*>(&accum[1][base]);
float4 d1b = *reinterpret_cast<const float4*>(&accum[1][base + 4]);
float4 d2a = *reinterpret_cast<const float4*>(&accum[2][base]);
float4 d2b = *reinterpret_cast<const float4*>(&accum[2][base + 4]);
float4 d3a = *reinterpret_cast<const float4*>(&accum[3][base]);
float4 d3b = *reinterpret_cast<const float4*>(&accum[3][base + 4]);
float ds[8] = {
d0a.x + d1a.x + d2a.x + d3a.x,
d0a.y + d1a.y + d2a.y + d3a.y,
d0a.z + d1a.z + d2a.z + d3a.z,
d0a.w + d1a.w + d2a.w + d3a.w,
d0b.x + d1b.x + d2b.x + d3b.x,
d0b.y + d1b.y + d2b.y + d3b.y,
d0b.z + d1b.z + d2b.z + d3b.z,
d0b.w + d1b.w + d2b.w + d3b.w
};
uint4 out_h;
__nv_bfloat16* ob = reinterpret_cast<__nv_bfloat16*>(&out_h);
#pragma unroll
for (int m = 0; m < 8; ++m) {
__nv_bfloat16 mo = __float2bfloat16_rn(ds[m]);
ob[m] = __float2bfloat16_rn(__bfloat162float(hb[m]) + __bfloat162float(mo));
}
*reinterpret_cast<uint4*>(&res_out[base]) = out_h;
}
}
}
template <int QGPW, bool STREAM_LOAD, bool CHECK_VALID = false>
__device__ __forceinline__ void warp_int4_gemv_128(
const float* __restrict__ sx,
const float* __restrict__ sx_qgsum,
const uint8_t* __restrict__ w_q,
const __nv_bfloat16* __restrict__ scales,
const __nv_bfloat16* __restrict__ zeros,
float* __restrict__ out_accum,
int N_stride,
int col_tile_offset,
int qg_start,
int valid_cols = 128
) {
const int lane = threadIdx.x & 31;
const int col_grp = lane & 15;
const int row_sub = lane >> 4;
const int local_col = col_grp * 8;
if constexpr (CHECK_VALID) {
if (local_col >= valid_cols) return;
}
const int col_base = col_tile_offset + local_col;
float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
const int kb_base = qg_start * 16 + row_sub;
constexpr int STEPS = QGPW * 8;
const uint64_t pol = STREAM_LOAD ? make_evict_first_policy() : make_evict_last_policy();
#pragma unroll 8
for (int step = 0; step < STEPS; ++step) {
int kb = kb_base + step * 2;
const void* w_ptr = &w_q[kb * N_stride + col_base];
uint2 w_vec = load_u2_policy(w_ptr, pol);
float x_lo = sx[kb * 2];
float x_hi = sx[kb * 2 + 1];
uint32_t words[2] = {w_vec.x, w_vec.y};
#pragma unroll
for (int w = 0; w < 2; ++w) {
uint32_t v = words[w];
uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
uint32_t v_shr8 = v >> 8;
uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;
float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));
g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x;
g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x;
g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y;
g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y;
}
}
uint32_t mask = 0xffffffffu;
if constexpr (CHECK_VALID) {
mask = (valid_cols >= 128) ? 0xffffffffu : 0x00ff00ffu;
}
#pragma unroll
for (int c = 0; c < 8; ++c) {
g_acc[c] += __shfl_down_sync(mask, g_acc[c], 16);
}
if (row_sub == 0) {
int g = qg_start >> 2;
float xsum = sx_qgsum[qg_start];
if constexpr (QGPW >= 2) xsum += sx_qgsum[qg_start + 1];
if constexpr (QGPW >= 4) xsum += sx_qgsum[qg_start + 2] + sx_qgsum[qg_start + 3];
const void* s_ptr = &scales[g * N_stride + col_base];
const void* z_ptr = &zeros[g * N_stride + col_base];
uint4 s_vec = load_u4_policy(s_ptr, pol);
uint4 z_vec = load_u4_policy(z_ptr, pol);
const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_vec);
const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_vec);
#pragma unroll
for (int c = 0; c < 8; ++c) {
float z_eff = __bfloat162float(zb[c]) + 128.0f;
float val = __bfloat162float(sb[c]) * (g_acc[c] - z_eff * xsum);
atomicAdd(&out_accum[local_col + c], val);
}
}
}
union alignas(16) SharedStorage {
struct {
alignas(16) float sx[4096]; // 16 KB
alignas(16) float sx_qgsum[128]; // 512 B
alignas(16) float s_red[32]; // 128 B
alignas(16) int topk_idx[8];
alignas(16) float topk_w[12];
} gemv;
struct {
alignas(16) float inter[512]; // 2 KB (Half-K = 512 per CTA!)
alignas(16) float qgsum[16]; // 64 B
alignas(16) float raw_acc[8][128]; // 4 KB (8-warp smem reduction!)
} moe_down;
struct {
alignas(16) float sq[128];
alignas(16) float sk[128];
alignas(16) float sv[128];
alignas(16) float sdec[128];
alignas(16) float s_red[8][16];
} kda_rec;
struct {
alignas(16) __nv_bfloat16 ckv[2][16][520]; // 33,280 B (2-stage cp.async double buffer!)
alignas(16) __nv_bfloat16 krp[2][16][72]; // 4,608 B
alignas(16) float s_part[4][32][16]; // 8,192 B
alignas(16) __nv_bfloat16 s_p[32][24]; // 1,536 B
alignas(16) float s_escale[32]; // 128 B
alignas(16) float s_m[32]; // 128 B
alignas(16) float s_l[32]; // 128 B
} mla_tc; // 48,000 B -> 2 blocks/SM (96,000 B <= 101,376 B)!
struct {
alignas(16) float scale_s[KV_SPLIT_STRIDE];
float g_inv_sum;
alignas(16) float partial[8][64]; // 2 KB (64 cols per CTA across 256 CTAs!)
} mla_comb;
};
__device__ void run_moe_block(
FastGridBarrier& grid,
const MegakernelParams& params,
Workspace* ws,
SharedStorage& smem,
int layer_idx,
bool write_final_out
) {
const int lane = threadIdx.x & 31;
const int wid = threadIdx.x >> 5;
const MoEPtrs& moe = params.moe[layer_idx];
// Stage M1: CTAs 0..64 fuse Residual + MoE RMSNorm + 64-Expert Router GEMV + Last-CTA Top-8!
// Only 65 CTAs read attn_o_accum (no 376-CTA hotspot, zero serial tail on o_proj!).
if (blockIdx.x < 65) {
float reg_h[16];
float sq_sum = 0.0f;
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 hv = *reinterpret_cast<const uint4*>(&ws->h[base]);
const __nv_bfloat16* hb = reinterpret_cast<const __nv_bfloat16*>(&hv);
float4 a0a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][0][base]);
float4 a0b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][0][base + 4]);
float4 a1a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][1][base]);
float4 a1b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][1][base + 4]);
float4 a2a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][2][base]);
float4 a2b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][2][base + 4]);
float4 a3a = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][3][base]);
float4 a3b = *reinterpret_cast<const float4*>(&ws->attn_o_accum[layer_idx][3][base + 4]);
float as[8] = {
a0a.x + a1a.x + a2a.x + a3a.x,
a0a.y + a1a.y + a2a.y + a3a.y,
a0a.z + a1a.z + a2a.z + a3a.z,
a0a.w + a1a.w + a2a.w + a3a.w,
a0b.x + a1b.x + a2b.x + a3b.x,
a0b.y + a1b.y + a2b.y + a3b.y,
a0b.z + a1b.z + a2b.z + a3b.z,
a0b.w + a1b.w + a2b.w + a3b.w
};
uint4 out_h;
__nv_bfloat16* ob = reinterpret_cast<__nv_bfloat16*>(&out_h);
#pragma unroll
for (int m = 0; m < 8; ++m) {
__nv_bfloat16 ao = __float2bfloat16_rn(as[m]);
__nv_bfloat16 hm = __float2bfloat16_rn(__bfloat162float(hb[m]) + __bfloat162float(ao));
ob[m] = hm;
float vf = __bfloat162float(hm);
reg_h[pass * 8 + m] = vf;
sq_sum += vf * vf;
}
if (blockIdx.x == 64) {
*reinterpret_cast<uint4*>(&ws->h_after_attn[base]) = out_h;
}
}
}
sq_sum = warp_sum(sq_sum);
if (lane == 0) smem.gemv.s_red[wid] = sq_sum;
__syncthreads();
if (wid == 0) {
float v = (lane < 8) ? smem.gemv.s_red[lane] : 0.0f;
v = warp_sum(v);
if (lane == 0) smem.gemv.s_red[0] = rsqrtf(v * (1.0f / float(HIDDEN)) + 1.0e-6f);
}
__syncthreads();
float inv_rms = smem.gemv.s_red[0];
const __nv_bfloat16* mw = params.moe_norm[layer_idx];
if (blockIdx.x == 64) {
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 wv = *reinterpret_cast<const uint4*>(&mw[base]);
const __nv_bfloat16* wb = reinterpret_cast<const __nv_bfloat16*>(&wv);
uint4 out_xn;
__nv_bfloat16* xnb = reinterpret_cast<__nv_bfloat16*>(&out_xn);
#pragma unroll
for (int m = 0; m < 8; ++m) {
__nv_bfloat16 xn = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m]));
xnb[m] = xn;
smem.gemv.sx[base + m] = __bfloat162float(xn);
}
*reinterpret_cast<uint4*>(&ws->x_norm[base]) = out_xn;
}
}
__syncthreads();
for (int qg = wid; qg < (HIDDEN / QGROUP); qg += 8) {
float s = warp_sum(smem.gemv.sx[qg * 32 + lane]);
if (lane == 0) ws->x_qgsum[qg] = s;
}
} else {
int e = blockIdx.x;
const __nv_bfloat16* rw = &moe.router_w[e * HIDDEN];
float dot = 0.0f;
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int vec_idx = pass * 256 + threadIdx.x;
if (vec_idx < 288) {
int base = vec_idx * 8;
uint4 wv = *reinterpret_cast<const uint4*>(&mw[base]);
uint4 rv = *reinterpret_cast<const uint4*>(&rw[base]);
const __nv_bfloat16* wb = reinterpret_cast<const __nv_bfloat16*>(&wv);
const __nv_bfloat16* rb = reinterpret_cast<const __nv_bfloat16*>(&rv);
#pragma unroll
for (int m = 0; m < 8; ++m) {
__nv_bfloat16 xn = __float2bfloat16_rn(reg_h[pass * 8 + m] * inv_rms * __bfloat162float(wb[m]));
dot += __bfloat162float(xn) * __bfloat162float(rb[m]);
}
}
}
dot = warp_sum(dot);
if (lane == 0) smem.gemv.s_red[8 + wid] = dot;
__syncthreads();
if (wid == 0) {
float s = (lane < 8) ? smem.gemv.s_red[8 + lane] : 0.0f;
s = warp_sum(s);
uint32_t prev_done = 0u;
if (lane == 0) {
float logit = __bfloat162float(__float2bfloat16_rn(s));
asm volatile("st.global.release.gpu.f32 [%0], %1;\n"
:: "l"(&ws->router_logits[e]), "f"(logit) : "memory");
asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;\n"
: "=r"(prev_done)
: "l"(&ws->router_done[layer_idx])
: "memory");
}
prev_done = __shfl_sync(0xffffffff, prev_done, 0);
if (prev_done == 63u) {
int i0 = 2 * lane;
int i1 = 2 * lane + 1;
float v0, v1;
asm volatile("ld.global.acquire.gpu.v2.f32 {%0, %1}, [%2];\n"
: "=f"(v0), "=f"(v1)
: "l"(&ws->router_logits[i0]));
float lmax = fmaxf(v0, v1);
#pragma unroll
for (int off = 16; off > 0; off >>= 1) {
lmax = fmaxf(lmax, __shfl_xor_sync(0xffffffff, lmax, off));
}
float p0 = __expf(v0 - lmax);
float p1 = __expf(v1 - lmax);
float psum = warp_sum(p0 + p1);
p0 /= psum;
p1 /= psum;
float w_sum = 0.0f;
#pragma unroll
for (int k = 0; k < 8; ++k) {
float best_v = (p0 >= p1) ? p0 : p1;
int best_i = (p0 >= p1) ? i0 : i1;
#pragma unroll
for (int off = 16; off > 0; off >>= 1) {
float ov = __shfl_xor_sync(0xffffffff, best_v, off);
int oi = __shfl_xor_sync(0xffffffff, best_i, off);
if (ov > best_v || (ov == best_v && oi < best_i)) {
best_v = ov;
best_i = oi;
}
}
if (lane == 0) {
ws->topk_idx[k] = best_i;
smem.gemv.topk_w[k] = best_v;
}
w_sum += best_v;
if (i0 == best_i) p0 = -1.0f;
if (i1 == best_i) p1 = -1.0f;
}
if (lane == 0) {
float scale = params.routed_scaling / (w_sum + 1.0e-9f);
#pragma unroll
for (int k = 0; k < 8; ++k) {
ws->topk_w[k] = smem.gemv.topk_w[k] * scale;
}
ws->topk_w[8] = 1.0f;
}
}
}
}
}
grid.sync();
// Stage M2: Row-Contiguous 1-Wave Gate+Up GEMV (zero Top-K stall, 4.5 KB x_norm load!)
if (blockIdx.x < 324) {
int slot = blockIdx.x / 36;
int rem = blockIdx.x % 36;
int is_up = rem / 18;
int kc = rem % 18;
int ltn = wid; // 0..7
if (threadIdx.x == 0 && slot < 8) {
smem.gemv.topk_idx[0] = ws->topk_idx[slot];
}
load_x_norm_to_smem(ws, smem.gemv.sx, smem.gemv.sx_qgsum);
size_t wq_exp_stride = size_t(HIDDEN / 2) * MOE_INTER;
size_t sz_exp_stride = size_t(HIDDEN / GROUP) * MOE_INTER;
float* out_ptr = &ws->moe_gu_accum[layer_idx][kc & 1][slot * 2048 + is_up * 1024 + ltn * 128];
if (slot < 8) {
int exp_id = smem.gemv.topk_idx[0];
const QuantLinearPtrs* ql = is_up ? &moe.up : &moe.gate;
warp_int4_gemv_128<4, true>(
smem.gemv.sx, smem.gemv.sx_qgsum,
ql->w_q + exp_id * wq_exp_stride,
ql->scales + exp_id * sz_exp_stride,
ql->zeros + exp_id * sz_exp_stride,
out_ptr, MOE_INTER, ltn * 128, kc * 4
);
} else {
const QuantLinearPtrs* ql = is_up ? &moe.s_up : &moe.s_gate;
warp_int4_gemv_128<4, false>(
smem.gemv.sx, smem.gemv.sx_qgsum,
ql->w_q, ql->scales, ql->zeros,
out_ptr, MOE_INTER, ltn * 128, kc * 4
);
}
}
grid.sync();
// Stage M3: Half-K SiLU*Up + 8-Warp In-CTA Shared-Memory Reduction Down GEMV + Last-CTA Residual!
if (blockIdx.x < 324) {
int slot = blockIdx.x / 36;
int rem = blockIdx.x % 36;
int k_half = rem / 18; // 0..1 (512 K-channels per CTA!)
int ltn = rem % 18; // 0..17 (128 output cols per CTA!)
float w_s = ws->topk_w[slot];
int exp_id = (slot < 8) ? ws->topk_idx[slot] : 0;
{
int m = threadIdx.x * 2;
int g_idx = slot * 2048 + k_half * 512 + m;
float2 g0 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][0][g_idx]);
float2 g1 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][1][g_idx]);
float2 u0 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][0][g_idx + 1024]);
float2 u1 = *reinterpret_cast<const float2*>(&ws->moe_gu_accum[layer_idx][1][g_idx + 1024]);
float2 prod = make_float2(
w_s * silu_f(g0.x + g1.x) * (u0.x + u1.x),
w_s * silu_f(g0.y + g1.y) * (u0.y + u1.y)
);
*reinterpret_cast<float2*>(&smem.moe_down.inter[m]) = prod;
}
__syncthreads();
for (int qg = wid; qg < 16; qg += 8) {
float s = warp_sum(smem.moe_down.inter[qg * 32 + lane]);
if (lane == 0) smem.moe_down.qgsum[qg] = s;
}
__syncthreads();
size_t wq_exp_stride = size_t(MOE_INTER / 2) * HIDDEN;
size_t sz_exp_stride = size_t(MOE_INTER / GROUP) * HIDDEN;
const uint8_t* w_q_ptr = ((slot < 8) ? (moe.down.w_q + exp_id * wq_exp_stride) : moe.s_down.w_q)
+ size_t(k_half * 256) * HIDDEN;
const __nv_bfloat16* s_ptr_base = ((slot < 8) ? (moe.down.scales + exp_id * sz_exp_stride) : moe.s_down.scales)
+ size_t(k_half * 4) * HIDDEN;
const __nv_bfloat16* z_ptr_base = ((slot < 8) ? (moe.down.zeros + exp_id * sz_exp_stride) : moe.s_down.zeros)
+ size_t(k_half * 4) * HIDDEN;
const uint64_t pol = (slot < 8) ? make_evict_first_policy() : make_evict_last_policy();
const int col_grp = lane & 15;
const int row_sub = lane >> 4;
const int local_col = col_grp * 8;
const int col_base = ltn * 128 + local_col;
const int kb_base = wid * 32 + row_sub;
float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
#pragma unroll 8
for (int step = 0; step < 16; ++step) {
int kb = kb_base + step * 2;
uint2 w_vec = load_u2_policy(&w_q_ptr[kb * HIDDEN + col_base], pol);
float x_lo = smem.moe_down.inter[kb * 2];
float x_hi = smem.moe_down.inter[kb * 2 + 1];
uint32_t words[2] = {w_vec.x, w_vec.y};
#pragma unroll
for (int w = 0; w < 2; ++w) {
uint32_t v = words[w];
uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
uint32_t v_shr8 = v >> 8;
uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;
float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));
g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x;
g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x;
g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y;
g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y;
}
}
#pragma unroll
for (int c = 0; c < 8; ++c) {
g_acc[c] += __shfl_down_sync(0xffffffffu, g_acc[c], 16);
}
if (row_sub == 0) {
*reinterpret_cast<float4*>(&smem.moe_down.raw_acc[wid][local_col]) = make_float4(g_acc[0], g_acc[1], g_acc[2], g_acc[3]);
*reinterpret_cast<float4*>(&smem.moe_down.raw_acc[wid][local_col + 4]) = make_float4(g_acc[4], g_acc[5], g_acc[6], g_acc[7]);
}
__syncthreads();
if (wid < 4) {
int g_loc = wid;
int col4 = lane * 4;
int g_col = ltn * 128 + col4;
float xsum = smem.moe_down.qgsum[g_loc * 4 + 0] + smem.moe_down.qgsum[g_loc * 4 + 1]
+ smem.moe_down.qgsum[g_loc * 4 + 2] + smem.moe_down.qgsum[g_loc * 4 + 3];
uint2 s_vec = load_u2_policy(&s_ptr_base[g_loc * HIDDEN + g_col], pol);
uint2 z_vec = load_u2_policy(&z_ptr_base[g_loc * HIDDEN + g_col], pol);
const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_vec);
const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_vec);
float4 r0 = *reinterpret_cast<const float4*>(&smem.moe_down.raw_acc[2 * g_loc][col4]);
float4 r1 = *reinterpret_cast<const float4*>(&smem.moe_down.raw_acc[2 * g_loc + 1][col4]);
float raw[4] = {r0.x + r1.x, r0.y + r1.y, r0.z + r1.z, r0.w + r1.w};
float deq[4];
#pragma unroll
for (int c = 0; c < 4; ++c) {
deq[c] = __bfloat162float(sb[c]) * (raw[c] - (__bfloat162float(zb[c]) + 128.0f) * xsum);
}
*reinterpret_cast<float4*>(&smem.moe_down.raw_acc[2 * g_loc][col4]) = make_float4(deq[0], deq[1], deq[2], deq[3]);
}
__syncthreads();
if (threadIdx.x < 128) {
int col = threadIdx.x;
float sum = smem.moe_down.raw_acc[0][col] + smem.moe_down.raw_acc[2][col]
+ smem.moe_down.raw_acc[4][col] + smem.moe_down.raw_acc[6][col];
atomicAdd(&ws->moe_down_accum[layer_idx][(slot * 2 + k_half) & 3][ltn * 128 + col], sum);
}
__syncthreads();
if (threadIdx.x == 0) {
uint32_t prev;
asm volatile("atom.add.acq_rel.gpu.u32 %0, [%1], 1;\n"
: "=r"(prev)
: "l"(&ws->moe_down_done[layer_idx])
: "memory");
smem.moe_down.qgsum[0] = __uint_as_float(prev);
}
__syncthreads();
uint32_t prev_m3 = __float_as_uint(smem.moe_down.qgsum[0]);
if (prev_m3 == 323u) {
reduce4_only_2304(
ws->h_after_attn, ws->moe_down_accum[layer_idx],
write_final_out ? params.hidden_out : ws->h
);
}
}
grid.sync();
}
__device__ void run_kda_layer(
FastGridBarrier& grid,
const MegakernelParams& params,
Workspace* ws,
SharedStorage& smem,
int layer_idx
) {
const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x;
const int global_warp = global_tid >> 5;
const int lane = threadIdx.x & 31;
const int wid = threadIdx.x >> 5;
const KDAPtrs& kda = params.kda[layer_idx];
const KDAStatePtrs& st = params.kda_st[layer_idx];
// Stage K1: CTAs 0..287 run RMSNorm + Row-Contiguous (q,k,v,g) GEMV; idle CTAs 288..291 compute beta[0..31]!
if (blockIdx.x < 288) {
cta_rmsnorm_from_h_2304(
ws->h, params.attn_norm[layer_idx],
smem.gemv.sx, smem.gemv.sx_qgsum, smem.gemv.s_red
);
int ltn = global_warp & 31;
int rem = global_warp >> 5; // 0..71
int which = rem / 18; // 0..3
int kc = rem % 18; // 0..17
int tn = which * 32 + ltn;
const QuantLinearPtrs* ql = (which == 0) ? &kda.q_proj :
(which == 1) ? &kda.k_proj :
(which == 2) ? &kda.v_proj : &kda.g_proj;
warp_int4_gemv_128<4, false>(
smem.gemv.sx, smem.gemv.sx_qgsum,
ql->w_q, ql->scales, ql->zeros,
&ws->kda_qkvg[layer_idx][kc & 1][tn * 128],
KDA_C, ltn * 128, kc * 4
);
} else if (blockIdx.x < 292) {
cta_rmsnorm_from_h_2304(
ws->h, params.attn_norm[layer_idx],
smem.gemv.sx, nullptr, smem.gemv.s_red
);
int h = (blockIdx.x - 288) * 8 + wid;
const __nv_bfloat16* bw = &kda.beta_w[h * HIDDEN];
float dot = 0.0f;
for (int k = lane * 8; k < HIDDEN; k += 256) {
uint4 bv = *reinterpret_cast<const uint4*>(&bw[k]);
const __nv_bfloat16* bb = reinterpret_cast<const __nv_bfloat16*>(&bv);
#pragma unroll
for (int m = 0; m < 8; ++m) {
dot += smem.gemv.sx[k + m] * __bfloat162float(bb[m]);
}
}
dot = warp_sum(dot);
if (lane == 0) {
float bx = __bfloat162float(__float2bfloat16_rn(dot));
ws->beta[h] = 1.0f / (1.0f + __expf(-bx));
}
}
grid.sync();
// Stage K2: Short Conv + Decay + Recurrent S[32, 128, 128] update!
{
const int col_grp = lane & 7;
const int row_sub = lane >> 3;
const int i_tid = wid * 4 + row_sub;
const float q_scale = rsqrtf(float(KDA_DK));
if (blockIdx.x < 256) {
int h = blockIdx.x >> 3;
int j_tile = blockIdx.x & 7;
int j_base = j_tile * 16;
if (threadIdx.x < 128) {
int d = threadIdx.x;
int c = h * 128 + d;
float q_sum = ws->kda_qkvg[layer_idx][0][c] + ws->kda_qkvg[layer_idx][1][c];
float q_raw = __bfloat162float(__float2bfloat16_rn(q_sum));
const __nv_bfloat16* cw_q = &kda.conv_w[(0 * KDA_C + c) * 4];
float q_c = __bfloat162float(st.cq[0 * KDA_C + c]) * __bfloat162float(cw_q[0])
+ __bfloat162float(st.cq[1 * KDA_C + c]) * __bfloat162float(cw_q[1])
+ __bfloat162float(st.cq[2 * KDA_C + c]) * __bfloat162float(cw_q[2])
+ q_raw * __bfloat162float(cw_q[3]);
smem.kda_rec.sq[d] = __bfloat162float(__float2bfloat16_rn(silu_f(q_c))) * q_scale;
float k_sum = ws->kda_qkvg[layer_idx][0][KDA_C + c] + ws->kda_qkvg[layer_idx][1][KDA_C + c];
float k_raw = __bfloat162float(__float2bfloat16_rn(k_sum));
const __nv_bfloat16* cw_k = &kda.conv_w[(1 * KDA_C + c) * 4];
float k_c = __bfloat162float(st.ck[0 * KDA_C + c]) * __bfloat162float(cw_k[0])
+ __bfloat162float(st.ck[1 * KDA_C + c]) * __bfloat162float(cw_k[1])
+ __bfloat162float(st.ck[2 * KDA_C + c]) * __bfloat162float(cw_k[2])
+ k_raw * __bfloat162float(cw_k[3]);
smem.kda_rec.sk[d] = __bfloat162float(__float2bfloat16_rn(silu_f(k_c)));
float v_sum = ws->kda_qkvg[layer_idx][0][2 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][2 * KDA_C + c];
float v_raw = __bfloat162float(__float2bfloat16_rn(v_sum));
const __nv_bfloat16* cw_v = &kda.conv_w[(2 * KDA_C + c) * 4];
float v_c = __bfloat162float(st.cv[0 * KDA_C + c]) * __bfloat162float(cw_v[0])
+ __bfloat162float(st.cv[1 * KDA_C + c]) * __bfloat162float(cw_v[1])
+ __bfloat162float(st.cv[2 * KDA_C + c]) * __bfloat162float(cw_v[2])
+ v_raw * __bfloat162float(cw_v[3]);
smem.kda_rec.sv[d] = __bfloat162float(__float2bfloat16_rn(silu_f(v_c)));
float g_sum = ws->kda_qkvg[layer_idx][0][3 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][3 * KDA_C + c];
float gx = __bfloat162float(__float2bfloat16_rn(g_sum));
float sp = (gx > 20.0f) ? gx : log1pf(__expf(gx));
smem.kda_rec.sdec[d] = __expf(-sp);
}
__syncthreads();
float* S_head = &st.S[h * 128 * 128];
int j0 = j_base + col_grp * 2;
float2 s_reg[4];
float2 pred = {0.f, 0.f};
#pragma unroll
for (int it = 0; it < 4; ++it) {
int i = it * 32 + i_tid;
float2 sv = *reinterpret_cast<const float2*>(&S_head[i * 128 + j0]);
float dec = smem.kda_rec.sdec[i];
sv.x *= dec;
sv.y *= dec;
s_reg[it] = sv;
float kv = smem.kda_rec.sk[i];
pred.x += sv.x * kv;
pred.y += sv.y * kv;
}
pred.x += __shfl_down_sync(0xffffffff, pred.x, 8);
pred.y += __shfl_down_sync(0xffffffff, pred.y, 8);
pred.x += __shfl_down_sync(0xffffffff, pred.x, 16);
pred.y += __shfl_down_sync(0xffffffff, pred.y, 16);
if (row_sub == 0) {
smem.kda_rec.s_red[wid][col_grp * 2 + 0] = pred.x;
smem.kda_rec.s_red[wid][col_grp * 2 + 1] = pred.y;
}
__syncthreads();
float beta_h = ws->beta[h];
float p0 = 0.f, p1 = 0.f;
#pragma unroll
for (int w = 0; w < 8; ++w) {
p0 += smem.kda_rec.s_red[w][col_grp * 2 + 0];
p1 += smem.kda_rec.s_red[w][col_grp * 2 + 1];
}
float d0 = beta_h * (smem.kda_rec.sv[j0 + 0] - p0);
float d1 = beta_h * (smem.kda_rec.sv[j0 + 1] - p1);
__syncthreads();
float2 o_part = {0.f, 0.f};
#pragma unroll
for (int it = 0; it < 4; ++it) {
int i = it * 32 + i_tid;
float kv = smem.kda_rec.sk[i];
float qv = smem.kda_rec.sq[i];
float2 sv = s_reg[it];
sv.x += kv * d0;
sv.y += kv * d1;
*reinterpret_cast<float2*>(&S_head[i * 128 + j0]) = sv;
o_part.x += sv.x * qv;
o_part.y += sv.y * qv;
}
o_part.x += __shfl_down_sync(0xffffffff, o_part.x, 8);
o_part.y += __shfl_down_sync(0xffffffff, o_part.y, 8);
o_part.x += __shfl_down_sync(0xffffffff, o_part.x, 16);
o_part.y += __shfl_down_sync(0xffffffff, o_part.y, 16);
if (row_sub == 0) {
smem.kda_rec.s_red[wid][col_grp * 2 + 0] = o_part.x;
smem.kda_rec.s_red[wid][col_grp * 2 + 1] = o_part.y;
}
__syncthreads();
if (threadIdx.x < 16) {
int jl = threadIdx.x;
float osum = 0.f;
#pragma unroll
for (int w = 0; w < 8; ++w) osum += smem.kda_rec.s_red[w][jl];
ws->attn_o_bf16[h * 128 + j_base + jl] = __float2bfloat16_rn(osum);
}
}
}
grid.sync();
// Stage K3: CTAs 0..143 run QGPW=2 o_proj; idle CTAs 144..159 shift cq, ck, cv off the critical path!
if (blockIdx.x < 144) {
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int base = (pass * 256 + threadIdx.x) * 8;
uint4 ov = *reinterpret_cast<const uint4*>(&ws->attn_o_bf16[base]);
const __nv_bfloat16* ob = reinterpret_cast<const __nv_bfloat16*>(&ov);
#pragma unroll
for (int m = 0; m < 8; ++m) smem.gemv.sx[base + m] = __bfloat162float(ob[m]);
}
__syncthreads();
for (int qg = wid; qg < (KDA_C / QGROUP); qg += 8) {
float s = warp_sum(smem.gemv.sx[qg * 32 + lane]);
if (lane == 0) smem.gemv.sx_qgsum[qg] = s;
}
__syncthreads();
int tn = global_warp % 18;
int kc = global_warp / 18; // 0..63
warp_int4_gemv_128<2, false>(
smem.gemv.sx, smem.gemv.sx_qgsum,
kda.o_proj.w_q, kda.o_proj.scales, kda.o_proj.zeros,
&ws->attn_o_accum[layer_idx][kc & 3][tn * 128],
HIDDEN, tn * 128, kc * 2
);
} else if (blockIdx.x < 160) {
int c = (blockIdx.x - 144) * 256 + threadIdx.x;
__nv_bfloat16 q1 = st.cq[1 * KDA_C + c], q2 = st.cq[2 * KDA_C + c];
__nv_bfloat16 qn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][c] + ws->kda_qkvg[layer_idx][1][c]);
st.cq[0 * KDA_C + c] = q1; st.cq[1 * KDA_C + c] = q2; st.cq[2 * KDA_C + c] = qn;
__nv_bfloat16 k1 = st.ck[1 * KDA_C + c], k2 = st.ck[2 * KDA_C + c];
__nv_bfloat16 kn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][KDA_C + c] + ws->kda_qkvg[layer_idx][1][KDA_C + c]);
st.ck[0 * KDA_C + c] = k1; st.ck[1 * KDA_C + c] = k2; st.ck[2 * KDA_C + c] = kn;
__nv_bfloat16 v1 = st.cv[1 * KDA_C + c], v2 = st.cv[2 * KDA_C + c];
__nv_bfloat16 vn = __float2bfloat16_rn(ws->kda_qkvg[layer_idx][0][2 * KDA_C + c] + ws->kda_qkvg[layer_idx][1][2 * KDA_C + c]);
st.cv[0 * KDA_C + c] = v1; st.cv[1 * KDA_C + c] = v2; st.cv[2 * KDA_C + c] = vn;
}
grid.sync();
}
__device__ void run_mla_layer(
FastGridBarrier& grid,
const MegakernelParams& params,
Workspace* ws,
SharedStorage& smem,
int layer_idx
) {
const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x;
const int global_warp = global_tid >> 5;
const int lane = threadIdx.x & 31;
const int wid = threadIdx.x >> 5;
const MLAPtrs& mla = params.mla;
const int pos = params.pos;
const int L = pos + 1;
const float mla_scale = rsqrtf(float(QK_NOPE + QK_ROPE));
// Stage A1: Active-CTA RMSNorm + Row-Contiguous q_proj (48 tiles) + kv_a (5 tiles)
if (blockIdx.x < 239) {
cta_rmsnorm_from_h_2304(
ws->h, params.attn_norm[layer_idx],
smem.gemv.sx, smem.gemv.sx_qgsum, smem.gemv.s_red
);
if (global_warp < 53 * 36) {
int tn = global_warp % 53;
int kc = global_warp / 53; // 0..35
if (tn < 48) {
warp_int4_gemv_128<2, false>(
smem.gemv.sx, smem.gemv.sx_qgsum,
mla.q_proj.w_q, mla.q_proj.scales, mla.q_proj.zeros,
&ws->mla_qkv[kc & 3][tn * 128],
6144, tn * 128, kc * 2
);
} else {
int ltn = tn - 48;
int valid = (ltn < 4) ? 128 : 64;
warp_int4_gemv_128<2, false, true>(
smem.gemv.sx, smem.gemv.sx_qgsum,
mla.kv_a.w_q, mla.kv_a.scales, mla.kv_a.zeros,
&ws->mla_qkv[kc & 3][6144 + ltn * 128],
576, ltn * 128, kc * 2, valid
);
}
}
}
grid.sync();
// Stage A2: CTAs 0..255 compute Absorbed q_latent_bf16; idle CTAs 256..262 write c_kv_out & RoPE!
if (blockIdx.x < 256) {
int h = global_warp >> 6;
int quad = global_warp & 63;
int g = quad >> 4;
int d_base = lane * 4;
int col = h * 256 + d_base;
float4 q0 = *reinterpret_cast<const float4*>(&ws->mla_qkv[0][h * 192 + d_base]);
float4 q1 = *reinterpret_cast<const float4*>(&ws->mla_qkv[1][h * 192 + d_base]);
float4 q2 = *reinterpret_cast<const float4*>(&ws->mla_qkv[2][h * 192 + d_base]);
float4 q3 = *reinterpret_cast<const float4*>(&ws->mla_qkv[3][h * 192 + d_base]);
float qn[4] = {
__bfloat162float(__float2bfloat16_rn(q0.x + q1.x + q2.x + q3.x)),
__bfloat162float(__float2bfloat16_rn(q0.y + q1.y + q2.y + q3.y)),
__bfloat162float(__float2bfloat16_rn(q0.z + q1.z + q2.z + q3.z)),
__bfloat162float(__float2bfloat16_rn(q0.w + q1.w + q2.w + q3.w))
};
uint2 s_raw = *reinterpret_cast<const uint2*>(&mla.kv_b.scales[g * 8192 + col]);
uint2 z_raw = *reinterpret_cast<const uint2*>(&mla.kv_b.zeros[g * 8192 + col]);
const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_raw);
const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_raw);
float qs[4];
float qz_local = 0.0f;
#pragma unroll
for (int m = 0; m < 4; ++m) {
qs[m] = qn[m] * __bfloat162float(sb[m]);
qz_local += qs[m] * (__bfloat162float(zb[m]) + 128.0f);
}
float qz_sum = warp_sum(qz_local);
int kb_start = quad * 4;
uint4 packed;
#pragma unroll
for (int s = 0; s < 4; ++s) {
int kb = kb_start + s;
uint32_t v = *reinterpret_cast<const uint32_t*>(&mla.kv_b.w_q[kb * 8192 + col]);
uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
uint32_t v_shr8 = v >> 8;
uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;
float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));
float d_lo = warp_sum(qs[0] * f02_lo.x + qs[1] * f13_lo.x + qs[2] * f02_lo.y + qs[3] * f13_lo.y);
float d_hi = warp_sum(qs[0] * f02_hi.x + qs[1] * f13_hi.x + qs[2] * f02_hi.y + qs[3] * f13_hi.y);
__nv_bfloat162 b2 = __floats2bfloat162_rn((d_lo - qz_sum) * mla_scale, (d_hi - qz_sum) * mla_scale);
uint32_t u2;
memcpy(&u2, &b2, sizeof(uint32_t));
if (s == 0) packed.x = u2;
else if (s == 1) packed.y = u2;
else if (s == 2) packed.z = u2;
else packed.w = u2;
}
if (lane == 0) {
*reinterpret_cast<uint4*>(&ws->q_latent_bf16[h * KV_LORA + 2 * kb_start]) = packed;
}
} else if (blockIdx.x < 258) {
int c = (blockIdx.x - 256) * 256 + threadIdx.x;
float v = ws->mla_qkv[0][6144 + c] + ws->mla_qkv[1][6144 + c]
+ ws->mla_qkv[2][6144 + c] + ws->mla_qkv[3][6144 + c];
params.c_kv_out[size_t(pos) * KV_LORA + c] = __float2bfloat16_rn(v);
} else if (blockIdx.x < 263) {
int rtid = (blockIdx.x - 258) * 256 + threadIdx.x;
if (rtid < 33 * 32) {
int who = rtid >> 5;
int pair = rtid & 31;
float inv = powf(10000.0f, -float(2 * pair) * (1.0f / 64.0f));
float ang = float(pos) * inv;
float cs, sn;
__sincosf(ang, &sn, &cs);
int idx_e = (who == 32) ? (6144 + 512 + 2 * pair) : (who * 192 + 128 + 2 * pair);
float ev_s = ws->mla_qkv[0][idx_e] + ws->mla_qkv[1][idx_e] + ws->mla_qkv[2][idx_e] + ws->mla_qkv[3][idx_e];
float od_s = ws->mla_qkv[0][idx_e + 1] + ws->mla_qkv[1][idx_e + 1] + ws->mla_qkv[2][idx_e + 1] + ws->mla_qkv[3][idx_e + 1];
float ev = __bfloat162float(__float2bfloat16_rn(ev_s));
float od = __bfloat162float(__float2bfloat16_rn(od_s));
__nv_bfloat16 r0 = __float2bfloat16_rn(ev * cs - od * sn);
__nv_bfloat16 r1 = __float2bfloat16_rn(od * cs + ev * sn);
if (who == 32) {
params.k_rope_out[size_t(pos) * QK_ROPE + 2 * pair + 0] = r0;
params.k_rope_out[size_t(pos) * QK_ROPE + 2 * pair + 1] = r1;
} else {
ws->q_rope_bf16[who * QK_ROPE + 2 * pair + 0] = __float2bfloat16_rn(__bfloat162float(r0) * mla_scale);
ws->q_rope_bf16[who * QK_ROPE + 2 * pair + 1] = __float2bfloat16_rn(__bfloat162float(r1) * mla_scale);
}
}
}
grid.sync();
// Stage A3: 2-Stage cp.async (L2::evict_first) + Hoisted-Q Register SM120 Tensor Core Flash-Decoding!
const int total_tiles16 = max(1, (L + 15) >> 4);
const int num_splits = min(MAX_KV_SPLITS, total_tiles16);
if (blockIdx.x < num_splits) {
const int split_id = blockIdx.x;
const int gid = lane >> 2;
const int tid = lane & 3;
const int h_half = wid >> 2;
const int sub_w = wid & 3;
const int h_base = h_half * 16;
int t_start = (total_tiles16 * split_id) / num_splits;
int t_end = (total_tiles16 * (split_id + 1)) / num_splits;
int n_iters = t_end - t_start;
int l_start = t_start * 16;
int l_end = min(L, t_end * 16);
if (threadIdx.x < 32) {
smem.mla_tc.s_m[threadIdx.x] = -1.0e30f;
smem.mla_tc.s_l[threadIdx.x] = 0.0f;
}
float o_acc[16][4];
#pragma unroll
for (int n = 0; n < 16; ++n) {
o_acc[n][0] = 0.f; o_acc[n][1] = 0.f; o_acc[n][2] = 0.f; o_acc[n][3] = 0.f;
}
uint32_t q_reg[8][4];
#pragma unroll
for (int kc = 0; kc < 8; ++kc) {
int k_col = (sub_w * 8 + kc) * 16;
q_reg[kc][0] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid) * KV_LORA + k_col + 2 * tid]);
q_reg[kc][1] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid + 8) * KV_LORA + k_col + 2 * tid]);
q_reg[kc][2] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid) * KV_LORA + k_col + 8 + 2 * tid]);
q_reg[kc][3] = *reinterpret_cast<const uint32_t*>(&ws->q_latent_bf16[(h_base + gid + 8) * KV_LORA + k_col + 8 + 2 * tid]);
}
uint32_t qr_reg[4];
{
int r_col = sub_w * 16;
qr_reg[0] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid) * QK_ROPE + r_col + 2 * tid]);
qr_reg[1] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid + 8) * QK_ROPE + r_col + 2 * tid]);
qr_reg[2] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid) * QK_ROPE + r_col + 8 + 2 * tid]);
qr_reg[3] = *reinterpret_cast<const uint32_t*>(&ws->q_rope_bf16[(h_base + gid + 8) * QK_ROPE + r_col + 8 + 2 * tid]);
}
const uint64_t evict_policy = make_evict_first_policy();
auto issue_tile = [&](int stage, int l_base) {
int valid_t = min(16, l_end - l_base);
#pragma unroll
for (int p = 0; p < 4; ++p) {
int idx = p * 256 + threadIdx.x;
int t = idx >> 6;
int c8 = (idx & 63) * 8;
bool ok = (t < valid_t);
const void* gptr = ¶ms.c_kv_out[size_t(ok ? (l_base + t) : 0) * KV_LORA + c8];
cp_async_16_evict_first(&smem.mla_tc.ckv[stage][t][c8], gptr, ok, evict_policy);
}
if (threadIdx.x < 128) {
int t = threadIdx.x >> 3;
int r8 = (threadIdx.x & 7) * 8;
bool ok = (t < valid_t);
const void* gptr = ¶ms.k_rope_out[size_t(ok ? (l_base + t) : 0) * QK_ROPE + r8];
cp_async_16_evict_first(&smem.mla_tc.krp[stage][t][r8], gptr, ok, evict_policy);
}
cp_async_commit();
};
issue_tile(0, l_start);
for (int iter = 0; iter < n_iters; ++iter) {
int stage = iter & 1;
int l_base = l_start + iter * 16;
int valid_t = min(16, l_end - l_base);
if (iter + 1 < n_iters) {
issue_tile(stage ^ 1, l_base + 16);
cp_async_wait<1>();
} else {
cp_async_wait<0>();
}
__syncthreads();
float s0[4] = {0.f, 0.f, 0.f, 0.f};
float s1[4] = {0.f, 0.f, 0.f, 0.f};
#pragma unroll
for (int kc = 0; kc < 8; ++kc) {
int k_col = (sub_w * 8 + kc) * 16;
uint32_t b0_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][gid][k_col + 2 * tid]);
uint32_t b1_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][gid][k_col + 8 + 2 * tid]);
mma_m16n8k16_bf16(s0[0], s0[1], s0[2], s0[3], q_reg[kc][0], q_reg[kc][1], q_reg[kc][2], q_reg[kc][3], b0_0, b1_0);
uint32_t b0_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][8 + gid][k_col + 2 * tid]);
uint32_t b1_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.ckv[stage][8 + gid][k_col + 8 + 2 * tid]);
mma_m16n8k16_bf16(s1[0], s1[1], s1[2], s1[3], q_reg[kc][0], q_reg[kc][1], q_reg[kc][2], q_reg[kc][3], b0_1, b1_1);
}
{
int r_col = sub_w * 16;
uint32_t b0_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][gid][r_col + 2 * tid]);
uint32_t b1_0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][gid][r_col + 8 + 2 * tid]);
mma_m16n8k16_bf16(s0[0], s0[1], s0[2], s0[3], qr_reg[0], qr_reg[1], qr_reg[2], qr_reg[3], b0_0, b1_0);
uint32_t b0_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][8 + gid][r_col + 2 * tid]);
uint32_t b1_1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.krp[stage][8 + gid][r_col + 8 + 2 * tid]);
mma_m16n8k16_bf16(s1[0], s1[1], s1[2], s1[3], qr_reg[0], qr_reg[1], qr_reg[2], qr_reg[3], b0_1, b1_1);
}
*reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid][2 * tid]) = make_float2(s0[0], s0[1]);
*reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid + 8][2 * tid]) = make_float2(s0[2], s0[3]);
*reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid][8 + 2 * tid]) = make_float2(s1[0], s1[1]);
*reinterpret_cast<float2*>(&smem.mla_tc.s_part[sub_w][h_base + gid + 8][8 + 2 * tid]) = make_float2(s1[2], s1[3]);
__syncthreads();
#pragma unroll
for (int hp = 0; hp < 2; ++hp) {
int h = wid * 4 + hp * 2 + (lane >> 4);
int t = lane & 15;
float sc = (t < valid_t)
? (smem.mla_tc.s_part[0][h][t] + smem.mla_tc.s_part[1][h][t]
+ smem.mla_tc.s_part[2][h][t] + smem.mla_tc.s_part[3][h][t])
: -1.0e30f;
float tmax = sc;
#pragma unroll
for (int off = 8; off > 0; off >>= 1) {
tmax = fmaxf(tmax, __shfl_xor_sync(0xffffffff, tmax, off));
}
float m_old = smem.mla_tc.s_m[h];
float m_new = fmaxf(m_old, tmax);
float e_prev = (m_old == -1.0e30f) ? 0.0f : __expf(m_old - m_new);
float p = (t < valid_t) ? __expf(sc - m_new) : 0.0f;
float psum = p;
#pragma unroll
for (int off = 8; off > 0; off >>= 1) {
psum += __shfl_xor_sync(0xffffffff, psum, off);
}
smem.mla_tc.s_p[h][t] = __float2bfloat16_rn(p);
if (t == 0) {
smem.mla_tc.s_escale[h] = e_prev;
smem.mla_tc.s_m[h] = m_new;
smem.mla_tc.s_l[h] = smem.mla_tc.s_l[h] * e_prev + psum;
}
}
__syncthreads();
float e_top = smem.mla_tc.s_escale[h_base + gid];
float e_bot = smem.mla_tc.s_escale[h_base + gid + 8];
uint32_t pa0 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid][2 * tid]);
uint32_t pa1 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid + 8][2 * tid]);
uint32_t pa2 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid][8 + 2 * tid]);
uint32_t pa3 = *reinterpret_cast<const uint32_t*>(&smem.mla_tc.s_p[h_base + gid + 8][8 + 2 * tid]);
int c_sub = sub_w * 128;
const __nv_bfloat16* ckv_row = &smem.mla_tc.ckv[stage][lane & 15][c_sub];
#pragma unroll
for (int n = 0; n < 8; ++n) {
o_acc[n][0] *= e_top;
o_acc[n][1] *= e_top;
o_acc[n][2] *= e_bot;
o_acc[n][3] *= e_bot;
uint32_t vb0, vb1;
ldmatrix_x2_trans(vb0, vb1, ckv_row + n * 8);
mma_m16n8k16_bf16(o_acc[n][0], o_acc[n][1], o_acc[n][2], o_acc[n][3], pa0, pa1, pa2, pa3, vb0, vb1);
}
asm volatile("" ::: "memory");
#pragma unroll
for (int n = 8; n < 16; ++n) {
o_acc[n][0] *= e_top;
o_acc[n][1] *= e_top;
o_acc[n][2] *= e_bot;
o_acc[n][3] *= e_bot;
uint32_t vb0, vb1;
ldmatrix_x2_trans(vb0, vb1, ckv_row + n * 8);
mma_m16n8k16_bf16(o_acc[n][0], o_acc[n][1], o_acc[n][2], o_acc[n][3], pa0, pa1, pa2, pa3, vb0, vb1);
}
}
if (threadIdx.x < 32) {
int h = threadIdx.x;
ws->attn_m[h * KV_SPLIT_STRIDE + split_id] = smem.mla_tc.s_m[h];
ws->attn_l[h * KV_SPLIT_STRIDE + split_id] = smem.mla_tc.s_l[h];
}
float inv_l_top = 1.0f / fmaxf(smem.mla_tc.s_l[h_base + gid], 1.0e-20f);
float inv_l_bot = 1.0f / fmaxf(smem.mla_tc.s_l[h_base + gid + 8], 1.0e-20f);
int c_sub = sub_w * 128;
__syncthreads();
#pragma unroll
for (int n = 0; n < 16; ++n) {
int col = c_sub + n * 8 + 2 * tid;
__nv_bfloat162 v_top = __floats2bfloat162_rn(o_acc[n][0] * inv_l_top, o_acc[n][1] * inv_l_top);
__nv_bfloat162 v_bot = __floats2bfloat162_rn(o_acc[n][2] * inv_l_bot, o_acc[n][3] * inv_l_bot);
*reinterpret_cast<__nv_bfloat162*>(&smem.mla_tc.ckv[h_half][gid][col]) = v_top;
*reinterpret_cast<__nv_bfloat162*>(&smem.mla_tc.ckv[h_half][gid + 8][col]) = v_bot;
}
__syncthreads();
#pragma unroll
for (int p = 0; p < 8; ++p) {
int idx = p * 256 + threadIdx.x;
int h = idx >> 6;
int c8 = (idx & 63) * 8;
uint4 v = *reinterpret_cast<const uint4*>(&smem.mla_tc.ckv[h >> 4][h & 15][c8]);
store_u4_streaming(&ws->attn_o_splits[(split_id * 32 + h) * KV_LORA + c8], v);
}
}
grid.sync();
// Stage A4: 256-CTA Coalesced Streaming Split-KV Combine (32 heads * 8 tiles of 64 = 256 CTAs!)
if (blockIdx.x < 256) {
int h = blockIdx.x >> 3;
int c_base = (blockIdx.x & 7) * 64;
if (wid == 0) {
const float* m_row = &ws->attn_m[h * KV_SPLIT_STRIDE];
const float* l_row = &ws->attn_l[h * KV_SPLIT_STRIDE];
float g_max = -1.0e30f;
for (int s = lane; s < num_splits; s += 32) {
if (l_row[s] > 0.0f) {
g_max = fmaxf(g_max, m_row[s]);
}
}
#pragma unroll
for (int off = 16; off > 0; off >>= 1) {
g_max = fmaxf(g_max, __shfl_xor_sync(0xffffffff, g_max, off));
}
float g_sum = 0.0f;
for (int s = lane; s < num_splits; s += 32) {
float ls = l_row[s];
float w = (ls > 0.0f) ? (ls * __expf(m_row[s] - g_max)) : 0.0f;
smem.mla_comb.scale_s[s] = w;
g_sum += w;
}
g_sum = warp_sum(g_sum);
if (lane == 0) smem.mla_comb.g_inv_sum = 1.0f / fmaxf(g_sum, 1.0e-20f);
}
__syncthreads();
float2 acc = {0.f, 0.f};
int col2 = c_base + lane * 2;
for (int s = wid; s < num_splits; s += 8) {
float w = smem.mla_comb.scale_s[s];
if (w > 0.0f) {
uint32_t bv = load_u32_streaming(&ws->attn_o_splits[(s * 32 + h) * KV_LORA + col2]);
float2 f01 = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&bv));
acc.x += w * f01.x;
acc.y += w * f01.y;
}
}
*reinterpret_cast<float2*>(&smem.mla_comb.partial[wid][lane * 2]) = acc;
__syncthreads();
if (threadIdx.x < 64) {
int cl = threadIdx.x;
float tot = 0.0f;
#pragma unroll
for (int w = 0; w < 8; ++w) tot += smem.mla_comb.partial[w][cl];
ws->o_latent[h * KV_LORA + c_base + cl] = tot * smem.mla_comb.g_inv_sum;
}
}
grid.sync();
// Stage A5: 1-Wave TILE_N=128 Absorbed o = o_latent @ W_uv (32 heads * 32 chunks of 16 = 1,024 warps, 4-way split!)
if (global_warp < 32 * 32) {
int h = global_warp & 31; // h innermost -> contiguous 4096B row across 32 warps!
int sg = global_warp >> 5; // 0..31 (8 kb rows = 16 c's per sixteenth-group!)
int g = sg >> 3; // 0..3
const float* ol = &ws->o_latent[h * KV_LORA];
float xsum = 0.0f;
if (lane < 16) xsum = ol[sg * 16 + lane];
#pragma unroll
for (int off = 8; off > 0; off >>= 1) xsum += __shfl_xor_sync(0xffffffff, xsum, off);
xsum = __shfl_sync(0xffffffff, xsum, 0);
int col_grp = lane & 15;
int row_sub = lane >> 4;
int local_col = col_grp * 8;
int col_base = h * 256 + 128 + local_col;
float g_acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
int kb_base = sg * 8 + row_sub;
const uint64_t pol_last = make_evict_last_policy();
#pragma unroll
for (int step = 0; step < 4; ++step) {
int kb = kb_base + step * 2;
uint2 w_vec = load_u2_policy(&mla.kv_b.w_q[kb * 8192 + col_base], pol_last);
float x_lo = ol[kb * 2];
float x_hi = ol[kb * 2 + 1];
uint32_t words[2] = {w_vec.x, w_vec.y};
#pragma unroll
for (int w = 0; w < 2; ++w) {
uint32_t v = words[w];
uint32_t b02_lo_u = (v & 0x000F000Fu) | 0x43004300u;
uint32_t b02_hi_u = ((v >> 4) & 0x000F000Fu) | 0x43004300u;
uint32_t v_shr8 = v >> 8;
uint32_t b13_lo_u = (v_shr8 & 0x000F000Fu) | 0x43004300u;
uint32_t b13_hi_u = ((v_shr8 >> 4) & 0x000F000Fu) | 0x43004300u;
float2 f02_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_lo_u));
float2 f02_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b02_hi_u));
float2 f13_lo = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_lo_u));
float2 f13_hi = __bfloat1622float2(*reinterpret_cast<__nv_bfloat162*>(&b13_hi_u));
g_acc[w * 4 + 0] += x_lo * f02_lo.x + x_hi * f02_hi.x;
g_acc[w * 4 + 1] += x_lo * f13_lo.x + x_hi * f13_hi.x;
g_acc[w * 4 + 2] += x_lo * f02_lo.y + x_hi * f02_hi.y;
g_acc[w * 4 + 3] += x_lo * f13_lo.y + x_hi * f13_hi.y;
}
}
#pragma unroll
for (int c = 0; c < 8; ++c) {
g_acc[c] += __shfl_down_sync(0xffffffff, g_acc[c], 16);
}
if (row_sub == 0) {
uint4 s_vec = load_u4_policy(&mla.kv_b.scales[g * 8192 + col_base], pol_last);
uint4 z_vec = load_u4_policy(&mla.kv_b.zeros[g * 8192 + col_base], pol_last);
const __nv_bfloat16* sb = reinterpret_cast<const __nv_bfloat16*>(&s_vec);
const __nv_bfloat16* zb = reinterpret_cast<const __nv_bfloat16*>(&z_vec);
float* op = &ws->mla_o_raw[sg & 3][h * V_HEAD + local_col];
#pragma unroll
for (int c = 0; c < 8; ++c) {
float z_eff = __bfloat162float(zb[c]) + 128.0f;
float val = __bfloat162float(sb[c]) * (g_acc[c] - z_eff * xsum);
atomicAdd(&op[c], val);
}
}
}
grid.sync();
// Stage A6: CTAs 0..143 run QGPW=2 Row-Contiguous o_proj!
if (blockIdx.x < 144) {
#pragma unroll
for (int pass = 0; pass < 4; ++pass) {
int base = (pass * 256 + threadIdx.x) * 4;
float4 o0 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[0][base]);
float4 o1 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[1][base]);
float4 o2 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[2][base]);
float4 o3 = *reinterpret_cast<const float4*>(&ws->mla_o_raw[3][base]);
smem.gemv.sx[base + 0] = __bfloat162float(__float2bfloat16_rn(o0.x + o1.x + o2.x + o3.x));
smem.gemv.sx[base + 1] = __bfloat162float(__float2bfloat16_rn(o0.y + o1.y + o2.y + o3.y));
smem.gemv.sx[base + 2] = __bfloat162float(__float2bfloat16_rn(o0.z + o1.z + o2.z + o3.z));
smem.gemv.sx[base + 3] = __bfloat162float(__float2bfloat16_rn(o0.w + o1.w + o2.w + o3.w));
}
__syncthreads();
for (int qg = wid; qg < (4096 / QGROUP); qg += 8) {
float s = warp_sum(smem.gemv.sx[qg * 32 + lane]);
if (lane == 0) smem.gemv.sx_qgsum[qg] = s;
}
__syncthreads();
int tn = global_warp % 18;
int kc = global_warp / 18; // 0..63
warp_int4_gemv_128<2, false>(
smem.gemv.sx, smem.gemv.sx_qgsum,
mla.o_proj.w_q, mla.o_proj.scales, mla.o_proj.zeros,
&ws->attn_o_accum[layer_idx][kc & 3][tn * 128],
HIDDEN, tn * 128, kc * 2
);
}
grid.sync();
}
__global__ __launch_bounds__(BLOCK_THREADS, 2)
void kimi_decode_megakernel(const __grid_constant__ MegakernelParams params) {
Workspace* ws = reinterpret_cast<Workspace*>(params.workspace);
FastGridBarrier grid{ws->grid_bar};
extern __shared__ char dyn_smem[];
SharedStorage& smem = *reinterpret_cast<SharedStorage*>(dyn_smem);
const int global_tid = blockIdx.x * BLOCK_THREADS + threadIdx.x;
const int total_threads = gridDim.x * BLOCK_THREADS;
constexpr size_t ZERO_FLOATS = offsetof(Workspace, h) / sizeof(float);
float4* ws_zero_f4 = reinterpret_cast<float4*>(ws);
for (size_t i = global_tid; i < ZERO_FLOATS / 4; i += total_threads) {
ws_zero_f4[i] = make_float4(0.f, 0.f, 0.f, 0.f);
}
if (global_tid < HIDDEN / 8) {
int i = global_tid * 8;
*reinterpret_cast<uint4*>(&ws->h[i]) = *reinterpret_cast<const uint4*>(¶ms.hidden_in[i]);
}
// Streaming copy on step 0 prevents evicting the 105 MB dense weights from L2 cache!
if (params.c_kv_in != params.c_kv_out && params.pos > 0) {
size_t n_ckv = size_t(params.pos) * KV_LORA;
const uint4* src_c = reinterpret_cast<const uint4*>(params.c_kv_in);
uint4* dst_c = reinterpret_cast<uint4*>(params.c_kv_out);
for (size_t i = global_tid; i < n_ckv / 8; i += total_threads) {
store_u4_streaming(&dst_c[i], load_u4_streaming(&src_c[i]));
}
}
if (params.k_rope_in != params.k_rope_out && params.pos > 0) {
size_t n_kr = size_t(params.pos) * QK_ROPE;
const uint4* src_k = reinterpret_cast<const uint4*>(params.k_rope_in);
uint4* dst_k = reinterpret_cast<uint4*>(params.k_rope_out);
for (size_t i = global_tid; i < n_kr / 8; i += total_threads) {
store_u4_streaming(&dst_k[i], load_u4_streaming(&src_k[i]));
}
}
grid.sync();
run_kda_layer(grid, params, ws, smem, 0);
run_moe_block(grid, params, ws, smem, 0, false);
run_kda_layer(grid, params, ws, smem, 1);
run_moe_block(grid, params, ws, smem, 1, false);
run_kda_layer(grid, params, ws, smem, 2);
run_moe_block(grid, params, ws, smem, 2, false);
run_mla_layer(grid, params, ws, smem, 3);
run_moe_block(grid, params, ws, smem, 3, true);
}
extern "C" size_t kimi_workspace_bytes() {
return sizeof(Workspace) + 4096;
}
extern "C" size_t kimi_params_bytes() {
return sizeof(MegakernelParams);
}
extern "C" int kimi_build_base_params(
const uint64_t* weight_ptrs,
float routed_scaling,
void* out_params,
int device_id
) {
cudaSetDevice(device_id);
int smem_bytes = sizeof(SharedStorage);
cudaFuncSetAttribute(kimi_decode_megakernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes);
int blocks_per_sm = 0;
cudaOccupancyMaxActiveBlocksPerMultiprocessor(&blocks_per_sm, kimi_decode_megakernel, BLOCK_THREADS, smem_bytes);
cudaDeviceProp prop;
cudaGetDeviceProperties(&prop, device_id);
int max_blocks = blocks_per_sm * prop.multiProcessorCount;
int target = 2 * prop.multiProcessorCount; // 376 blocks
int num_blocks = (max_blocks < target) ? max_blocks : target;
MegakernelParams p = {};
int idx = 0;
for (int i = 0; i < 4; ++i) p.attn_norm[i] = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
for (int i = 0; i < 4; ++i) p.moe_norm[i] = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
auto read_ql = [&](QuantLinearPtrs& ql) {
ql.w_q = reinterpret_cast<const uint8_t*>(weight_ptrs[idx++]);
ql.scales = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
ql.zeros = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
};
for (int k = 0; k < 3; ++k) {
read_ql(p.kda[k].q_proj);
read_ql(p.kda[k].k_proj);
read_ql(p.kda[k].v_proj);
read_ql(p.kda[k].g_proj);
p.kda[k].beta_w = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
p.kda[k].conv_w = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
read_ql(p.kda[k].o_proj);
}
read_ql(p.mla.q_proj);
read_ql(p.mla.kv_a);
read_ql(p.mla.kv_b);
read_ql(p.mla.o_proj);
for (int i = 0; i < 4; ++i) {
p.moe[i].router_w = reinterpret_cast<const __nv_bfloat16*>(weight_ptrs[idx++]);
read_ql(p.moe[i].gate);
read_ql(p.moe[i].up);
read_ql(p.moe[i].down);
read_ql(p.moe[i].s_gate);
read_ql(p.moe[i].s_up);
read_ql(p.moe[i].s_down);
}
p.routed_scaling = routed_scaling;
memcpy(out_params, &p, sizeof(MegakernelParams));
return num_blocks;
}
extern "C" void launch_kimi_fast(
const void* base_params,
const uint64_t* state_ptrs,
int pos,
int num_blocks,
cudaStream_t stream
) {
MegakernelParams p;
memcpy(&p, base_params, sizeof(MegakernelParams));
int sidx = 0;
for (int k = 0; k < 3; ++k) {
p.kda_st[k].S = reinterpret_cast<float*>(state_ptrs[sidx++]);
p.kda_st[k].cq = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
p.kda_st[k].ck = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
p.kda_st[k].cv = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
}
p.c_kv_in = reinterpret_cast<const __nv_bfloat16*>(state_ptrs[sidx++]);
p.c_kv_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
p.k_rope_in = reinterpret_cast<const __nv_bfloat16*>(state_ptrs[sidx++]);
p.k_rope_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
p.hidden_in = reinterpret_cast<const __nv_bfloat16*>(state_ptrs[sidx++]);
p.hidden_out = reinterpret_cast<__nv_bfloat16*>(state_ptrs[sidx++]);
p.workspace = reinterpret_cast<void*>(state_ptrs[sidx++]);
p.pos = pos;
kimi_decode_megakernel<<<num_blocks, BLOCK_THREADS, sizeof(SharedStorage), stream>>>(p);
}
"""
_CPP_SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <stdint.h>
#include <tuple>
extern "C" size_t kimi_workspace_bytes();
extern "C" size_t kimi_params_bytes();
extern "C" int kimi_build_base_params(
const uint64_t* weight_ptrs,
float routed_scaling,
void* out_params,
int device_id
);
extern "C" void launch_kimi_fast(
const void* base_params,
const uint64_t* state_ptrs,
int pos,
int num_blocks,
cudaStream_t stream
);
int64_t get_workspace_bytes() {
return static_cast<int64_t>(kimi_workspace_bytes());
}
std::tuple<torch::Tensor, int64_t> build_base_params(
torch::Tensor weight_ptrs_cpu,
double routed_scaling,
int64_t device_id
) {
auto buf = torch::empty({static_cast<int64_t>(kimi_params_bytes())}, torch::dtype(torch::kUInt8));
int nb = kimi_build_base_params(
reinterpret_cast<const uint64_t*>(weight_ptrs_cpu.data_ptr<int64_t>()),
static_cast<float>(routed_scaling),
buf.data_ptr(),
static_cast<int>(device_id)
);
return std::make_tuple(buf, static_cast<int64_t>(nb));
}
static inline torch::Tensor grow_cache_2d(const torch::Tensor& tin, int64_t new_len, int64_t cols) {
size_t req_bytes = static_cast<size_t>(new_len * cols) * 2;
if (tin.is_contiguous() && tin.storage_offset() == 0 && tin.storage().nbytes() >= req_bytes) {
auto out = at::detail::make_tensor<c10::TensorImpl>(
c10::Storage(tin.storage()), tin.key_set(), tin.dtype()
);
int64_t sizes[2] = {new_len, cols};
int64_t strides[2] = {cols, 1};
out.unsafeGetTensorImpl()->set_sizes_and_strides(
c10::IntArrayRef(sizes, 2), c10::IntArrayRef(strides, 2)
);
return out;
}
auto buf = at::empty({new_len + 64, cols}, tin.options());
return buf.slice(0, 0, new_len);
}
std::tuple<torch::Tensor, torch::Tensor, torch::Tensor> step_fast(
torch::Tensor base_params_cpu,
torch::Tensor s0_S, torch::Tensor s0_cq, torch::Tensor s0_ck, torch::Tensor s0_cv,
torch::Tensor s1_S, torch::Tensor s1_cq, torch::Tensor s1_ck, torch::Tensor s1_cv,
torch::Tensor s2_S, torch::Tensor s2_cq, torch::Tensor s2_ck, torch::Tensor s2_cv,
torch::Tensor c_kv_in, torch::Tensor k_rope_in,
torch::Tensor hidden_in,
torch::Tensor workspace,
int64_t num_blocks
) {
int64_t pos = c_kv_in.size(0);
int64_t new_len = pos + 1;
torch::Tensor c_kv_out = grow_cache_2d(c_kv_in, new_len, 512);
torch::Tensor k_rope_out = grow_cache_2d(k_rope_in, new_len, 64);
torch::Tensor hidden_out = at::empty_like(hidden_in);
uint64_t state_ptrs[19] = {
reinterpret_cast<uint64_t>(s0_S.data_ptr()),
reinterpret_cast<uint64_t>(s0_cq.data_ptr()),
reinterpret_cast<uint64_t>(s0_ck.data_ptr()),
reinterpret_cast<uint64_t>(s0_cv.data_ptr()),
reinterpret_cast<uint64_t>(s1_S.data_ptr()),
reinterpret_cast<uint64_t>(s1_cq.data_ptr()),
reinterpret_cast<uint64_t>(s1_ck.data_ptr()),
reinterpret_cast<uint64_t>(s1_cv.data_ptr()),
reinterpret_cast<uint64_t>(s2_S.data_ptr()),
reinterpret_cast<uint64_t>(s2_cq.data_ptr()),
reinterpret_cast<uint64_t>(s2_ck.data_ptr()),
reinterpret_cast<uint64_t>(s2_cv.data_ptr()),
reinterpret_cast<uint64_t>(c_kv_in.data_ptr()),
reinterpret_cast<uint64_t>(c_kv_out.data_ptr()),
reinterpret_cast<uint64_t>(k_rope_in.data_ptr()),
reinterpret_cast<uint64_t>(k_rope_out.data_ptr()),
reinterpret_cast<uint64_t>(hidden_in.data_ptr()),
reinterpret_cast<uint64_t>(hidden_out.data_ptr()),
reinterpret_cast<uint64_t>(workspace.data_ptr()),
};
cudaStream_t stream = at::cuda::getCurrentCUDAStream(hidden_in.device().index()).stream();
launch_kimi_fast(
base_params_cpu.data_ptr(),
state_ptrs,
static_cast<int>(pos),
static_cast<int>(num_blocks),
stream
);
return std::make_tuple(hidden_out, c_kv_out, k_rope_out);
}
"""
_EXT = None
def _get_ext():
global _EXT
if _EXT is None:
_EXT = load_inline(
name="kimi_linear_megakernel_v9",
cpp_sources=_CPP_SRC,
cuda_sources=_CUDA_SRC,
functions=["get_workspace_bytes", "build_base_params", "step_fast"],
no_implicit_headers=True,
extra_cuda_cflags=["-O3", "--use_fast_math"],
)
return _EXT
class Model(nn.Module):
def __init__(self, cfg: Config):
super().__init__()
self.cfg = cfg
self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern)
self._ws: torch.Tensor | None = None
self._base_params: torch.Tensor | None = None
self._num_blocks: int = 376
_get_ext()
def _apply(self, fn):
self._base_params = None
self._ws = None
return super()._apply(fn)
def reset_parameters(self):
g = torch.Generator(device="cpu").manual_seed(1234)
for mod in self.modules():
if isinstance(mod, (QuantLinear, QuantExperts)):
mod.init_random(g)
elif isinstance(mod, nn.Linear):
nn.init.normal_(mod.weight, 0.0, 0.02, generator=g)
elif isinstance(mod, KDA):
nn.init.normal_(mod.conv_w, 0.0, 0.1, generator=g)
self._base_params = None
def _prepare(self, device: torch.device) -> None:
ext = _get_ext()
if self._ws is None or self._ws.device != device:
self._ws = torch.zeros(ext.get_workspace_bytes(), dtype=torch.uint8, device=device)
ptrs: list[int] = []
for blk in self.blocks:
ptrs.append(blk.attn_norm.data_ptr())
for blk in self.blocks:
ptrs.append(blk.moe_norm.data_ptr())
def add_ql(ql):
ptrs.append(ql.w_q.data_ptr())
ptrs.append(ql.scales.data_ptr())
ptrs.append(ql.zeros.data_ptr())
for k in range(3):
attn: KDA = self.blocks[k].attn
add_ql(attn.q_proj)
add_ql(attn.k_proj)
add_ql(attn.v_proj)
add_ql(attn.g_proj)
ptrs.append(attn.beta_proj.weight.data_ptr())
ptrs.append(attn.conv_w.data_ptr())
add_ql(attn.o_proj)
mla: MLA = self.blocks[3].attn
add_ql(mla.q_proj)
add_ql(mla.kv_a)
add_ql(mla.kv_b)
add_ql(mla.o_proj)
for i in range(4):
moe: MoE = self.blocks[i].moe
ptrs.append(moe.router.weight.data_ptr())
add_ql(moe.gate)
add_ql(moe.up)
add_ql(moe.down)
add_ql(moe.s_gate)
add_ql(moe.s_up)
add_ql(moe.s_down)
dev_idx = device.index if device.index is not None else 0
w_t = torch.tensor(ptrs, dtype=torch.int64, device="cpu").contiguous()
self._base_params, nb = ext.build_base_params(w_t, self.cfg.routed_scaling, dev_idx)
self._num_blocks = int(nb)
def step(self, hidden: torch.Tensor, state: list):
if self._base_params is None:
self._prepare(hidden.device)
s0, s1, s2, s3 = state[0], state[1], state[2], state[3]
hidden_out, c_kv_out, k_rope_out = _EXT.step_fast(
self._base_params,
s0["S"], s0["cq"], s0["ck"], s0["cv"],
s1["S"], s1["cq"], s1["ck"], s1["cv"],
s2["S"], s2["cq"], s2["ck"], s2["cv"],
s3["c_kv"], s3["k_rope"],
hidden,
self._ws,
self._num_blocks,
)
s3["c_kv"] = c_kv_out
s3["k_rope"] = k_rope_out
return hidden_out, state
def init_state(cfg: Config, context_len: int, seed: int) -> list:
dev = torch.device("cuda:0")
g = torch.Generator(device=dev).manual_seed(seed)
H, Dk = cfg.kda_heads, cfg.kda_head_dim
C = H * Dk
state = []
for kind in cfg.pattern:
if kind == "K":
state.append({
"S": torch.randn(H, Dk, Dk, device=dev, generator=g) * 0.05,
"cq": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
"ck": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
"cv": torch.randn(cfg.short_conv - 1, C, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
})
else:
state.append({
"c_kv": torch.randn(context_len, cfg.kv_lora, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
"k_rope": torch.randn(context_len, cfg.qk_rope, device=dev, generator=g, dtype=cfg.dtype) * 0.1,
})
return state
def init_token(cfg: Config, seed: int) -> torch.Tensor:
dev = torch.device("cuda:0")
g = torch.Generator(device=dev).manual_seed(seed + 1)
return torch.randn(cfg.hidden, device=dev, generator=g, dtype=cfg.dtype) * 0.25
20261110_113616_agy_gemini-4-carbon-high_02_kimi_linear_decode