KernelBench mega · RTX PRO 6000
Kimi-Linear Decode DeepSeek V4.1 Flash (1017)
Published row with no score. The kernel passes check.py but its MLA attention is wrong. In the M4 stage (solution.py ~1266-1319) each lane is one head and owns latent dims lane*16..lane*16+15, so it fills only the diagonal 16 of 512 latent dims per head; the other 31/32 stay at the zeros written in M1, and M5 reads all 512. The version it replaced (transcript edit #170) summed over all heads per latent dim and was correct. check.py cannot see it: on the graded random cache softmax is near uniform and the MLA attention output is ~0.14% of the hidden state. On caches where attention matters the solution diverges while baseline.py stays at 0.9999 (quotes). Separately, outputs are not reproducible: across 20 fresh-state runs per seed, seeds 1, 3 and 5 fall below the 0.98 gate (min 0.9577, one NaN), and the solution's persisting-L2 pool (cudaLimitPersistingL2CacheSize plus a stream access-policy window) slows the baseline timed after it by about 4%.
Kernel source (redacted)
"""Single-kernel W4A16 decode step for the Kimi-Linear hybrid motif.
Model exposes the same module tree / buffer names as reference.py so a
state_dict loads with strict=True; step() launches the fused megakernel once.
"""
from __future__ import annotations
import ctypes
import hashlib
import os
import subprocess
import tempfile
import torch
import torch.nn as nn
GROUP = 128
NSTEPS_HEAD = 4096
class QuantLinear(nn.Module):
def __init__(self, in_f, out_f, group=GROUP):
super().__init__()
self.in_f, self.out_f, self.group = in_f, out_f, group
ng = in_f // group
self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8))
self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16))
class QuantExperts(nn.Module):
def __init__(self, n, in_f, out_f, group=GROUP):
super().__init__()
self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group
ng = in_f // group
self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8))
self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16))
class KDA(nn.Module):
def __init__(self, cfg):
super().__init__()
self.cfg = cfg
H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden
self.q_proj = QuantLinear(d, H * Dk, cfg.group)
self.k_proj = QuantLinear(d, H * Dk, cfg.group)
self.v_proj = QuantLinear(d, H * Dk, cfg.group)
self.g_proj = QuantLinear(d, H * Dk, cfg.group)
self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype)
self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype))
self.o_proj = QuantLinear(H * Dk, d, cfg.group)
class MLA(nn.Module):
def __init__(self, cfg):
super().__init__()
self.cfg = cfg
H, d = cfg.mla_heads, cfg.hidden
self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group)
self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group)
self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group)
self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group)
class MoE(nn.Module):
def __init__(self, cfg):
super().__init__()
d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts
self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype)
self.gate = QuantExperts(E, d, m, cfg.group)
self.up = QuantExperts(E, d, m, cfg.group)
self.down = QuantExperts(E, m, d, cfg.group)
self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group)
self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group)
self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group)
class Block(nn.Module):
def __init__(self, cfg, kind):
super().__init__()
self.kind = kind
self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype))
self.attn = KDA(cfg) if kind == "K" else MLA(cfg)
self.moe = MoE(cfg)
# --------------------------------------------------------------------------- #
def _nvcc_build(src: str, tag: str):
h = hashlib.sha256((src + "|" + os.environ.get("NVFLAGS", "")).encode()).hexdigest()[:16]
d = os.path.join(tempfile.gettempdir(), "megak_" + tag + "_" + h)
so = os.path.join(d, "lib.so")
if not os.path.exists(so):
os.makedirs(d, exist_ok=True)
cu = os.path.join(d, "k.cu")
with open(cu, "w") as f:
f.write(src)
cmd = ["nvcc", "-shared", "-Xcompiler", "-fPIC", "-arch=sm_120a", "-O3",
"-std=c++17"] + os.environ.get("NVFLAGS","").split() + ["-o", so, cu]
r = subprocess.run(cmd, capture_output=True, text=True)
if r.returncode != 0:
raise RuntimeError("nvcc failed:\n" + r.stdout + r.stderr)
return ctypes.CDLL(so)
def _ptr(t):
return t.data_ptr()
PERSIST_CAP = int(os.environ.get("PERSIST_MB", "83")) * 1024 * 1024
LSTRIDE = 38
class Model(nn.Module):
def __init__(self, cfg):
super().__init__()
self.cfg = cfg
self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern)
self._lib = None
self._plan = None
self._plan_key = None
self._scratch = None
self._bar = None
self._stash = {}
# ---- plan: int64 device array of raw pointers, 38 slots per layer ----
def _build_plan(self):
cfg = self.cfg
L = []
for li, blk in enumerate(self.blocks):
ent = []
P = self._ptr
if blk.kind == "K":
a = blk.attn
for p in (a.q_proj, a.k_proj, a.v_proj, a.g_proj, a.o_proj):
ent += [P(p.w_q), P(p.scales), P(p.zeros)]
ent += [P(a.beta_proj.weight), P(a.conv_w)]
ent += [P(blk.attn_norm), P(blk.moe_norm)]
m = blk.moe
ent += [P(m.router.weight)]
g = self._gu[li]
ent += [P(g["w"]), P(g["s"]), P(g["z"]), P(g["sw"]), P(g["ss"]), P(g["sz"])]
for p in (m.down, m.s_gate, m.s_up, m.s_down):
ent += [P(p.w_q), P(p.scales), P(p.zeros)]
assert len(ent) == 38, len(ent)
else:
a = blk.attn
for p in (a.q_proj, a.kv_a, a.kv_b, a.o_proj):
ent += [P(p.w_q), P(p.scales), P(p.zeros)]
ent += [P(blk.attn_norm), P(blk.moe_norm)]
m = blk.moe
ent += [P(m.router.weight)]
g = self._gu[li]
ent += [P(g["w"]), P(g["s"]), P(g["z"]), P(g["sw"]), P(g["ss"]), P(g["sz"])]
for p in (m.down, m.s_gate, m.s_up, m.s_down):
ent += [P(p.w_q), P(p.scales), P(p.zeros)]
assert len(ent) == 33, len(ent)
ent += [0] * (LSTRIDE - len(ent))
L += ent
t = torch.tensor(L, dtype=torch.int64, device=self.blocks[0].attn_norm.device)
self._plan = t
self._plan_key = self._weights_ptr()
def _prepare(self, dev):
lib = _nvcc_build(SRC, "kimi")
lib.mega_launch.restype = ctypes.c_int
lib.mega_launch.argtypes = [ctypes.c_void_p] * 24
self._lib = lib
self._scratch = torch.empty(1300000, dtype=torch.float32, device=dev)
self._bar = torch.zeros(4, dtype=torch.int32, device=dev)
self._hout = torch.empty(self.cfg.hidden, dtype=torch.bfloat16, device=dev)
self._build_pool(dev)
self._build_gu(dev)
self._l2_stream = None
def _build_pool(self, dev):
"""Copy the always-read projection weights into one contiguous arena and
pin it in L2 (persisting window). Every token reads these bytes exactly
once, and they are static, so after the first token they are L2 hits."""
ent = []
for blk in self.blocks:
if blk.kind != "K":
continue
a = blk.attn
for q in (a.q_proj, a.k_proj, a.v_proj, a.g_proj, a.o_proj):
ent += [q.w_q, q.scales, q.zeros]
keep = []
used = 0
for t in ent:
n = t.numel() * t.element_size()
if used + n > PERSIST_CAP:
continue
keep.append(t)
used += n
ent = keep
tot = used
self._pool = None
self._pool_map = {}
self._pool_base = 0
self._pool_bytes = 0
if tot == 0:
return
pool = torch.empty(tot + 8192, dtype=torch.uint8, device=dev)
base = pool.data_ptr()
off = (-base) % 4096
for t in ent:
src = t.contiguous().view(torch.uint8).reshape(-1)
n = src.numel()
pool[off:off + n].copy_(src)
self._pool_map[t.data_ptr()] = base + off
off += n
self._pool = pool
self._pool_base = base + ((-base) % 4096)
self._pool_bytes = tot
def _build_gu(self, dev):
"""Row-interleave gate||up into one (E, in//2, 2*out) matrix per expert so
a decode step streams 2 KB rows instead of two 1 KB streams."""
gu = []
for blk in self.blocks:
m = blk.moe
gu.append(dict(
w=torch.cat([m.gate.w_q, m.up.w_q], dim=2).contiguous(),
s=torch.cat([m.gate.scales, m.up.scales], dim=2).contiguous(),
z=torch.cat([m.gate.zeros, m.up.zeros], dim=2).contiguous(),
sw=torch.cat([m.s_gate.w_q, m.s_up.w_q], dim=2).contiguous(),
ss=torch.cat([m.s_gate.scales, m.s_up.scales], dim=2).contiguous(),
sz=torch.cat([m.s_gate.zeros, m.s_up.zeros], dim=2).contiguous(),
))
self._gu = gu
def _ptr(self, t):
p = self._pool_map.get(t.data_ptr())
return t.data_ptr() if p is None else p
def _weights_ptr(self):
return _ptr(self.blocks[0].attn_norm)
def step(self, hidden, state):
dev = hidden.device
if self._lib is None or self._plan is None or self._plan_key is None:
self._prepare(dev)
self._build_plan()
elif self._weights_ptr() != self._plan_key:
self._build_plan()
ml = self.cfg.pattern.index("M")
ckv = state[ml]["c_kv"]
pos = ckv.shape[0]
bufs = state[ml].get("_mb")
if bufs is None or bufs[0].shape[0] <= pos:
ckv_buf = torch.empty(pos + NSTEPS_HEAD, self.cfg.kv_lora, dtype=torch.bfloat16, device=dev)
ckr_buf = torch.empty(pos + NSTEPS_HEAD, self.cfg.qk_rope, dtype=torch.bfloat16, device=dev)
bufs = [ckv_buf, ckr_buf]
state[ml]["_mb"] = bufs
ckv_buf, ckr_buf = bufs[0], bufs[1]
# the cache already lives in our own buffer once the state comes back from us
fresh = 0 if (ckv.data_ptr() == ckv_buf.data_ptr()
and state[ml]["k_rope"].data_ptr() == ckr_buf.data_ptr()) else 1
a0 = self.blocks[0].attn
kd = [i for i, k in enumerate(self.cfg.pattern) if k == "K"]
bd = [i for i, k in enumerate(self.cfg.pattern) if k == "M"]
S = [state[i]["S"] for i in kd]
cq = [state[i]["cq"] for i in kd]
ck = [state[i]["ck"] for i in kd]
cv = [state[i]["cv"] for i in kd]
hout = self._hout
st = torch.cuda.current_stream().cuda_stream
if self._pool_bytes and self._l2_stream != st:
self._lib.mega_setup.restype = ctypes.c_int
self._lib.mega_setup.argtypes = [ctypes.c_void_p, ctypes.c_longlong, ctypes.c_void_p]
self._lib.mega_setup(ctypes.c_void_p(self._pool_base), self._pool_bytes, ctypes.c_void_p(st))
self._l2_stream = st
r = self._lib.mega_launch(
self._plan.data_ptr(), pos, fresh,
_ptr(hidden), _ptr(hout),
_ptr(S[0]), _ptr(S[1]), _ptr(S[2]),
_ptr(cq[0]), _ptr(ck[0]), _ptr(cv[0]),
_ptr(cq[1]), _ptr(ck[1]), _ptr(cv[1]),
_ptr(cq[2]), _ptr(ck[2]), _ptr(cv[2]),
_ptr(ckv), _ptr(state[ml]["k_rope"]),
_ptr(ckv_buf), _ptr(ckr_buf),
self._scratch.data_ptr(), self._bar.data_ptr(), st)
if r != 0:
raise RuntimeError(f"launch failed {r}")
state[ml]["c_kv"] = ckv_buf[: pos + 1]
state[ml]["k_rope"] = ckr_buf[: pos + 1]
return hout, state
SRC = r'''// Fused single-kernel Kimi-Linear W4A16 decode step.
// One launch = the whole 4-layer motif (3x KDA + 1x MLA), each with a 64-expert MoE.
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <cstdint>
#include <cstring>
#ifndef TPB
#define TPB 384
#endif
#ifndef NLAY
#define NLAY 4
#endif
#ifndef NB
#define NB 188
#endif
#define HID 2304
#define HK 4096
#define MI 1024
#define HQ 6144
#define KVL 512
#define KVR 64
#define NEXP 64
#define NSLOT 9
#define LSTRIDE 38
// number of partial-sum slots used for the wide (k-split) output accumulators:
// each block adds into slot (blockIdx & (PSL-1)) and the consumer sums the slots,
// which divides the per-address atomic contention by PSL.
#define PSL 4
#ifndef B1
#define B1 2
#endif
#ifndef BR
#define BR 2
#endif
#ifndef BRS
#define BRS 1
#endif
#ifndef BT
#define BT 2
#define BT4 4
#endif
#ifndef CR
#define CR 8
#endif
#ifndef CR3
#define CR3 8
#endif
#ifndef CRM
#define CRM 8
#endif
#ifndef CRI
#define CRI 16
#endif
#ifndef NBI
#define NBI 188
#endif
#ifndef CT
#define CT 8
#endif
// plan slots, per layer, base = L*LSTRIDE
// KDA: 0-2 q(w,s,z) 3-5 k 6-8 v 9-11 g 12-14 o 15 beta_w 16 conv_w 17 attn_norm
// 18 moe_norm 19 router 20-22 gate 23-25 up 26-28 down
// 29-31 s_gate 32-34 s_up 35-37 s_down
// MLA: 0-2 q 3-5 kv_a 6-8 kv_b 9-11 o 12 attn_norm 13 moe_norm 14 router
// 15-17 gate 18-20 up 21-23 down 24-26 s_gate 27-29 s_up 30-32 s_down
#define KDA_MOE 19
#define MLA_MOE 14
typedef __nv_bfloat16 bf16;
typedef __nv_bfloat162 bf162;
#define SM_XS 0
#define SM_HB 9216
#define SM_WN 13824
#define SM_MISC 46592
#define SM_WNF 54784
#define SM_QR 88064
#define SM_RC 92672
#define SM_BYTES 101376
// ---------------------------------------------------------------- barrier
// sense-reversing grid barrier; `lg` is a per-block local generation counter
// (all blocks pass the same barriers in the same order, so it stays in sync).
__device__ __forceinline__ void gbar(volatile int* cntv, volatile int* genv, int* lg) {
int* cnt = (int*)cntv;
int* gen = (int*)genv;
__syncthreads();
if (threadIdx.x == 0) {
// the arrive is a device-scope release (publishes this block's writes)
// and the release/acquire pair on `gen` hands them to every waiter, so
// no separate __threadfence is needed.
int old;
asm volatile("atom.release.gpu.global.add.u32 %0, [%1], 1;" : "=r"(old) : "l"(cnt) : "memory");
if (old == NB - 1) {
*cnt = 0;
int g = *lg + 1;
asm volatile("st.release.gpu.global.u32 [%0], %1;" ::"l"(gen), "r"(g) : "memory");
} else {
int g;
do {
asm volatile("ld.acquire.gpu.global.u32 %0, [%1];" : "=r"(g) : "l"(gen) : "memory");
} while (g <= *lg);
}
*lg += 1;
}
__syncthreads();
}
// ---------------------------------------------------------------- scalar ops
__device__ __forceinline__ float silu_f(float x) { return x / (1.f + __expf(-x)); }
__device__ __forceinline__ float sigm_f(float x) { return 1.f / (1.f + __expf(-x)); }
__device__ __forceinline__ float bf2f(bf16 x) { return __bfloat162float(x); }
__device__ __forceinline__ bf16 f2bf(float x) { return __float2bfloat16(x); }
__device__ __forceinline__ float rbf(float x) { return __bfloat162float(__float2bfloat16(x)); }
__device__ __forceinline__ float block_reduce(float s, float* red) {
s += __shfl_down_sync(0xffffffffu, s, 16);
s += __shfl_down_sync(0xffffffffu, s, 8);
s += __shfl_down_sync(0xffffffffu, s, 4);
s += __shfl_down_sync(0xffffffffu, s, 2);
s += __shfl_down_sync(0xffffffffu, s, 1);
if ((threadIdx.x & 31) == 0) red[threadIdx.x >> 5] = s;
__syncthreads();
float t = 0.f;
for (int i = 0; i < TPB / 32; i++) t += red[i];
__syncthreads();
return t;
}
// ---------------------------------------------------------------- work split
struct Work { int tile, k0, k1; };
__device__ __forceinline__ Work assign_work_n(int ntiles, const int* cj, int K, int nblk) {
int tot = 0;
for (int j = 0; j < ntiles; j++) tot += cj[j];
int start[24];
int acc = 0;
for (int j = 0; j < ntiles - 1; j++) {
int bj = (int)(((long long)nblk * cj[j] + (tot >> 1)) / tot);
if (bj < 1) bj = 1;
start[j] = acc;
acc += bj;
}
start[ntiles - 1] = acc;
int b = blockIdx.x, j = 0;
while (j + 1 < ntiles && b >= start[j + 1]) j++;
int idx = b - start[j];
int cnt = ((j + 1 < ntiles) ? start[j + 1] : nblk) - start[j];
if (cnt < 1) cnt = 1;
Work w;
w.tile = j;
w.k0 = (int)(((long long)idx * K) / cnt) & ~1;
w.k1 = (idx >= cnt - 1) ? K : ((int)(((long long)(idx + 1) * K) / cnt) & ~1);
if (w.k1 > K) w.k1 = K;
if (w.k0 >= w.k1) { w.k0 = K; w.k1 = K; }
return w;
}
__device__ __forceinline__ Work assign_work(int ntiles, const int* cj, int K) {
return assign_work_n(ntiles, cj, K, NB);
}
__device__ __forceinline__ Work work_all(int K) {
Work w;
w.tile = 0;
w.k0 = (int)(((long long)blockIdx.x * K) / NB) & ~1;
w.k1 = (blockIdx.x >= NB - 1) ? K : ((int)(((long long)(blockIdx.x + 1) * K) / NB) & ~1);
if (w.k1 > K) w.k1 = K;
return w;
}
// ------------------------------------------------- fused int4 dequant-GEMV tile
// thread owns CPT consecutive output channels starting at nch (byte offset nch/2)
// out[nch+i] += mul * sum_k x[k] * (unpack(w)[k, nch+i] - z) * s
// decomposed per 128-group: s*(sum x q) - (s*z)*(sum x)
// every phase streams its weights exactly once, so the loop is purely memory
// bound: BATCH row-pairs are kept in flight and the *next* BATCH is issued
// before the current one is consumed (double buffering), which keeps the load
// pipe continuously busy instead of bursting once per batch.
__device__ __forceinline__ void pf_l2(const void* p) {
asm volatile("prefetch.global.L2 [%0];" :: "l"(p));
}
template <int CPT, int BATCH, int POL = 0>
__device__ __forceinline__ void ld_rows(uint32_t (&w)[BATCH][CPT / 4], const uint8_t* __restrict__ p,
int N, int t, int np) {
const int NW = CPT / 4;
#pragma unroll
for (int j = 0; j < BATCH; j++) {
int rr = (t + j < np) ? (t + j) : (np - 1);
const uint8_t* pp = p + (size_t)rr * N;
if (NW % 4 == 0) {
#pragma unroll
for (int u = 0; u < NW / 4; u++) {
const uint4 v = POL ? __ldg((const uint4*)pp + u) : __ldcs((const uint4*)pp + u);
w[j][4 * u] = v.x; w[j][4 * u + 1] = v.y; w[j][4 * u + 2] = v.z; w[j][4 * u + 3] = v.w;
}
} else if (NW == 2) {
const uint2 v = __ldg((const uint2*)pp);
w[j][0] = v.x; w[j][1] = v.y;
} else {
w[j][0] = __ldg((const uint32_t*)pp);
}
}
}
// fp16 magic-number unpack: (a & mask) | 0x64006400 turns two packed nibbles
// into two fp16 lanes holding (1024 + nib); subtracting 1024 is exact. One
// LOP3 + one HSUB2 per two weights, one HFMA2 per two weights -- no I2F.
#define HMAGIC 0x64006400u
__device__ __forceinline__ __half2 h2of(uint32_t u) {
return __halves2half2(__ushort_as_half((unsigned short)(u & 0xffffu)),
__ushort_as_half((unsigned short)(u >> 16)));
}
__device__ __forceinline__ uint32_t u2h(__half2 h) {
return (uint32_t)__half_as_ushort(__low2half(h)) |
((uint32_t)__half_as_ushort(__high2half(h)) << 16);
}
template <int CPT, int BATCH>
__device__ __forceinline__ void acc_rows(const uint32_t (&w)[BATCH][CPT / 4], const float* __restrict__ x,
int idx, int nb, uint32_t (&h)[CPT / 4][4], float& Sx) {
const int NW = CPT / 4;
const __half2 mg = h2of(HMAGIC);
#pragma unroll
for (int j = 0; j < BATCH; j++) {
float x0 = (j < nb) ? x[idx + 2 * j] : 0.f;
float x1 = (j < nb) ? x[idx + 2 * j + 1] : 0.f;
Sx += x0 + x1;
const __half2 xx0 = __half2half2(__float2half_rn(x0));
const __half2 xx1 = __half2half2(__float2half_rn(x1));
#pragma unroll
for (int q = 0; q < NW; q++) {
const uint32_t a0 = w[j][q];
// lanes: byte0|byte2 = columns 4q,4q+2 ; byte1|byte3 = 4q+1,4q+3
__half2 c0 = __hsub2(h2of((a0 & 0x000F000Fu) | HMAGIC), mg);
__half2 c1 = __hsub2(h2of(((a0 >> 4) & 0x000F000Fu) | HMAGIC), mg);
__half2 c2 = __hsub2(h2of(((a0 >> 8) & 0x000F000Fu) | HMAGIC), mg);
__half2 c3 = __hsub2(h2of((((a0 >> 12) & 0x000F000Fu) | HMAGIC)), mg);
h[q][0] = u2h(__hfma2(c0, xx0, h2of(h[q][0])));
h[q][1] = u2h(__hfma2(c1, xx1, h2of(h[q][1])));
h[q][2] = u2h(__hfma2(c2, xx0, h2of(h[q][2])));
h[q][3] = u2h(__hfma2(c3, xx1, h2of(h[q][3])));
}
}
}
// fold the fp16 partial sums into the fp32 column accumulators
template <int CPT>
__device__ __forceinline__ void hflush(uint32_t (&h)[CPT / 4][4], float (&A)[CPT]) {
const int NW = CPT / 4;
#pragma unroll
for (int q = 0; q < NW; q++) {
#pragma unroll
for (int b = 0; b < 4; b += 2) {
const float2 f = __half22float2(__hadd2(h2of(h[q][b]), h2of(h[q][b + 1])));
A[4 * q + (b >> 1)] += f.x;
A[4 * q + (b >> 1) + 2] += f.y;
h[q][b] = 0u;
h[q][b + 1] = 0u;
}
}
}
template <int CPT, int BATCH, int POL = 0>
__device__ __forceinline__ void gemv_tile(const uint8_t* __restrict__ wq,
const bf16* __restrict__ sc,
const bf16* __restrict__ zz,
const float* __restrict__ x,
float* __restrict__ out,
int N, int k0, int k1, int nch, float mul) {
if (k0 >= k1) return;
const int NW = CPT / 4;
float acc[CPT];
#pragma unroll
for (int i = 0; i < CPT; i++) acc[i] = 0.f;
const uint8_t* base = wq + (size_t)nch;
int g0 = k0 >> 7, g1 = (k1 + 127) >> 7;
for (int g = g0; g < g1; g++) {
int r0 = max(k0, g << 7), r1 = min(k1, (g + 1) << 7);
float A[CPT];
float Sx = 0.f;
#pragma unroll
for (int i = 0; i < CPT; i++) A[i] = 0.f;
const uint8_t* p = base + (size_t)(r0 >> 1) * N;
int np = (r1 - r0) >> 1;
// branch-free 2-deep pipeline: ld_rows clamps at np-1 so an overrun
// prefetch is harmless, and acc_rows zeroes x past nb, which keeps the
// loads unconditional so ptxas can hoist them above the maths.
uint32_t cw[BATCH][NW], pw[BATCH][NW];
uint32_t h[NW][4];
#pragma unroll
for (int q = 0; q < NW; q++) { h[q][0] = 0u; h[q][1] = 0u; h[q][2] = 0u; h[q][3] = 0u; }
int cnt = 0;
ld_rows<CPT, BATCH, POL>(cw, p, N, 0, np);
for (int t = 0; t < np; t += 2 * BATCH) {
const int t2 = t + BATCH;
int nb = np - t; if (nb > BATCH) nb = BATCH;
int nb2 = np - t2; if (nb2 < 0) nb2 = 0; if (nb2 > BATCH) nb2 = BATCH;
ld_rows<CPT, BATCH, POL>(pw, p, N, t2, np);
acc_rows<CPT, BATCH>(cw, x, r0 + 2 * t, nb, h, Sx);
ld_rows<CPT, BATCH, POL>(cw, p, N, t2 + BATCH, np);
acc_rows<CPT, BATCH>(pw, x, r0 + 2 * t2, nb2, h, Sx);
if (++cnt == 8) { hflush<CPT>(h, A); cnt = 0; }
}
hflush<CPT>(h, A);
const bf16* sp = sc + (size_t)g * N + nch;
const bf16* zp = zz + (size_t)g * N + nch;
#pragma unroll
for (int j = 0; j < CPT; j += 8) {
const uint4 sv = __ldcs((const uint4*)(sp + j));
const uint4 zv = __ldcs((const uint4*)(zp + j));
const bf16* sb = (const bf16*)&sv;
const bf16* zb = (const bf16*)&zv;
#pragma unroll
for (int i = 0; i < 8; i++) {
float s = bf2f(sb[i]), z = bf2f(zb[i]);
acc[j + i] += mul * (s * A[j + i] - (s * z) * Sx);
}
}
}
if ((nch & 3) == 0) {
#pragma unroll
for (int i = 0; i < CPT; i += 4) {
float4 v = make_float4(acc[i], acc[i + 1], acc[i + 2], acc[i + 3]);
atomicAdd((float4*)(out + nch + i), v);
}
} else {
#pragma unroll
for (int i = 0; i < CPT; i++) atomicAdd(out + nch + i, acc[i]);
}
}
// two packed matrices sharing one activation vector: both load streams are in
// flight together, which doubles the memory-level parallelism of the phase.
template <int CPT, int BATCH>
__device__ __forceinline__ void ld2_rows(uint32_t (&w0)[BATCH][CPT / 4], uint32_t (&w1)[BATCH][CPT / 4],
const uint8_t* __restrict__ p0, const uint8_t* __restrict__ p1,
int N, int t, int np) {
const int NW = CPT / 4;
#pragma unroll
for (int j = 0; j < BATCH; j++) {
int rr = (t + j < np) ? (t + j) : (np - 1);
const uint8_t* pp0 = p0 + (size_t)rr * N;
const uint8_t* pp1 = p1 + (size_t)rr * N;
if (NW % 4 == 0) {
#pragma unroll
for (int u = 0; u < NW / 4; u++) {
const uint4 a = __ldcs((const uint4*)pp0 + u), c = __ldcs((const uint4*)pp1 + u);
w0[j][4*u] = a.x; w0[j][4*u+1] = a.y; w0[j][4*u+2] = a.z; w0[j][4*u+3] = a.w;
w1[j][4*u] = c.x; w1[j][4*u+1] = c.y; w1[j][4*u+2] = c.z; w1[j][4*u+3] = c.w;
}
} else if (NW == 2) {
const uint2 a = __ldg((const uint2*)pp0), c = __ldg((const uint2*)pp1);
w0[j][0]=a.x; w0[j][1]=a.y; w1[j][0]=c.x; w1[j][1]=c.y;
} else {
w0[j][0] = __ldg((const uint32_t*)pp0);
w1[j][0] = __ldg((const uint32_t*)pp1);
}
}
}
template <int CPT, int BATCH>
__device__ __forceinline__ void acc2_rows(const uint32_t (&w0)[BATCH][CPT / 4], const uint32_t (&w1)[BATCH][CPT / 4],
const float* __restrict__ x, int idx, int nb,
float (&A0)[CPT], float (&A1)[CPT], float& Sx) {
const int NW = CPT / 4;
#pragma unroll
for (int j = 0; j < BATCH; j++) {
float x0 = (j < nb) ? x[idx + 2 * j] : 0.f;
float x1 = (j < nb) ? x[idx + 2 * j + 1] : 0.f;
Sx += x0 + x1;
#pragma unroll
for (int q = 0; q < NW; q++) {
uint32_t a0 = w0[j][q], a1 = w1[j][q];
#pragma unroll
for (int c4 = 0; c4 < 4; c4++) {
int c = 4 * q + c4;
uint32_t m = 0xFu << (8 * c4);
A0[c] += x0 * (float)((a0 & m) >> (8 * c4)) + x1 * (float)((a0 & (m << 4)) >> (8 * c4 + 4));
A1[c] += x0 * (float)((a1 & m) >> (8 * c4)) + x1 * (float)((a1 & (m << 4)) >> (8 * c4 + 4));
}
}
}
}
template <int CPT, int BATCH>
__device__ __forceinline__ void gemv2(const uint8_t* __restrict__ wq0, const bf16* __restrict__ sc0,
const bf16* __restrict__ zz0,
const uint8_t* __restrict__ wq1, const bf16* __restrict__ sc1,
const bf16* __restrict__ zz1,
const float* __restrict__ x,
float* __restrict__ out0, float* __restrict__ out1,
int N, int k0, int k1, int nch, float mul) {
if (k0 >= k1) return;
const int NW = CPT / 4;
float acc0[CPT], acc1[CPT];
#pragma unroll
for (int i = 0; i < CPT; i++) { acc0[i] = 0.f; acc1[i] = 0.f; }
const uint8_t* base0 = wq0 + (size_t)nch;
const uint8_t* base1 = wq1 + (size_t)nch;
int g0 = k0 >> 7, g1 = (k1 + 127) >> 7;
for (int g = g0; g < g1; g++) {
int r0 = max(k0, g << 7), r1 = min(k1, (g + 1) << 7);
float A0[CPT], A1[CPT];
float Sx = 0.f;
#pragma unroll
for (int i = 0; i < CPT; i++) { A0[i] = 0.f; A1[i] = 0.f; }
const uint8_t* p0 = base0 + (size_t)(r0 >> 1) * N;
const uint8_t* p1 = base1 + (size_t)(r0 >> 1) * N;
int np = (r1 - r0) >> 1;
uint32_t c0[BATCH][NW], c1[BATCH][NW], n0[BATCH][NW], n1[BATCH][NW];
ld2_rows<CPT, BATCH>(c0, c1, p0, p1, N, 0, np);
for (int t = 0; t < np; t += 2 * BATCH) {
const int t2 = t + BATCH;
int nb = np - t; if (nb > BATCH) nb = BATCH;
if (t2 < np) ld2_rows<CPT, BATCH>(n0, n1, p0, p1, N, t2, np);
acc2_rows<CPT, BATCH>(c0, c1, x, r0 + 2 * t, nb, A0, A1, Sx);
if (t2 >= np) break;
int nb2 = np - t2; if (nb2 > BATCH) nb2 = BATCH;
if (t2 + BATCH < np) ld2_rows<CPT, BATCH>(c0, c1, p0, p1, N, t2 + BATCH, np);
acc2_rows<CPT, BATCH>(n0, n1, x, r0 + 2 * t2, nb2, A0, A1, Sx);
}
const bf16* sp0 = sc0 + (size_t)g * N + nch;
const bf16* zp0 = zz0 + (size_t)g * N + nch;
const bf16* sp1 = sc1 + (size_t)g * N + nch;
const bf16* zp1 = zz1 + (size_t)g * N + nch;
#pragma unroll
for (int j = 0; j < CPT; j += 8) {
const uint4 sv0 = *(const uint4*)(sp0 + j), zv0 = *(const uint4*)(zp0 + j);
const uint4 sv1 = *(const uint4*)(sp1 + j), zv1 = *(const uint4*)(zp1 + j);
const bf16* sb0 = (const bf16*)&sv0; const bf16* zb0 = (const bf16*)&zv0;
const bf16* sb1 = (const bf16*)&sv1; const bf16* zb1 = (const bf16*)&zv1;
#pragma unroll
for (int i = 0; i < 8; i++) {
float s0 = bf2f(sb0[i]), z0 = bf2f(zb0[i]);
float s1 = bf2f(sb1[i]), z1 = bf2f(zb1[i]);
acc0[j + i] += mul * (s0 * A0[j + i] - (s0 * z0) * Sx);
acc1[j + i] += mul * (s1 * A1[j + i] - (s1 * z1) * Sx);
}
}
}
#pragma unroll
for (int i = 0; i < CPT; i += 4) {
atomicAdd((float4*)(out0 + nch + i), make_float4(acc0[i], acc0[i+1], acc0[i+2], acc0[i+3]));
atomicAdd((float4*)(out1 + nch + i), make_float4(acc1[i], acc1[i+1], acc1[i+2], acc1[i+3]));
}
}
template <int CPT, int BATCH, int POL = 0>
__device__ __forceinline__ void gemv_row(const uint8_t* __restrict__ wq,
const bf16* __restrict__ sc,
const bf16* __restrict__ zz,
const float* __restrict__ x,
float* __restrict__ out,
int N, int k0, int k1, float mul) {
int nch = threadIdx.x * CPT;
if (nch < N) gemv_tile<CPT, BATCH, POL>(wq, sc, zz, x, out, N, k0, k1, nch, mul);
nch = TPB * CPT + threadIdx.x * CPT;
if (nch < N) gemv_tile<CPT, BATCH, POL>(wq, sc, zz, x, out, N, k0, k1, nch, mul);
}
// one m16n8k16 bf16 mma tile with fp32 accumulation
__device__ __forceinline__ void mma16816(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
uint32_t b0, uint32_t b1, float (&c)[4]) {
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};"
: "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
: "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1));
}
// Like gemv_row, but the block's k range is split three ways across the three
// 128-thread groups so the block keeps three independent weight streams in
// flight (the pattern phase H uses); the tail columns past 128*CPT are picked
// up by the first few lanes of each group.
template <int CPT, int BATCH>
__device__ __forceinline__ void gemv_row3(const uint8_t* __restrict__ wq,
const bf16* __restrict__ sc,
const bf16* __restrict__ zz,
const float* __restrict__ x,
float* __restrict__ out,
int N, int k0, int k1, float mul) {
const int grp = threadIdx.x >> 7;
const int lane = threadIdx.x & 127;
int a0 = (int)(k0 + (((long long)(k1 - k0) * grp) / 3)) & ~1;
int a1 = (grp == 2) ? k1 : ((int)(k0 + (((long long)(k1 - k0) * (grp + 1)) / 3)) & ~1);
int nch = lane * CPT;
if (nch < N) gemv_tile<CPT, BATCH>(wq, sc, zz, x, out, N, a0, a1, nch, mul);
int nt = (N - 128 * CPT) / CPT;
if (lane < nt) gemv_tile<CPT, BATCH>(wq, sc, zz, x, out, N, a0, a1, 128 * CPT + nch, mul);
}
// ---------------------------------------------------------------- row dot
__device__ __forceinline__ float dot_row(const bf16* __restrict__ w, const float* __restrict__ x,
int K, float* red) {
float s = 0.f;
for (int i = threadIdx.x; i < K; i += TPB) s += bf2f(w[i]) * x[i];
return block_reduce(s, red);
}
// ---------------------------------------------------------------- zero fill
__device__ __forceinline__ void zfill(float* __restrict__ p, int n, int b, int tid) {
int per = (n + NB - 1) / NB;
int s = b * per, e = min(n, s + per);
for (int i = s + tid; i < e; i += TPB) p[i] = 0.f;
}
// pick partial slot `sl` of a wide output accumulator
__device__ __forceinline__ float* pslot(float* s0, float* sx, int sl) {
return sl ? sx + (sl - 1) * HID : s0;
}
// ---------------------------------------------------------------- norm
__device__ __forceinline__ void prep(const bf16* __restrict__ hin, const float* __restrict__ prev,
const float* __restrict__ prevx, const bf16* __restrict__ w,
int first, bf16* HB, float* XS, float* red) {
// register-resident: HID == NPT * TPB
const int NPT = HID / TPB;
// all three input streams are issued together so their latencies overlap
float v[NPT], wv[NPT], hv[NPT];
#pragma unroll
for (int j = 0; j < NPT; j++) {
int i = threadIdx.x + j * TPB;
wv[j] = bf2f(w[i]);
hv[j] = first ? bf2f(hin[i]) : bf2f(HB[i]);
}
if (!first) {
#pragma unroll
for (int j = 0; j < NPT; j++) {
int i = threadIdx.x + j * TPB;
// the producer split the projection over blocks; sum its slots
float ps = prev[i];
#pragma unroll
for (int q = 0; q < PSL - 1; q++) ps += prevx[q * HID + i];
hv[j] += rbf(ps);
}
}
#pragma unroll
for (int j = 0; j < NPT; j++) {
v[j] = hv[j];
HB[threadIdx.x + j * TPB] = f2bf(v[j]);
}
float s = 0.f;
#pragma unroll
for (int j = 0; j < NPT; j++) s += v[j] * v[j];
s = block_reduce(s, red);
float r = rsqrtf(s / (float)HID + 1e-6f);
#pragma unroll
for (int j = 0; j < NPT; j++) XS[threadIdx.x + j * TPB] = rbf(v[j] * r * wv[j]);
__syncthreads();
}
// ---------------------------------------------------------------- exp enc
__device__ __forceinline__ unsigned fenc(float f) {
unsigned u = __float_as_uint(f);
return (u & 0x80000000u) ? ~u : (u | 0x80000000u);
}
__device__ __forceinline__ float fdec(unsigned e) {
return __uint_as_float((e & 0x80000000u) ? (e & 0x7fffffffu) : ~e);
}
// ===========================================================================
extern "C" __global__ void __launch_bounds__(TPB) mega(const int64_t* __restrict__ plan, int pos, int fresh,
const bf16* __restrict__ hin, bf16* __restrict__ hout,
float* S0, float* S1, float* S2,
bf16* cq0, bf16* ck0, bf16* cv0,
bf16* cq1, bf16* ck1, bf16* cv1,
bf16* cq2, bf16* ck2, bf16* cv2,
const bf16* __restrict__ ckv_in, const bf16* __restrict__ ckr_in,
bf16* ckv_buf, bf16* ckr_buf,
float* __restrict__ scratch, int* bar) {
extern __shared__ char smem[];
volatile int* bcnt = (volatile int*)bar;
volatile int* bgen = (volatile int*)bar + 1;
int lgen = *bgen;
const int b = blockIdx.x, tid = threadIdx.x, warp = tid >> 5, lane = tid & 31;
float* XS = (float*)(smem + SM_XS);
bf16* HB = (bf16*)(smem + SM_HB);
bf16* WNB = (bf16*)(smem + SM_WN);
float* MISC = (float*)(smem + SM_MISC);
float* red = MISC;
int* sel = (int*)(smem + SM_BYTES - 128);
float* wts = (float*)(smem + SM_BYTES - 64);
float* AF = scratch; // slot 0 of the residual partials
float* YF = AF + 2 * HID; // slot 0 of the moe partials
float* QF = YF + 2 * HID;
float* KF = QF + HQ;
float* VF = KF + HK;
float* GF = VF + HK;
float* BETAF = GF + HK;
float* OB = BETAF + 32;
float* LOGIT = OB + HK;
float* GU = LOGIT + 64;
float* UU = GU + NSLOT * 2 * MI;
float* WNG = UU + NSLOT * MI;
float* CVG = WNG + 32 * 512;
float* SC = CVG + 32 * 512;
float* PMAXV = SC + 32768 * 32;
float* PSUM = PMAXV + 32;
float* AX = PSUM + 32; // residual partial slots 1..3
float* YX = AX + (PSL - 1) * 2 * HID; // moe partial slots 1..3
float* CVGX = YX + (PSL - 1) * 2 * HID; // mla cavg partial slots 1..3
float* PSUMX = CVGX + (PSL - 1) * 32 * 512; // mla prob-mass partial slots
float* Sd[3] = {S0, S1, S2};
bf16* cqd[3] = {cq0, cq1, cq2};
bf16* ckd[3] = {ck0, ck1, ck2};
bf16* cvd[3] = {cv0, cv1, cv2};
zfill(AF, 2 * HID, b, tid);
zfill(YF, 2 * HID, b, tid);
zfill(AX, (PSL - 1) * 2 * HID, b, tid);
zfill(YX, (PSL - 1) * 2 * HID, b, tid);
zfill(QF, HK, b, tid); zfill(KF, HK, b, tid); zfill(VF, HK, b, tid);
zfill(GF, HK, b, tid); zfill(OB, HK, b, tid);
if (b == 0) {
for (int i = tid; i < 32; i += TPB) { *(unsigned int*)(PMAXV + i) = fenc(-3.0e38f); PSUM[i] = 0.f; BETAF[i] = 0.f;
for (int q = 0; q < PSL - 1; q++) PSUMX[q * 32 + i] = 0.f; }
}
gbar(bcnt, bgen, &lgen);
for (int L = 0; L < NLAY; L++) {
const int64_t* pl = plan + (size_t)L * LSTRIDE;
const int mo = (L == NLAY - 1 && NLAY == 4) ? MLA_MOE : KDA_MOE;
const bf16* anorm = (const bf16*)pl[(L == 3) ? 12 : 17];
const bf16* mnorm = (const bf16*)pl[(L == 3) ? 13 : 18];
float* Acur = AF + (L & 1) * HID;
float* Ycur = YF + (L & 1) * HID;
float* Yprev = YF + ((L & 1) ^ 1) * HID;
float* AcurX = AX + (L & 1) * (PSL - 1) * HID;
float* YcurX = YX + (L & 1) * (PSL - 1) * HID;
float* YprevX = YX + (((L & 1) ^ 1)) * (PSL - 1) * HID;
// ---- attention input norm: HB = hidden_L ----
prep((L == 0) ? hin : nullptr, (L == 0) ? nullptr : Yprev,
(L == 0) ? nullptr : YprevX, anorm, (L == 0) ? 1 : 0, HB, XS, red);
if (L < 3) {
// ================= KDA =================
// ---- phase 1: q,k,v,g + beta + conv-window shift ----
if (b < 156) {
int mat = b / 39, sub = b % 39;
int k0 = (int)(((long long)sub * HID) / 39) & ~1;
int k1 = (sub == 38) ? HID : ((int)(((long long)(sub + 1) * HID) / 39) & ~1);
const uint8_t* W = (const uint8_t*)pl[mat * 3];
const bf16* Sc = (const bf16*)pl[mat * 3 + 1];
const bf16* Zz = (const bf16*)pl[mat * 3 + 2];
float* O = (mat == 0) ? QF : (mat == 1) ? KF : (mat == 2) ? VF : GF;
if (tid < 256) gemv_tile<16, B1, 1>(W, Sc, Zz, XS, O, HK, k0, k1, tid * 16, 1.0f);
} else {
int ch = b - 156;
float d = dot_row((const bf16*)pl[15] + (size_t)ch * HID, XS, HID, red);
if (tid == 0) BETAF[ch] = rbf(d);
}
gbar(bcnt, bgen, &lgen);
// ---- phase 2: short conv + KDA state update ----
if (b < 128) {
int h = b >> 2, jt = b & 3;
float* smq = MISC;
float* smk = MISC + 128;
float* smv = MISC + 256;
float* sme = MISC + 384;
float* smred = MISC + 512;
float* smkap = MISC + 1088;
if (tid < 128) {
int c = 128 * h + tid;
const bf16* cw = (const bf16*)pl[16];
bf16* st3[3] = {cqd[L], ckd[L], cvd[L]};
const float* src3[3] = {QF, KF, VF};
float so[3];
#pragma unroll
for (int a = 0; a < 3; a++) {
const bf16* cwp = cw + (size_t)a * HK * 4 + (size_t)c * 4;
const bf16* pw = st3[a];
float val = rbf(src3[a][c]);
float o = bf2f(pw[c]) * bf2f(cwp[0]) + bf2f(pw[HK + c]) * bf2f(cwp[1]) +
bf2f(pw[2 * HK + c]) * bf2f(cwp[2]) + val * bf2f(cwp[3]);
so[a] = rbf(silu_f(o));
}
smq[tid] = so[0] * 0.08838834764831845f;
smk[tid] = so[1];
smv[tid] = so[2];
float gv = rbf(GF[c]);
sme[tid] = 1.f / (1.f + __expf(gv));
}
__syncthreads();
if (tid < 32) {
float s = 0.f;
for (int i = tid; i < 128; i += 32) s += smk[i] * smq[i];
s += __shfl_down_sync(0xffffffffu, s, 16);
s += __shfl_down_sync(0xffffffffu, s, 8);
s += __shfl_down_sync(0xffffffffu, s, 4);
s += __shfl_down_sync(0xffffffffu, s, 2);
s += __shfl_down_sync(0xffffffffu, s, 1);
if (tid == 0) smkap[0] = s;
}
float beta = sigm_f(rbf(BETAF[h]));
int i0 = warp * 16;
int j = jt * 32 + lane;
float* Sp = Sd[L] + (size_t)h * 16384 + j;
float Sv[16];
float pp = 0.f, aa = 0.f;
if (warp < 8) {
#pragma unroll
for (int u = 0; u < 16; u++) {
Sv[u] = Sp[(size_t)(i0 + u) * 128];
float se = Sv[u] * sme[i0 + u];
pp += se * smk[i0 + u];
aa += se * smq[i0 + u];
}
smred[warp * 32 + lane] = pp;
smred[256 + warp * 32 + lane] = aa;
}
__syncthreads();
float pred = 0.f, Aacc = 0.f;
#pragma unroll
for (int w2 = 0; w2 < 8; w2++) {
pred += smred[w2 * 32 + lane];
Aacc += smred[256 + w2 * 32 + lane];
}
float kappa = smkap[0];
float d = smv[j] - pred;
float bk = beta * d;
#pragma unroll
if (warp < 8) {
#pragma unroll
for (int u = 0; u < 16; u++) {
Sp[(size_t)(i0 + u) * 128] = Sv[u] * sme[i0 + u] + bk * smk[i0 + u];
}
}
if (warp == 0 && tid < 32) OB[128 * h + j] = rbf(Aacc + beta * kappa * d);
}
gbar(bcnt, bgen, &lgen);
// ---- phase 3: o_proj (+ conv-window shift, after all conv reads) ----
if (b >= 156 && tid < 128) {
int hh = b - 156;
int c = 128 * hh + tid;
bf16* st3[3] = {cqd[L], ckd[L], cvd[L]};
const float* src3[3] = {QF, KF, VF};
#pragma unroll
for (int a = 0; a < 3; a++) {
bf16* pw = st3[a];
pw[c] = pw[HK + c];
pw[HK + c] = pw[2 * HK + c];
pw[2 * HK + c] = f2bf(rbf(src3[a][c]));
}
}
{
Work w = work_all(HK);
gemv_row<CR3, BRS, 1>((const uint8_t*)pl[12], (const bf16*)pl[13], (const bf16*)pl[14],
OB, pslot(Acur, AcurX, b & (PSL - 1)), HID, w.k0, w.k1, 1.0f);
}
gbar(bcnt, bgen, &lgen);
} else {
// ================= MLA =================
int nrow = pos + 1;
// ---- M1: q + kv_a + fresh copy + zero WNG/CVG ----
{
int cj[4] = {2048, 2048, 2048, 576};
Work w = assign_work(4, cj, HID);
if (w.tile < 3) {
if (tid < 256)
gemv_tile<CT, BT, 1>((const uint8_t*)pl[0], (const bf16*)pl[1], (const bf16*)pl[2],
XS, QF, HQ, w.k0, w.k1, w.tile * 2048 + tid * 8, 1.0f);
} else {
if (tid < 72)
gemv_tile<CT, BT, 1>((const uint8_t*)pl[3], (const bf16*)pl[4], (const bf16*)pl[5],
XS, KF, KVL + KVR, w.k0, w.k1, tid * 8, 1.0f);
}
zfill(WNG, 32 * 512, b, tid);
zfill(CVG, 32 * 512, b, tid);
zfill(CVGX, (PSL - 1) * 32 * 512, b, tid);
for (int i = tid; i < 32; i += TPB) { *(unsigned int*)(PMAXV + i) = fenc(-3.0e38f); PSUM[i] = 0.f; }
if (fresh && b < NB) {
int per = (pos + NB - 1) / NB;
int r0 = b * per, r1 = min(pos, r0 + per);
for (int r = r0; r < r1; r++) {
for (int i = tid; i < KVL; i += TPB) ckv_buf[(size_t)r * KVL + i] = ckv_in[(size_t)r * KVL + i];
for (int i = tid; i < KVR; i += TPB) ckr_buf[(size_t)r * KVR + i] = ckr_in[(size_t)r * KVR + i];
}
}
}
gbar(bcnt, bgen, &lgen);
// ---- M2: Wn absorb (b<128) + append/rope (b==128) ----
if (b < 128 && b * TPB + tid < 32768) {
int gid = b * TPB + tid;
int dpart = gid & 3, mp = (gid >> 2) & 255, h = gid >> 10;
const uint8_t* wrow = (const uint8_t*)pl[6] + (size_t)mp * 8192 + h * 256 + dpart * 32;
int grp = mp >> 6;
const bf16* srow = (const bf16*)pl[7] + (size_t)grp * 8192 + h * 256 + dpart * 32;
const bf16* zrow = (const bf16*)pl[8] + (size_t)grp * 8192 + h * 256 + dpart * 32;
float a0 = 0.f, a1 = 0.f;
const float* qp = QF + (size_t)h * 192 + dpart * 32;
for (int q = 0; q < 32; q += 16) {
const uint4 wv = *(const uint4*)(wrow + q);
const uint4 sv0 = *(const uint4*)(srow + q), sv1 = *(const uint4*)(srow + q + 8);
const uint4 zv0 = *(const uint4*)(zrow + q), zv1 = *(const uint4*)(zrow + q + 8);
const uint32_t* ww = (const uint32_t*)&wv;
bf16 sb[16], zb[16];
*(uint4*)&sb[0] = sv0; *(uint4*)&sb[8] = sv1;
*(uint4*)&zb[0] = zv0; *(uint4*)&zb[8] = zv1;
#pragma unroll
for (int j = 0; j < 4; j++) {
uint32_t w32 = ww[j];
int d0 = q + 4 * j;
float c0 = rbf(qp[d0]) * bf2f(sb[4 * j]);
float c1 = rbf(qp[d0 + 1]) * bf2f(sb[4 * j + 1]);
float c2 = rbf(qp[d0 + 2]) * bf2f(sb[4 * j + 2]);
float c3 = rbf(qp[d0 + 3]) * bf2f(sb[4 * j + 3]);
float zn = c0 * bf2f(zb[4 * j]) + c1 * bf2f(zb[4 * j + 1]) +
c2 * bf2f(zb[4 * j + 2]) + c3 * bf2f(zb[4 * j + 3]);
a0 += (float)(w32 & 0xF) * c0 + (float)((w32 >> 8) & 0xF) * c1 +
(float)((w32 >> 16) & 0xF) * c2 + (float)((w32 >> 24) & 0xF) * c3 - zn;
a1 += (float)((w32 >> 4) & 0xF) * c0 + (float)((w32 >> 12) & 0xF) * c1 +
(float)((w32 >> 20) & 0xF) * c2 + (float)((w32 >> 28) & 0xF) * c3 - zn;
}
}
int m0 = 2 * mp;
atomicAdd(WNG + (size_t)(m0 >> 2) * 128 + h * 4 + (m0 & 3), a0);
atomicAdd(WNG + (size_t)((m0 + 1) >> 2) * 128 + h * 4 + ((m0 + 1) & 3), a1);
} else if (b == 128) {
for (int gid = tid; gid < KVL + KVR; gid += TPB) {
if (gid < KVL) {
ckv_buf[(size_t)pos * KVL + gid] = f2bf(KF[gid]);
} else {
int d = gid - KVL, k = d >> 1;
float inv = 1.f / powf(10000.f, (float)k / 32.f);
float ang = (float)pos * inv;
float co = cosf(ang), si = sinf(ang);
float e = rbf(KF[KVL + d]), e2 = rbf(KF[KVL + (d ^ 1)]);
float o = (d & 1) ? (e * co + e2 * si) : (e * co - e2 * si);
ckr_buf[(size_t)pos * KVR + d] = f2bf(o);
}
}
}
gbar(bcnt, bgen, &lgen);
// ---- M3: scores ----
// SC[rows, 32 heads] = ckv[rows, 512] @ Wn[512, 32] plus the rope
// term ckr[rows, 64] @ qrot[64, 32]. Both are tiny-matmul shapes
// (n = 32), so 16-row tiles go through four m16n8k16 bf16 tensor-core
// accumulators instead of scalar dots: the phase becomes DRAM bound
// on the kv cache instead of instruction bound.
{
// The angle only depends on k = d >> 1, so there are 32 of them:
// compute the transcendentals once instead of 64*32 times.
float* rco = (float*)(smem + SM_RC);
float* rsi = rco + 32;
if (tid < 32) {
float ang = (float)pos / powf(10000.f, (float)tid / 32.f);
rco[tid] = cosf(ang);
rsi[tid] = sinf(ang);
}
__syncthreads();
// Wn as bf16 in [head][520] so a lane's two consecutive k weights
// are a single 4-byte load (the mma B fragment layout); the +8
// pad keeps the head stride off a bank multiple.
bf16* wnf = (bf16*)(smem + SM_WNF);
{
// WNG is blocked [k/8][k%8 first half?][head][4]; a float4 of it
// is one head's four consecutive k, so the read is coalesced and
// the store is a single 8-byte smem write.
const float4* wg4 = (const float4*)WNG;
for (int i = tid; i < 32 * 512 / 4; i += TPB) {
const float4 v = wg4[i];
const int h = i & 31, s2 = (i >> 5) & 1, q2 = i >> 6;
bf16 r4[4] = {f2bf(v.x), f2bf(v.y), f2bf(v.z), f2bf(v.w)};
*(uint2*)(wnf + h * 520 + 8 * q2 + 4 * s2) = *(const uint2*)r4;
}
}
// rotated q_rope as bf16 in [head][64+8]
bf16* qrb = (bf16*)(smem + SM_QR);
if (tid < 256) {
int h = tid >> 3, d0 = (tid & 7) * 8;
const float* qs = QF + h * 192 + 128 + d0;
const float4 va = *(const float4*)qs, vb = *(const float4*)(qs + 4);
float e[8] = {rbf(va.x), rbf(va.y), rbf(va.z), rbf(va.w),
rbf(vb.x), rbf(vb.y), rbf(vb.z), rbf(vb.w)};
#pragma unroll
for (int j = 0; j < 8; j += 2) {
const int d = d0 + j;
const float co = rco[d >> 1], si = rsi[d >> 1];
qrb[h * 72 + d] = f2bf(rbf(e[j] * co - e[j + 1] * si));
qrb[h * 72 + d + 1] = f2bf(rbf(e[j + 1] * co + e[j] * si));
}
}
__syncthreads();
const int per = ((((nrow + NB - 1) / NB) + 15) & ~15);
const int r0 = b * per, r1 = min(nrow, r0 + per);
const int row = lane >> 2, kq = (lane & 3) * 2;
const int hb = (lane >> 2) * 520, hq = (lane >> 2) * 72;
const int hcol = (lane & 3) * 2;
float lmax = -3.0e38f;
for (int t0 = r0 + warp * 16; t0 < r1; t0 += 128) {
float acc[4][4];
#pragma unroll
for (int n = 0; n < 4; n++)
#pragma unroll
for (int j = 0; j < 4; j++) acc[n][j] = 0.f;
const int tA = t0 + row, tB = tA + 8;
const bf16* pa = ckv_buf + (size_t)(tA < r1 ? tA : r1 - 1) * KVL;
const bf16* pb = ckv_buf + (size_t)(tB < r1 ? tB : r1 - 1) * KVL;
for (int kt = 0; kt < 32; kt++) {
const int kk = kt * 16 + kq;
const uint32_t a0 = *(const uint32_t*)(pa + kk);
const uint32_t a1 = *(const uint32_t*)(pb + kk);
const uint32_t a2 = *(const uint32_t*)(pa + kk + 8);
const uint32_t a3 = *(const uint32_t*)(pb + kk + 8);
const bf16* wq = wnf + hb + kk;
#pragma unroll
for (int n = 0; n < 4; n++) {
const uint32_t b0 = *(const uint32_t*)(wq + n * 8 * 520);
const uint32_t b1 = *(const uint32_t*)(wq + n * 8 * 520 + 8);
mma16816(a0, a1, a2, a3, b0, b1, acc[n]);
}
}
const bf16* ra = ckr_buf + (size_t)(tA < r1 ? tA : r1 - 1) * KVR;
const bf16* rb = ckr_buf + (size_t)(tB < r1 ? tB : r1 - 1) * KVR;
for (int kt = 0; kt < 4; kt++) {
const int kk = kt * 16 + kq;
const uint32_t a0 = *(const uint32_t*)(ra + kk);
const uint32_t a1 = *(const uint32_t*)(rb + kk);
const uint32_t a2 = *(const uint32_t*)(ra + kk + 8);
const uint32_t a3 = *(const uint32_t*)(rb + kk + 8);
const bf16* qq = qrb + hq + kk;
#pragma unroll
for (int n = 0; n < 4; n++) {
const uint32_t b0 = *(const uint32_t*)(qq + n * 8 * 72);
const uint32_t b1 = *(const uint32_t*)(qq + n * 8 * 72 + 8);
mma16816(a0, a1, a2, a3, b0, b1, acc[n]);
}
}
// c0,c1 = C[row][hcol + n*8 .. +1]; c2,c3 = C[row+8][same]
#pragma unroll
for (int n = 0; n < 4; n++) {
const int hd = n * 8 + hcol;
if (tA < r1) {
float s0 = acc[n][0] * 0.07216878364870322f;
float s1 = acc[n][1] * 0.07216878364870322f;
*(float2*)(SC + (size_t)tA * 32 + hd) = make_float2(s0, s1);
lmax = fmaxf(lmax, fmaxf(s0, s1));
}
if (tB < r1) {
float s0 = acc[n][2] * 0.07216878364870322f;
float s1 = acc[n][3] * 0.07216878364870322f;
*(float2*)(SC + (size_t)tB * 32 + hd) = make_float2(s0, s1);
lmax = fmaxf(lmax, fmaxf(s0, s1));
}
}
}
__syncthreads();
MISC[warp * 32 + lane] = lmax;
__syncthreads();
if (warp == 0) {
float m = MISC[lane];
for (int w2 = 1; w2 < TPB / 32; w2++) m = fmaxf(m, MISC[w2 * 32 + lane]);
atomicMax((unsigned int*)(PMAXV + lane), fenc(m));
}
}
gbar(bcnt, bgen, &lgen);
// ---- M4: softmax + cavg (lane = head, 16 m per lane, warp = row) ----
{
float pmax = fdec(*(const unsigned int*)(PMAXV + lane));
int r0 = (int)(((long long)b * nrow) / NB);
int r1 = (int)(((long long)(b + 1) * nrow) / NB);
int mbase = lane * 16;
const bf16* cbase = ckv_buf + mbase;
float pacc = 0.f;
float acc[16];
#pragma unroll
for (int i = 0; i < 16; i++) acc[i] = 0.f;
for (int rb = r0 + warp; warp < 8 && rb < r1; rb += 32) {
uint4 c0[4], c1[4];
float pv[4];
#pragma unroll
for (int j = 0; j < 4; j++) {
int rr = rb + 8 * j;
int rc = (rr < r1) ? rr : (r1 - 1);
pv[j] = (rr < r1) ? __expf(SC[(size_t)rc * 32 + lane] - pmax) : 0.f;
c0[j] = *(const uint4*)(cbase + (size_t)rc * KVL);
c1[j] = *(const uint4*)(cbase + (size_t)rc * KVL + 8);
}
#pragma unroll
for (int j = 0; j < 4; j++) {
float2 f0 = __bfloat1622float2(*(const bf162*)&c0[j].x);
float2 f1 = __bfloat1622float2(*(const bf162*)&c0[j].y);
float2 f2 = __bfloat1622float2(*(const bf162*)&c0[j].z);
float2 f3 = __bfloat1622float2(*(const bf162*)&c0[j].w);
float2 f4 = __bfloat1622float2(*(const bf162*)&c1[j].x);
float2 f5 = __bfloat1622float2(*(const bf162*)&c1[j].y);
float2 f6 = __bfloat1622float2(*(const bf162*)&c1[j].z);
float2 f7 = __bfloat1622float2(*(const bf162*)&c1[j].w);
float p = pv[j];
acc[0] += p * f0.x; acc[1] += p * f0.y;
acc[2] += p * f1.x; acc[3] += p * f1.y;
acc[4] += p * f2.x; acc[5] += p * f2.y;
acc[6] += p * f3.x; acc[7] += p * f3.y;
acc[8] += p * f4.x; acc[9] += p * f4.y;
acc[10] += p * f5.x; acc[11] += p * f5.y;
acc[12] += p * f6.x; acc[13] += p * f6.y;
acc[14] += p * f7.x; acc[15] += p * f7.y;
}
pacc += (pv[0] + pv[1]) + (pv[2] + pv[3]);
}
const int sl = b & (PSL - 1);
float* cp = sl ? CVGX + (sl - 1) * 32 * 512 + lane * 512 + mbase
: CVG + lane * 512 + mbase;
#pragma unroll
for (int i = 0; i < 16; i += 4) {
float4 v = make_float4(acc[i], acc[i + 1], acc[i + 2], acc[i + 3]);
atomicAdd((float4*)(cp + i), v);
}
atomicAdd(sl ? PSUMX + (sl - 1) * 32 + lane : PSUM + lane, pacc);
}
gbar(bcnt, bgen, &lgen);
// ---- M5: o = cavg @ kv_b[v] ----
{
int gid = b * TPB + tid;
if (gid < 32768) {
int dg = gid & 15, ms = (gid >> 4) & 63, h = gid >> 10;
float ps = PSUM[h];
#pragma unroll
for (int q = 0; q < PSL - 1; q++) ps += PSUMX[q * 32 + h];
float inv = 1.f / (ps + 1e-30f);
const uint8_t* wrow = (const uint8_t*)pl[6] + (size_t)(ms * 4) * 8192 +
h * 256 + 128 + dg * 8;
float S1[8], S0 = 0.f;
#pragma unroll
for (int i = 0; i < 8; i++) S1[i] = 0.f;
#pragma unroll
for (int t = 0; t < 4; t++) {
const uint2 wv = *(const uint2*)(wrow + (size_t)t * 8192);
const uint32_t* ww = (const uint32_t*)&wv;
int m0 = (ms * 4 + t) * 2;
const float2* q0 = (const float2*)(CVG + h * 512 + m0);
float2 a0 = q0[0], a1, a2, a3;
{ const float2* p1 = (const float2*)(CVGX + h * 512 + m0);
a1 = p1[0]; a2 = p1[8192]; a3 = p1[16384]; }
float c0 = (a0.x + a1.x + a2.x + a3.x) * inv;
float c1 = (a0.y + a1.y + a2.y + a3.y) * inv;
S0 += c0 + c1;
#pragma unroll
for (int j = 0; j < 2; j++) {
S1[4 * j] += c0 * (float)((ww[j] >> 0) & 0xF) + c1 * (float)((ww[j] >> 4) & 0xF);
S1[4 * j + 1] += c0 * (float)((ww[j] >> 8) & 0xF) + c1 * (float)((ww[j] >> 12) & 0xF);
S1[4 * j + 2] += c0 * (float)((ww[j] >> 16) & 0xF) + c1 * (float)((ww[j] >> 20) & 0xF);
S1[4 * j + 3] += c0 * (float)((ww[j] >> 24) & 0xF) + c1 * (float)((ww[j] >> 28) & 0xF);
}
}
int grp = (ms * 4) >> 6;
const bf16* srow = (const bf16*)pl[7] + (size_t)grp * 8192 + h * 256 + 128 + dg * 8;
const bf16* zrow = (const bf16*)pl[8] + (size_t)grp * 8192 + h * 256 + 128 + dg * 8;
const uint4 sv = *(const uint4*)srow;
const uint4 zv = *(const uint4*)zrow;
const bf16* sb = (const bf16*)&sv;
const bf16* zb = (const bf16*)&zv;
#pragma unroll
for (int i = 0; i < 8; i += 4) {
float4 v = make_float4(0.f, 0.f, 0.f, 0.f);
const bf16* sb4 = sb + i;
const bf16* zb4 = zb + i;
float s0 = bf2f(sb4[0]), z0 = bf2f(zb4[0]);
float s1 = bf2f(sb4[1]), z1 = bf2f(zb4[1]);
float s2 = bf2f(sb4[2]), z2 = bf2f(zb4[2]);
float s3 = bf2f(sb4[3]), z3 = bf2f(zb4[3]);
// each output element has exactly one writer, so store
// (rounded to bf16, matching reference o.to(bf16))
v.x = rbf(s0 * S1[i] - (s0 * z0) * S0);
v.y = rbf(s1 * S1[i + 1] - (s1 * z1) * S0);
v.z = rbf(s2 * S1[i + 2] - (s2 * z2) * S0);
v.w = rbf(s3 * S1[i + 3] - (s3 * z3) * S0);
*(float4*)(OB + h * 128 + dg * 8 + i) = v;
}
}
}
gbar(bcnt, bgen, &lgen);
// ---- M6: o_proj ----
{
Work w = work_all(HK);
gemv_row<CRM, BRS, 1>((const uint8_t*)pl[9], (const bf16*)pl[10], (const bf16*)pl[11],
OB, pslot(Acur, AcurX, b & (PSL - 1)), HID, w.k0, w.k1, 1.0f);
}
gbar(bcnt, bgen, &lgen);
}
// ---- phase G: residual + moe norm + router ----
// the router row is loaded up front: its latency hides under prep
bf16 rw[HID / TPB];
if (b < 64) {
#pragma unroll
for (int j = 0; j < HID / TPB; j++)
rw[j] = ((const bf16*)pl[mo])[(size_t)b * HID + tid + j * TPB];
}
prep(nullptr, Acur, AcurX, mnorm, 0, HB, XS, red);
zfill(GU, NSLOT * 2 * MI, b, tid);
// Clear the accumulators the coming layers will add into. Everything
// cleared here has already been consumed (this layer's prep read the
// previous moe partials, and the q/k/v/g/o buffers are dead once this
// layer's attention is done), and phase G already ends in a grid
// barrier, so this replaces a whole barrier per layer for free.
if (L + 1 < NLAY) {
zfill(AF + ((L + 1) & 1) * HID, HID, b, tid);
zfill(AX + ((L + 1) & 1) * (PSL - 1) * HID, (PSL - 1) * HID, b, tid);
}
if (L >= 1) {
zfill(YF + ((L - 1) & 1) * HID, HID, b, tid);
zfill(YX + ((L - 1) & 1) * (PSL - 1) * HID, (PSL - 1) * HID, b, tid);
}
zfill(QF, HK, b, tid); zfill(KF, HK, b, tid); zfill(VF, HK, b, tid);
zfill(GF, HK, b, tid); zfill(OB, HK, b, tid);
if (b < 64) {
float d = 0.f;
#pragma unroll
for (int j = 0; j < HID / TPB; j++) d += bf2f(rw[j]) * XS[tid + j * TPB];
d = block_reduce(d, red);
if (tid == 0) LOGIT[b] = rbf(d);
}
gbar(bcnt, bgen, &lgen);
// ---- phase H: topk + gate/up ----
// top-8 over 64 experts by rank counting: each lane counts how many
// logits beat it (ties broken by lower index), so the expert with rank r
// is the r-th largest. One broadcast sweep instead of eight butterfly
// reductions -- the whole selection costs a couple hundred cycles.
float* sl = MISC + 512;
{
if (tid < 64) sl[tid] = LOGIT[tid];
__syncthreads();
if (tid < 64) {
float v = sl[tid];
int r = 0;
#pragma unroll 4
for (int j = 0; j < 64; j++) {
float u = sl[j];
r += (u > v || (u == v && j > tid)) ? 1 : 0;
}
if (r < 8) { sel[r] = tid; sl[64 + r] = v; }
}
}
__syncthreads();
if (tid < 32) {
float p = (tid < 8) ? __expf(sl[64 + tid] - sl[64]) : 0.f;
float s8 = p + __shfl_down_sync(0xffffffffu, p, 4);
s8 += __shfl_down_sync(0xffffffffu, s8, 2);
s8 += __shfl_down_sync(0xffffffffu, s8, 1);
s8 = __shfl_sync(0xffffffffu, s8, 0);
if (tid < 8) wts[tid] = p / (s8 + 1e-9f) * 2.446f;
if (tid == 8) { sel[8] = 0; wts[8] = 1.f; }
}
__syncthreads();
if (b < 186) {
int tile = b / 62, sub = b % 62;
int k0 = (int)(((long long)sub * HID) / 62) & ~1;
int k1 = (sub == 61) ? HID : ((int)(((long long)(sub + 1) * HID) / 62) & ~1);
// gate and up are repacked row-interleaved (2048 cols per row), so a
// block streams one 2 KB row per slot with 16-byte loads.
int slot = tile * 3 + (tid >> 7);
int nch = (tid & 127) * 16;
if (slot < NSLOT) {
int e = sel[slot];
const uint8_t* w = (slot == 8) ? (const uint8_t*)pl[mo + 4]
: (const uint8_t*)pl[mo + 1] + (size_t)e * 2359296;
const bf16* sm = (slot == 8) ? (const bf16*)pl[mo + 5]
: (const bf16*)pl[mo + 2] + (size_t)e * 36864;
const bf16* zm = (slot == 8) ? (const bf16*)pl[mo + 6]
: (const bf16*)pl[mo + 3] + (size_t)e * 36864;
gemv_tile<16, BT>(w, sm, zm, XS, GU + slot * (2 * MI), 2 * MI, k0, k1, nch, 1.0f);
}
}
gbar(bcnt, bgen, &lgen);
// ---- phase I: down ----
{
int cj[9];
for (int j = 0; j < 9; j++) cj[j] = 2304;
Work w = assign_work_n(9, cj, MI, NBI);
int slot = w.tile;
const float* gsl = GU + slot * (2 * MI);
for (int i = tid; i < MI; i += TPB) MISC[i] = silu_f(gsl[i]) * gsl[MI + i];
__syncthreads();
int e = sel[slot];
const uint8_t* wd = (slot == 8) ? (const uint8_t*)pl[mo + 16] : (const uint8_t*)pl[mo + 7] + (size_t)e * 1179648;
const bf16* sd = (slot == 8) ? (const bf16*)pl[mo + 17] : (const bf16*)pl[mo + 8] + (size_t)e * 18432;
const bf16* zd = (slot == 8) ? (const bf16*)pl[mo + 18] : (const bf16*)pl[mo + 9] + (size_t)e * 18432;
gemv_row<CRI, BT4>(wd, sd, zd, MISC, pslot(Ycur, YcurX, b & (PSL - 1)), HID,
w.k0, w.k1, wts[slot]);
}
gbar(bcnt, bgen, &lgen);
}
// final hidden = HB + YF[HID..]
if (b == 0) {
const float* Yl = YF + ((NLAY - 1) & 1) * HID;
const float* Ylx = YX + ((NLAY - 1) & 1) * (PSL - 1) * HID;
for (int i = tid; i < HID; i += TPB) {
float y = Yl[i];
#pragma unroll
for (int q = 0; q < PSL - 1; q++) y += Ylx[q * HID + i];
hout[i] = f2bf(bf2f(HB[i]) + rbf(y));
}
}
}
// ---------------------------------------------------------------- launcher
static int g_smem_set = 0;
extern "C" int mega_launch(const int64_t* plan, int pos, int fresh,
const void* hin, void* hout,
void* S0, void* S1, void* S2,
void* cq0, void* ck0, void* cv0,
void* cq1, void* ck1, void* cv1,
void* cq2, void* ck2, void* cv2,
const void* ckv_in, const void* ckr_in,
void* ckv_buf, void* ckr_buf,
void* scratch, void* bar, void* stream) {
if (!g_smem_set) {
cudaError_t e = cudaFuncSetAttribute((const void*)mega,
cudaFuncAttributeMaxDynamicSharedMemorySize, SM_BYTES);
if (e != cudaSuccess) return (int)e + 1000;
g_smem_set = 1;
}
mega<<<NB, TPB, SM_BYTES, (cudaStream_t)stream>>>(
plan, pos, fresh, (const bf16*)hin, (bf16*)hout,
(float*)S0, (float*)S1, (float*)S2,
(bf16*)cq0, (bf16*)ck0, (bf16*)cv0,
(bf16*)cq1, (bf16*)ck1, (bf16*)cv1,
(bf16*)cq2, (bf16*)ck2, (bf16*)cv2,
(const bf16*)ckv_in, (const bf16*)ckr_in,
(bf16*)ckv_buf, (bf16*)ckr_buf,
(float*)scratch, (int*)bar);
return (int)cudaGetLastError();
}
extern "C" int last_err() { return (int)cudaGetLastError(); }
// pin the always-read weight arena in L2 for the duration of the stream
extern "C" int mega_setup(void* base, long long bytes, void* stream) {
cudaError_t e = cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, (size_t)bytes);
if (e != cudaSuccess) return 1;
cudaStreamAttrValue av;
memset(&av, 0, sizeof(av));
av.accessPolicyWindow.base_ptr = base;
av.accessPolicyWindow.num_bytes = (size_t)bytes;
av.accessPolicyWindow.hitRatio = 1.0f;
av.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting;
av.accessPolicyWindow.missProp = cudaAccessPropertyStreaming;
e = cudaStreamSetAttribute((cudaStream_t)stream, cudaStreamAttributeAccessPolicyWindow, &av);
return e == cudaSuccess ? 0 : 2;
}
'''
20261017_215137_or-fable_deepseek_deepseek-v4.1-flash-1017_02_kimi_linear_decode