"""Single-kernel W4A16 decode step for the Kimi-Linear hybrid motif. Model exposes the same module tree / buffer names as reference.py so a state_dict loads with strict=True; step() launches the fused megakernel once. """ from __future__ import annotations import ctypes import hashlib import os import subprocess import tempfile import torch import torch.nn as nn GROUP = 128 NSTEPS_HEAD = 4096 class QuantLinear(nn.Module): def __init__(self, in_f, out_f, group=GROUP): super().__init__() self.in_f, self.out_f, self.group = in_f, out_f, group ng = in_f // group self.register_buffer("w_q", torch.zeros(in_f // 2, out_f, dtype=torch.uint8)) self.register_buffer("scales", torch.zeros(ng, out_f, dtype=torch.bfloat16)) self.register_buffer("zeros", torch.zeros(ng, out_f, dtype=torch.bfloat16)) class QuantExperts(nn.Module): def __init__(self, n, in_f, out_f, group=GROUP): super().__init__() self.n, self.in_f, self.out_f, self.group = n, in_f, out_f, group ng = in_f // group self.register_buffer("w_q", torch.zeros(n, in_f // 2, out_f, dtype=torch.uint8)) self.register_buffer("scales", torch.zeros(n, ng, out_f, dtype=torch.bfloat16)) self.register_buffer("zeros", torch.zeros(n, ng, out_f, dtype=torch.bfloat16)) class KDA(nn.Module): def __init__(self, cfg): super().__init__() self.cfg = cfg H, Dk, d = cfg.kda_heads, cfg.kda_head_dim, cfg.hidden self.q_proj = QuantLinear(d, H * Dk, cfg.group) self.k_proj = QuantLinear(d, H * Dk, cfg.group) self.v_proj = QuantLinear(d, H * Dk, cfg.group) self.g_proj = QuantLinear(d, H * Dk, cfg.group) self.beta_proj = nn.Linear(d, H, bias=False, dtype=cfg.dtype) self.conv_w = nn.Parameter(torch.empty(3, H * Dk, cfg.short_conv, dtype=cfg.dtype)) self.o_proj = QuantLinear(H * Dk, d, cfg.group) class MLA(nn.Module): def __init__(self, cfg): super().__init__() self.cfg = cfg H, d = cfg.mla_heads, cfg.hidden self.q_proj = QuantLinear(d, H * (cfg.qk_nope + cfg.qk_rope), cfg.group) self.kv_a = QuantLinear(d, cfg.kv_lora + cfg.qk_rope, cfg.group) self.kv_b = QuantLinear(cfg.kv_lora, H * (cfg.qk_nope + cfg.v_head), cfg.group) self.o_proj = QuantLinear(H * cfg.v_head, d, cfg.group) class MoE(nn.Module): def __init__(self, cfg): super().__init__() d, m, E = cfg.hidden, cfg.moe_inter, cfg.n_experts self.router = nn.Linear(d, E, bias=False, dtype=cfg.dtype) self.gate = QuantExperts(E, d, m, cfg.group) self.up = QuantExperts(E, d, m, cfg.group) self.down = QuantExperts(E, m, d, cfg.group) self.s_gate = QuantExperts(cfg.n_shared, d, m, cfg.group) self.s_up = QuantExperts(cfg.n_shared, d, m, cfg.group) self.s_down = QuantExperts(cfg.n_shared, m, d, cfg.group) class Block(nn.Module): def __init__(self, cfg, kind): super().__init__() self.kind = kind self.attn_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype)) self.moe_norm = nn.Parameter(torch.ones(cfg.hidden, dtype=cfg.dtype)) self.attn = KDA(cfg) if kind == "K" else MLA(cfg) self.moe = MoE(cfg) # --------------------------------------------------------------------------- # def _nvcc_build(src: str, tag: str): h = hashlib.sha256((src + "|" + os.environ.get("NVFLAGS", "")).encode()).hexdigest()[:16] d = os.path.join(tempfile.gettempdir(), "megak_" + tag + "_" + h) so = os.path.join(d, "lib.so") if not os.path.exists(so): os.makedirs(d, exist_ok=True) cu = os.path.join(d, "k.cu") with open(cu, "w") as f: f.write(src) cmd = ["nvcc", "-shared", "-Xcompiler", "-fPIC", "-arch=sm_120a", "-O3", "-std=c++17"] + os.environ.get("NVFLAGS","").split() + ["-o", so, cu] r = subprocess.run(cmd, capture_output=True, text=True) if r.returncode != 0: raise RuntimeError("nvcc failed:\n" + r.stdout + r.stderr) return ctypes.CDLL(so) def _ptr(t): return t.data_ptr() PERSIST_CAP = int(os.environ.get("PERSIST_MB", "83")) * 1024 * 1024 LSTRIDE = 38 class Model(nn.Module): def __init__(self, cfg): super().__init__() self.cfg = cfg self.blocks = nn.ModuleList(Block(cfg, k) for k in cfg.pattern) self._lib = None self._plan = None self._plan_key = None self._scratch = None self._bar = None self._stash = {} # ---- plan: int64 device array of raw pointers, 38 slots per layer ---- def _build_plan(self): cfg = self.cfg L = [] for li, blk in enumerate(self.blocks): ent = [] P = self._ptr if blk.kind == "K": a = blk.attn for p in (a.q_proj, a.k_proj, a.v_proj, a.g_proj, a.o_proj): ent += [P(p.w_q), P(p.scales), P(p.zeros)] ent += [P(a.beta_proj.weight), P(a.conv_w)] ent += [P(blk.attn_norm), P(blk.moe_norm)] m = blk.moe ent += [P(m.router.weight)] g = self._gu[li] ent += [P(g["w"]), P(g["s"]), P(g["z"]), P(g["sw"]), P(g["ss"]), P(g["sz"])] for p in (m.down, m.s_gate, m.s_up, m.s_down): ent += [P(p.w_q), P(p.scales), P(p.zeros)] assert len(ent) == 38, len(ent) else: a = blk.attn for p in (a.q_proj, a.kv_a, a.kv_b, a.o_proj): ent += [P(p.w_q), P(p.scales), P(p.zeros)] ent += [P(blk.attn_norm), P(blk.moe_norm)] m = blk.moe ent += [P(m.router.weight)] g = self._gu[li] ent += [P(g["w"]), P(g["s"]), P(g["z"]), P(g["sw"]), P(g["ss"]), P(g["sz"])] for p in (m.down, m.s_gate, m.s_up, m.s_down): ent += [P(p.w_q), P(p.scales), P(p.zeros)] assert len(ent) == 33, len(ent) ent += [0] * (LSTRIDE - len(ent)) L += ent t = torch.tensor(L, dtype=torch.int64, device=self.blocks[0].attn_norm.device) self._plan = t self._plan_key = self._weights_ptr() def _prepare(self, dev): lib = _nvcc_build(SRC, "kimi") lib.mega_launch.restype = ctypes.c_int lib.mega_launch.argtypes = [ctypes.c_void_p] * 24 self._lib = lib self._scratch = torch.empty(1300000, dtype=torch.float32, device=dev) self._bar = torch.zeros(4, dtype=torch.int32, device=dev) self._hout = torch.empty(self.cfg.hidden, dtype=torch.bfloat16, device=dev) self._build_pool(dev) self._build_gu(dev) self._l2_stream = None def _build_pool(self, dev): """Copy the always-read projection weights into one contiguous arena and pin it in L2 (persisting window). Every token reads these bytes exactly once, and they are static, so after the first token they are L2 hits.""" ent = [] for blk in self.blocks: if blk.kind != "K": continue a = blk.attn for q in (a.q_proj, a.k_proj, a.v_proj, a.g_proj, a.o_proj): ent += [q.w_q, q.scales, q.zeros] keep = [] used = 0 for t in ent: n = t.numel() * t.element_size() if used + n > PERSIST_CAP: continue keep.append(t) used += n ent = keep tot = used self._pool = None self._pool_map = {} self._pool_base = 0 self._pool_bytes = 0 if tot == 0: return pool = torch.empty(tot + 8192, dtype=torch.uint8, device=dev) base = pool.data_ptr() off = (-base) % 4096 for t in ent: src = t.contiguous().view(torch.uint8).reshape(-1) n = src.numel() pool[off:off + n].copy_(src) self._pool_map[t.data_ptr()] = base + off off += n self._pool = pool self._pool_base = base + ((-base) % 4096) self._pool_bytes = tot def _build_gu(self, dev): """Row-interleave gate||up into one (E, in//2, 2*out) matrix per expert so a decode step streams 2 KB rows instead of two 1 KB streams.""" gu = [] for blk in self.blocks: m = blk.moe gu.append(dict( w=torch.cat([m.gate.w_q, m.up.w_q], dim=2).contiguous(), s=torch.cat([m.gate.scales, m.up.scales], dim=2).contiguous(), z=torch.cat([m.gate.zeros, m.up.zeros], dim=2).contiguous(), sw=torch.cat([m.s_gate.w_q, m.s_up.w_q], dim=2).contiguous(), ss=torch.cat([m.s_gate.scales, m.s_up.scales], dim=2).contiguous(), sz=torch.cat([m.s_gate.zeros, m.s_up.zeros], dim=2).contiguous(), )) self._gu = gu def _ptr(self, t): p = self._pool_map.get(t.data_ptr()) return t.data_ptr() if p is None else p def _weights_ptr(self): return _ptr(self.blocks[0].attn_norm) def step(self, hidden, state): dev = hidden.device if self._lib is None or self._plan is None or self._plan_key is None: self._prepare(dev) self._build_plan() elif self._weights_ptr() != self._plan_key: self._build_plan() ml = self.cfg.pattern.index("M") ckv = state[ml]["c_kv"] pos = ckv.shape[0] bufs = state[ml].get("_mb") if bufs is None or bufs[0].shape[0] <= pos: ckv_buf = torch.empty(pos + NSTEPS_HEAD, self.cfg.kv_lora, dtype=torch.bfloat16, device=dev) ckr_buf = torch.empty(pos + NSTEPS_HEAD, self.cfg.qk_rope, dtype=torch.bfloat16, device=dev) bufs = [ckv_buf, ckr_buf] state[ml]["_mb"] = bufs ckv_buf, ckr_buf = bufs[0], bufs[1] # the cache already lives in our own buffer once the state comes back from us fresh = 0 if (ckv.data_ptr() == ckv_buf.data_ptr() and state[ml]["k_rope"].data_ptr() == ckr_buf.data_ptr()) else 1 a0 = self.blocks[0].attn kd = [i for i, k in enumerate(self.cfg.pattern) if k == "K"] bd = [i for i, k in enumerate(self.cfg.pattern) if k == "M"] S = [state[i]["S"] for i in kd] cq = [state[i]["cq"] for i in kd] ck = [state[i]["ck"] for i in kd] cv = [state[i]["cv"] for i in kd] hout = self._hout st = torch.cuda.current_stream().cuda_stream if self._pool_bytes and self._l2_stream != st: self._lib.mega_setup.restype = ctypes.c_int self._lib.mega_setup.argtypes = [ctypes.c_void_p, ctypes.c_longlong, ctypes.c_void_p] self._lib.mega_setup(ctypes.c_void_p(self._pool_base), self._pool_bytes, ctypes.c_void_p(st)) self._l2_stream = st r = self._lib.mega_launch( self._plan.data_ptr(), pos, fresh, _ptr(hidden), _ptr(hout), _ptr(S[0]), _ptr(S[1]), _ptr(S[2]), _ptr(cq[0]), _ptr(ck[0]), _ptr(cv[0]), _ptr(cq[1]), _ptr(ck[1]), _ptr(cv[1]), _ptr(cq[2]), _ptr(ck[2]), _ptr(cv[2]), _ptr(ckv), _ptr(state[ml]["k_rope"]), _ptr(ckv_buf), _ptr(ckr_buf), self._scratch.data_ptr(), self._bar.data_ptr(), st) if r != 0: raise RuntimeError(f"launch failed {r}") state[ml]["c_kv"] = ckv_buf[: pos + 1] state[ml]["k_rope"] = ckr_buf[: pos + 1] return hout, state SRC = r'''// Fused single-kernel Kimi-Linear W4A16 decode step. // One launch = the whole 4-layer motif (3x KDA + 1x MLA), each with a 64-expert MoE. #include #include #include #include #include #ifndef TPB #define TPB 384 #endif #ifndef NLAY #define NLAY 4 #endif #ifndef NB #define NB 188 #endif #define HID 2304 #define HK 4096 #define MI 1024 #define HQ 6144 #define KVL 512 #define KVR 64 #define NEXP 64 #define NSLOT 9 #define LSTRIDE 38 // number of partial-sum slots used for the wide (k-split) output accumulators: // each block adds into slot (blockIdx & (PSL-1)) and the consumer sums the slots, // which divides the per-address atomic contention by PSL. #define PSL 4 #ifndef B1 #define B1 2 #endif #ifndef BR #define BR 2 #endif #ifndef BRS #define BRS 1 #endif #ifndef BT #define BT 2 #define BT4 4 #endif #ifndef CR #define CR 8 #endif #ifndef CR3 #define CR3 8 #endif #ifndef CRM #define CRM 8 #endif #ifndef CRI #define CRI 16 #endif #ifndef NBI #define NBI 188 #endif #ifndef CT #define CT 8 #endif // plan slots, per layer, base = L*LSTRIDE // KDA: 0-2 q(w,s,z) 3-5 k 6-8 v 9-11 g 12-14 o 15 beta_w 16 conv_w 17 attn_norm // 18 moe_norm 19 router 20-22 gate 23-25 up 26-28 down // 29-31 s_gate 32-34 s_up 35-37 s_down // MLA: 0-2 q 3-5 kv_a 6-8 kv_b 9-11 o 12 attn_norm 13 moe_norm 14 router // 15-17 gate 18-20 up 21-23 down 24-26 s_gate 27-29 s_up 30-32 s_down #define KDA_MOE 19 #define MLA_MOE 14 typedef __nv_bfloat16 bf16; typedef __nv_bfloat162 bf162; #define SM_XS 0 #define SM_HB 9216 #define SM_WN 13824 #define SM_MISC 46592 #define SM_WNF 54784 #define SM_QR 88064 #define SM_RC 92672 #define SM_BYTES 101376 // ---------------------------------------------------------------- barrier // sense-reversing grid barrier; `lg` is a per-block local generation counter // (all blocks pass the same barriers in the same order, so it stays in sync). __device__ __forceinline__ void gbar(volatile int* cntv, volatile int* genv, int* lg) { int* cnt = (int*)cntv; int* gen = (int*)genv; __syncthreads(); if (threadIdx.x == 0) { // the arrive is a device-scope release (publishes this block's writes) // and the release/acquire pair on `gen` hands them to every waiter, so // no separate __threadfence is needed. int old; asm volatile("atom.release.gpu.global.add.u32 %0, [%1], 1;" : "=r"(old) : "l"(cnt) : "memory"); if (old == NB - 1) { *cnt = 0; int g = *lg + 1; asm volatile("st.release.gpu.global.u32 [%0], %1;" ::"l"(gen), "r"(g) : "memory"); } else { int g; do { asm volatile("ld.acquire.gpu.global.u32 %0, [%1];" : "=r"(g) : "l"(gen) : "memory"); } while (g <= *lg); } *lg += 1; } __syncthreads(); } // ---------------------------------------------------------------- scalar ops __device__ __forceinline__ float silu_f(float x) { return x / (1.f + __expf(-x)); } __device__ __forceinline__ float sigm_f(float x) { return 1.f / (1.f + __expf(-x)); } __device__ __forceinline__ float bf2f(bf16 x) { return __bfloat162float(x); } __device__ __forceinline__ bf16 f2bf(float x) { return __float2bfloat16(x); } __device__ __forceinline__ float rbf(float x) { return __bfloat162float(__float2bfloat16(x)); } __device__ __forceinline__ float block_reduce(float s, float* red) { s += __shfl_down_sync(0xffffffffu, s, 16); s += __shfl_down_sync(0xffffffffu, s, 8); s += __shfl_down_sync(0xffffffffu, s, 4); s += __shfl_down_sync(0xffffffffu, s, 2); s += __shfl_down_sync(0xffffffffu, s, 1); if ((threadIdx.x & 31) == 0) red[threadIdx.x >> 5] = s; __syncthreads(); float t = 0.f; for (int i = 0; i < TPB / 32; i++) t += red[i]; __syncthreads(); return t; } // ---------------------------------------------------------------- work split struct Work { int tile, k0, k1; }; __device__ __forceinline__ Work assign_work_n(int ntiles, const int* cj, int K, int nblk) { int tot = 0; for (int j = 0; j < ntiles; j++) tot += cj[j]; int start[24]; int acc = 0; for (int j = 0; j < ntiles - 1; j++) { int bj = (int)(((long long)nblk * cj[j] + (tot >> 1)) / tot); if (bj < 1) bj = 1; start[j] = acc; acc += bj; } start[ntiles - 1] = acc; int b = blockIdx.x, j = 0; while (j + 1 < ntiles && b >= start[j + 1]) j++; int idx = b - start[j]; int cnt = ((j + 1 < ntiles) ? start[j + 1] : nblk) - start[j]; if (cnt < 1) cnt = 1; Work w; w.tile = j; w.k0 = (int)(((long long)idx * K) / cnt) & ~1; w.k1 = (idx >= cnt - 1) ? K : ((int)(((long long)(idx + 1) * K) / cnt) & ~1); if (w.k1 > K) w.k1 = K; if (w.k0 >= w.k1) { w.k0 = K; w.k1 = K; } return w; } __device__ __forceinline__ Work assign_work(int ntiles, const int* cj, int K) { return assign_work_n(ntiles, cj, K, NB); } __device__ __forceinline__ Work work_all(int K) { Work w; w.tile = 0; w.k0 = (int)(((long long)blockIdx.x * K) / NB) & ~1; w.k1 = (blockIdx.x >= NB - 1) ? K : ((int)(((long long)(blockIdx.x + 1) * K) / NB) & ~1); if (w.k1 > K) w.k1 = K; return w; } // ------------------------------------------------- fused int4 dequant-GEMV tile // thread owns CPT consecutive output channels starting at nch (byte offset nch/2) // out[nch+i] += mul * sum_k x[k] * (unpack(w)[k, nch+i] - z) * s // decomposed per 128-group: s*(sum x q) - (s*z)*(sum x) // every phase streams its weights exactly once, so the loop is purely memory // bound: BATCH row-pairs are kept in flight and the *next* BATCH is issued // before the current one is consumed (double buffering), which keeps the load // pipe continuously busy instead of bursting once per batch. __device__ __forceinline__ void pf_l2(const void* p) { asm volatile("prefetch.global.L2 [%0];" :: "l"(p)); } template __device__ __forceinline__ void ld_rows(uint32_t (&w)[BATCH][CPT / 4], const uint8_t* __restrict__ p, int N, int t, int np) { const int NW = CPT / 4; #pragma unroll for (int j = 0; j < BATCH; j++) { int rr = (t + j < np) ? (t + j) : (np - 1); const uint8_t* pp = p + (size_t)rr * N; if (NW % 4 == 0) { #pragma unroll for (int u = 0; u < NW / 4; u++) { const uint4 v = POL ? __ldg((const uint4*)pp + u) : __ldcs((const uint4*)pp + u); w[j][4 * u] = v.x; w[j][4 * u + 1] = v.y; w[j][4 * u + 2] = v.z; w[j][4 * u + 3] = v.w; } } else if (NW == 2) { const uint2 v = __ldg((const uint2*)pp); w[j][0] = v.x; w[j][1] = v.y; } else { w[j][0] = __ldg((const uint32_t*)pp); } } } // fp16 magic-number unpack: (a & mask) | 0x64006400 turns two packed nibbles // into two fp16 lanes holding (1024 + nib); subtracting 1024 is exact. One // LOP3 + one HSUB2 per two weights, one HFMA2 per two weights -- no I2F. #define HMAGIC 0x64006400u __device__ __forceinline__ __half2 h2of(uint32_t u) { return __halves2half2(__ushort_as_half((unsigned short)(u & 0xffffu)), __ushort_as_half((unsigned short)(u >> 16))); } __device__ __forceinline__ uint32_t u2h(__half2 h) { return (uint32_t)__half_as_ushort(__low2half(h)) | ((uint32_t)__half_as_ushort(__high2half(h)) << 16); } template __device__ __forceinline__ void acc_rows(const uint32_t (&w)[BATCH][CPT / 4], const float* __restrict__ x, int idx, int nb, uint32_t (&h)[CPT / 4][4], float& Sx) { const int NW = CPT / 4; const __half2 mg = h2of(HMAGIC); #pragma unroll for (int j = 0; j < BATCH; j++) { float x0 = (j < nb) ? x[idx + 2 * j] : 0.f; float x1 = (j < nb) ? x[idx + 2 * j + 1] : 0.f; Sx += x0 + x1; const __half2 xx0 = __half2half2(__float2half_rn(x0)); const __half2 xx1 = __half2half2(__float2half_rn(x1)); #pragma unroll for (int q = 0; q < NW; q++) { const uint32_t a0 = w[j][q]; // lanes: byte0|byte2 = columns 4q,4q+2 ; byte1|byte3 = 4q+1,4q+3 __half2 c0 = __hsub2(h2of((a0 & 0x000F000Fu) | HMAGIC), mg); __half2 c1 = __hsub2(h2of(((a0 >> 4) & 0x000F000Fu) | HMAGIC), mg); __half2 c2 = __hsub2(h2of(((a0 >> 8) & 0x000F000Fu) | HMAGIC), mg); __half2 c3 = __hsub2(h2of((((a0 >> 12) & 0x000F000Fu) | HMAGIC)), mg); h[q][0] = u2h(__hfma2(c0, xx0, h2of(h[q][0]))); h[q][1] = u2h(__hfma2(c1, xx1, h2of(h[q][1]))); h[q][2] = u2h(__hfma2(c2, xx0, h2of(h[q][2]))); h[q][3] = u2h(__hfma2(c3, xx1, h2of(h[q][3]))); } } } // fold the fp16 partial sums into the fp32 column accumulators template __device__ __forceinline__ void hflush(uint32_t (&h)[CPT / 4][4], float (&A)[CPT]) { const int NW = CPT / 4; #pragma unroll for (int q = 0; q < NW; q++) { #pragma unroll for (int b = 0; b < 4; b += 2) { const float2 f = __half22float2(__hadd2(h2of(h[q][b]), h2of(h[q][b + 1]))); A[4 * q + (b >> 1)] += f.x; A[4 * q + (b >> 1) + 2] += f.y; h[q][b] = 0u; h[q][b + 1] = 0u; } } } template __device__ __forceinline__ void gemv_tile(const uint8_t* __restrict__ wq, const bf16* __restrict__ sc, const bf16* __restrict__ zz, const float* __restrict__ x, float* __restrict__ out, int N, int k0, int k1, int nch, float mul) { if (k0 >= k1) return; const int NW = CPT / 4; float acc[CPT]; #pragma unroll for (int i = 0; i < CPT; i++) acc[i] = 0.f; const uint8_t* base = wq + (size_t)nch; int g0 = k0 >> 7, g1 = (k1 + 127) >> 7; for (int g = g0; g < g1; g++) { int r0 = max(k0, g << 7), r1 = min(k1, (g + 1) << 7); float A[CPT]; float Sx = 0.f; #pragma unroll for (int i = 0; i < CPT; i++) A[i] = 0.f; const uint8_t* p = base + (size_t)(r0 >> 1) * N; int np = (r1 - r0) >> 1; // branch-free 2-deep pipeline: ld_rows clamps at np-1 so an overrun // prefetch is harmless, and acc_rows zeroes x past nb, which keeps the // loads unconditional so ptxas can hoist them above the maths. uint32_t cw[BATCH][NW], pw[BATCH][NW]; uint32_t h[NW][4]; #pragma unroll for (int q = 0; q < NW; q++) { h[q][0] = 0u; h[q][1] = 0u; h[q][2] = 0u; h[q][3] = 0u; } int cnt = 0; ld_rows(cw, p, N, 0, np); for (int t = 0; t < np; t += 2 * BATCH) { const int t2 = t + BATCH; int nb = np - t; if (nb > BATCH) nb = BATCH; int nb2 = np - t2; if (nb2 < 0) nb2 = 0; if (nb2 > BATCH) nb2 = BATCH; ld_rows(pw, p, N, t2, np); acc_rows(cw, x, r0 + 2 * t, nb, h, Sx); ld_rows(cw, p, N, t2 + BATCH, np); acc_rows(pw, x, r0 + 2 * t2, nb2, h, Sx); if (++cnt == 8) { hflush(h, A); cnt = 0; } } hflush(h, A); const bf16* sp = sc + (size_t)g * N + nch; const bf16* zp = zz + (size_t)g * N + nch; #pragma unroll for (int j = 0; j < CPT; j += 8) { const uint4 sv = __ldcs((const uint4*)(sp + j)); const uint4 zv = __ldcs((const uint4*)(zp + j)); const bf16* sb = (const bf16*)&sv; const bf16* zb = (const bf16*)&zv; #pragma unroll for (int i = 0; i < 8; i++) { float s = bf2f(sb[i]), z = bf2f(zb[i]); acc[j + i] += mul * (s * A[j + i] - (s * z) * Sx); } } } if ((nch & 3) == 0) { #pragma unroll for (int i = 0; i < CPT; i += 4) { float4 v = make_float4(acc[i], acc[i + 1], acc[i + 2], acc[i + 3]); atomicAdd((float4*)(out + nch + i), v); } } else { #pragma unroll for (int i = 0; i < CPT; i++) atomicAdd(out + nch + i, acc[i]); } } // two packed matrices sharing one activation vector: both load streams are in // flight together, which doubles the memory-level parallelism of the phase. template __device__ __forceinline__ void ld2_rows(uint32_t (&w0)[BATCH][CPT / 4], uint32_t (&w1)[BATCH][CPT / 4], const uint8_t* __restrict__ p0, const uint8_t* __restrict__ p1, int N, int t, int np) { const int NW = CPT / 4; #pragma unroll for (int j = 0; j < BATCH; j++) { int rr = (t + j < np) ? (t + j) : (np - 1); const uint8_t* pp0 = p0 + (size_t)rr * N; const uint8_t* pp1 = p1 + (size_t)rr * N; if (NW % 4 == 0) { #pragma unroll for (int u = 0; u < NW / 4; u++) { const uint4 a = __ldcs((const uint4*)pp0 + u), c = __ldcs((const uint4*)pp1 + u); w0[j][4*u] = a.x; w0[j][4*u+1] = a.y; w0[j][4*u+2] = a.z; w0[j][4*u+3] = a.w; w1[j][4*u] = c.x; w1[j][4*u+1] = c.y; w1[j][4*u+2] = c.z; w1[j][4*u+3] = c.w; } } else if (NW == 2) { const uint2 a = __ldg((const uint2*)pp0), c = __ldg((const uint2*)pp1); w0[j][0]=a.x; w0[j][1]=a.y; w1[j][0]=c.x; w1[j][1]=c.y; } else { w0[j][0] = __ldg((const uint32_t*)pp0); w1[j][0] = __ldg((const uint32_t*)pp1); } } } template __device__ __forceinline__ void acc2_rows(const uint32_t (&w0)[BATCH][CPT / 4], const uint32_t (&w1)[BATCH][CPT / 4], const float* __restrict__ x, int idx, int nb, float (&A0)[CPT], float (&A1)[CPT], float& Sx) { const int NW = CPT / 4; #pragma unroll for (int j = 0; j < BATCH; j++) { float x0 = (j < nb) ? x[idx + 2 * j] : 0.f; float x1 = (j < nb) ? x[idx + 2 * j + 1] : 0.f; Sx += x0 + x1; #pragma unroll for (int q = 0; q < NW; q++) { uint32_t a0 = w0[j][q], a1 = w1[j][q]; #pragma unroll for (int c4 = 0; c4 < 4; c4++) { int c = 4 * q + c4; uint32_t m = 0xFu << (8 * c4); A0[c] += x0 * (float)((a0 & m) >> (8 * c4)) + x1 * (float)((a0 & (m << 4)) >> (8 * c4 + 4)); A1[c] += x0 * (float)((a1 & m) >> (8 * c4)) + x1 * (float)((a1 & (m << 4)) >> (8 * c4 + 4)); } } } } template __device__ __forceinline__ void gemv2(const uint8_t* __restrict__ wq0, const bf16* __restrict__ sc0, const bf16* __restrict__ zz0, const uint8_t* __restrict__ wq1, const bf16* __restrict__ sc1, const bf16* __restrict__ zz1, const float* __restrict__ x, float* __restrict__ out0, float* __restrict__ out1, int N, int k0, int k1, int nch, float mul) { if (k0 >= k1) return; const int NW = CPT / 4; float acc0[CPT], acc1[CPT]; #pragma unroll for (int i = 0; i < CPT; i++) { acc0[i] = 0.f; acc1[i] = 0.f; } const uint8_t* base0 = wq0 + (size_t)nch; const uint8_t* base1 = wq1 + (size_t)nch; int g0 = k0 >> 7, g1 = (k1 + 127) >> 7; for (int g = g0; g < g1; g++) { int r0 = max(k0, g << 7), r1 = min(k1, (g + 1) << 7); float A0[CPT], A1[CPT]; float Sx = 0.f; #pragma unroll for (int i = 0; i < CPT; i++) { A0[i] = 0.f; A1[i] = 0.f; } const uint8_t* p0 = base0 + (size_t)(r0 >> 1) * N; const uint8_t* p1 = base1 + (size_t)(r0 >> 1) * N; int np = (r1 - r0) >> 1; uint32_t c0[BATCH][NW], c1[BATCH][NW], n0[BATCH][NW], n1[BATCH][NW]; ld2_rows(c0, c1, p0, p1, N, 0, np); for (int t = 0; t < np; t += 2 * BATCH) { const int t2 = t + BATCH; int nb = np - t; if (nb > BATCH) nb = BATCH; if (t2 < np) ld2_rows(n0, n1, p0, p1, N, t2, np); acc2_rows(c0, c1, x, r0 + 2 * t, nb, A0, A1, Sx); if (t2 >= np) break; int nb2 = np - t2; if (nb2 > BATCH) nb2 = BATCH; if (t2 + BATCH < np) ld2_rows(c0, c1, p0, p1, N, t2 + BATCH, np); acc2_rows(n0, n1, x, r0 + 2 * t2, nb2, A0, A1, Sx); } const bf16* sp0 = sc0 + (size_t)g * N + nch; const bf16* zp0 = zz0 + (size_t)g * N + nch; const bf16* sp1 = sc1 + (size_t)g * N + nch; const bf16* zp1 = zz1 + (size_t)g * N + nch; #pragma unroll for (int j = 0; j < CPT; j += 8) { const uint4 sv0 = *(const uint4*)(sp0 + j), zv0 = *(const uint4*)(zp0 + j); const uint4 sv1 = *(const uint4*)(sp1 + j), zv1 = *(const uint4*)(zp1 + j); const bf16* sb0 = (const bf16*)&sv0; const bf16* zb0 = (const bf16*)&zv0; const bf16* sb1 = (const bf16*)&sv1; const bf16* zb1 = (const bf16*)&zv1; #pragma unroll for (int i = 0; i < 8; i++) { float s0 = bf2f(sb0[i]), z0 = bf2f(zb0[i]); float s1 = bf2f(sb1[i]), z1 = bf2f(zb1[i]); acc0[j + i] += mul * (s0 * A0[j + i] - (s0 * z0) * Sx); acc1[j + i] += mul * (s1 * A1[j + i] - (s1 * z1) * Sx); } } } #pragma unroll for (int i = 0; i < CPT; i += 4) { atomicAdd((float4*)(out0 + nch + i), make_float4(acc0[i], acc0[i+1], acc0[i+2], acc0[i+3])); atomicAdd((float4*)(out1 + nch + i), make_float4(acc1[i], acc1[i+1], acc1[i+2], acc1[i+3])); } } template __device__ __forceinline__ void gemv_row(const uint8_t* __restrict__ wq, const bf16* __restrict__ sc, const bf16* __restrict__ zz, const float* __restrict__ x, float* __restrict__ out, int N, int k0, int k1, float mul) { int nch = threadIdx.x * CPT; if (nch < N) gemv_tile(wq, sc, zz, x, out, N, k0, k1, nch, mul); nch = TPB * CPT + threadIdx.x * CPT; if (nch < N) gemv_tile(wq, sc, zz, x, out, N, k0, k1, nch, mul); } // one m16n8k16 bf16 mma tile with fp32 accumulation __device__ __forceinline__ void mma16816(uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1, float (&c)[4]) { asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};" : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3]) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)); } // Like gemv_row, but the block's k range is split three ways across the three // 128-thread groups so the block keeps three independent weight streams in // flight (the pattern phase H uses); the tail columns past 128*CPT are picked // up by the first few lanes of each group. template __device__ __forceinline__ void gemv_row3(const uint8_t* __restrict__ wq, const bf16* __restrict__ sc, const bf16* __restrict__ zz, const float* __restrict__ x, float* __restrict__ out, int N, int k0, int k1, float mul) { const int grp = threadIdx.x >> 7; const int lane = threadIdx.x & 127; int a0 = (int)(k0 + (((long long)(k1 - k0) * grp) / 3)) & ~1; int a1 = (grp == 2) ? k1 : ((int)(k0 + (((long long)(k1 - k0) * (grp + 1)) / 3)) & ~1); int nch = lane * CPT; if (nch < N) gemv_tile(wq, sc, zz, x, out, N, a0, a1, nch, mul); int nt = (N - 128 * CPT) / CPT; if (lane < nt) gemv_tile(wq, sc, zz, x, out, N, a0, a1, 128 * CPT + nch, mul); } // ---------------------------------------------------------------- row dot __device__ __forceinline__ float dot_row(const bf16* __restrict__ w, const float* __restrict__ x, int K, float* red) { float s = 0.f; for (int i = threadIdx.x; i < K; i += TPB) s += bf2f(w[i]) * x[i]; return block_reduce(s, red); } // ---------------------------------------------------------------- zero fill __device__ __forceinline__ void zfill(float* __restrict__ p, int n, int b, int tid) { int per = (n + NB - 1) / NB; int s = b * per, e = min(n, s + per); for (int i = s + tid; i < e; i += TPB) p[i] = 0.f; } // pick partial slot `sl` of a wide output accumulator __device__ __forceinline__ float* pslot(float* s0, float* sx, int sl) { return sl ? sx + (sl - 1) * HID : s0; } // ---------------------------------------------------------------- norm __device__ __forceinline__ void prep(const bf16* __restrict__ hin, const float* __restrict__ prev, const float* __restrict__ prevx, const bf16* __restrict__ w, int first, bf16* HB, float* XS, float* red) { // register-resident: HID == NPT * TPB const int NPT = HID / TPB; // all three input streams are issued together so their latencies overlap float v[NPT], wv[NPT], hv[NPT]; #pragma unroll for (int j = 0; j < NPT; j++) { int i = threadIdx.x + j * TPB; wv[j] = bf2f(w[i]); hv[j] = first ? bf2f(hin[i]) : bf2f(HB[i]); } if (!first) { #pragma unroll for (int j = 0; j < NPT; j++) { int i = threadIdx.x + j * TPB; // the producer split the projection over blocks; sum its slots float ps = prev[i]; #pragma unroll for (int q = 0; q < PSL - 1; q++) ps += prevx[q * HID + i]; hv[j] += rbf(ps); } } #pragma unroll for (int j = 0; j < NPT; j++) { v[j] = hv[j]; HB[threadIdx.x + j * TPB] = f2bf(v[j]); } float s = 0.f; #pragma unroll for (int j = 0; j < NPT; j++) s += v[j] * v[j]; s = block_reduce(s, red); float r = rsqrtf(s / (float)HID + 1e-6f); #pragma unroll for (int j = 0; j < NPT; j++) XS[threadIdx.x + j * TPB] = rbf(v[j] * r * wv[j]); __syncthreads(); } // ---------------------------------------------------------------- exp enc __device__ __forceinline__ unsigned fenc(float f) { unsigned u = __float_as_uint(f); return (u & 0x80000000u) ? ~u : (u | 0x80000000u); } __device__ __forceinline__ float fdec(unsigned e) { return __uint_as_float((e & 0x80000000u) ? (e & 0x7fffffffu) : ~e); } // =========================================================================== extern "C" __global__ void __launch_bounds__(TPB) mega(const int64_t* __restrict__ plan, int pos, int fresh, const bf16* __restrict__ hin, bf16* __restrict__ hout, float* S0, float* S1, float* S2, bf16* cq0, bf16* ck0, bf16* cv0, bf16* cq1, bf16* ck1, bf16* cv1, bf16* cq2, bf16* ck2, bf16* cv2, const bf16* __restrict__ ckv_in, const bf16* __restrict__ ckr_in, bf16* ckv_buf, bf16* ckr_buf, float* __restrict__ scratch, int* bar) { extern __shared__ char smem[]; volatile int* bcnt = (volatile int*)bar; volatile int* bgen = (volatile int*)bar + 1; int lgen = *bgen; const int b = blockIdx.x, tid = threadIdx.x, warp = tid >> 5, lane = tid & 31; float* XS = (float*)(smem + SM_XS); bf16* HB = (bf16*)(smem + SM_HB); bf16* WNB = (bf16*)(smem + SM_WN); float* MISC = (float*)(smem + SM_MISC); float* red = MISC; int* sel = (int*)(smem + SM_BYTES - 128); float* wts = (float*)(smem + SM_BYTES - 64); float* AF = scratch; // slot 0 of the residual partials float* YF = AF + 2 * HID; // slot 0 of the moe partials float* QF = YF + 2 * HID; float* KF = QF + HQ; float* VF = KF + HK; float* GF = VF + HK; float* BETAF = GF + HK; float* OB = BETAF + 32; float* LOGIT = OB + HK; float* GU = LOGIT + 64; float* UU = GU + NSLOT * 2 * MI; float* WNG = UU + NSLOT * MI; float* CVG = WNG + 32 * 512; float* SC = CVG + 32 * 512; float* PMAXV = SC + 32768 * 32; float* PSUM = PMAXV + 32; float* AX = PSUM + 32; // residual partial slots 1..3 float* YX = AX + (PSL - 1) * 2 * HID; // moe partial slots 1..3 float* CVGX = YX + (PSL - 1) * 2 * HID; // mla cavg partial slots 1..3 float* PSUMX = CVGX + (PSL - 1) * 32 * 512; // mla prob-mass partial slots float* Sd[3] = {S0, S1, S2}; bf16* cqd[3] = {cq0, cq1, cq2}; bf16* ckd[3] = {ck0, ck1, ck2}; bf16* cvd[3] = {cv0, cv1, cv2}; zfill(AF, 2 * HID, b, tid); zfill(YF, 2 * HID, b, tid); zfill(AX, (PSL - 1) * 2 * HID, b, tid); zfill(YX, (PSL - 1) * 2 * HID, b, tid); zfill(QF, HK, b, tid); zfill(KF, HK, b, tid); zfill(VF, HK, b, tid); zfill(GF, HK, b, tid); zfill(OB, HK, b, tid); if (b == 0) { for (int i = tid; i < 32; i += TPB) { *(unsigned int*)(PMAXV + i) = fenc(-3.0e38f); PSUM[i] = 0.f; BETAF[i] = 0.f; for (int q = 0; q < PSL - 1; q++) PSUMX[q * 32 + i] = 0.f; } } gbar(bcnt, bgen, &lgen); for (int L = 0; L < NLAY; L++) { const int64_t* pl = plan + (size_t)L * LSTRIDE; const int mo = (L == NLAY - 1 && NLAY == 4) ? MLA_MOE : KDA_MOE; const bf16* anorm = (const bf16*)pl[(L == 3) ? 12 : 17]; const bf16* mnorm = (const bf16*)pl[(L == 3) ? 13 : 18]; float* Acur = AF + (L & 1) * HID; float* Ycur = YF + (L & 1) * HID; float* Yprev = YF + ((L & 1) ^ 1) * HID; float* AcurX = AX + (L & 1) * (PSL - 1) * HID; float* YcurX = YX + (L & 1) * (PSL - 1) * HID; float* YprevX = YX + (((L & 1) ^ 1)) * (PSL - 1) * HID; // ---- attention input norm: HB = hidden_L ---- prep((L == 0) ? hin : nullptr, (L == 0) ? nullptr : Yprev, (L == 0) ? nullptr : YprevX, anorm, (L == 0) ? 1 : 0, HB, XS, red); if (L < 3) { // ================= KDA ================= // ---- phase 1: q,k,v,g + beta + conv-window shift ---- if (b < 156) { int mat = b / 39, sub = b % 39; int k0 = (int)(((long long)sub * HID) / 39) & ~1; int k1 = (sub == 38) ? HID : ((int)(((long long)(sub + 1) * HID) / 39) & ~1); const uint8_t* W = (const uint8_t*)pl[mat * 3]; const bf16* Sc = (const bf16*)pl[mat * 3 + 1]; const bf16* Zz = (const bf16*)pl[mat * 3 + 2]; float* O = (mat == 0) ? QF : (mat == 1) ? KF : (mat == 2) ? VF : GF; if (tid < 256) gemv_tile<16, B1, 1>(W, Sc, Zz, XS, O, HK, k0, k1, tid * 16, 1.0f); } else { int ch = b - 156; float d = dot_row((const bf16*)pl[15] + (size_t)ch * HID, XS, HID, red); if (tid == 0) BETAF[ch] = rbf(d); } gbar(bcnt, bgen, &lgen); // ---- phase 2: short conv + KDA state update ---- if (b < 128) { int h = b >> 2, jt = b & 3; float* smq = MISC; float* smk = MISC + 128; float* smv = MISC + 256; float* sme = MISC + 384; float* smred = MISC + 512; float* smkap = MISC + 1088; if (tid < 128) { int c = 128 * h + tid; const bf16* cw = (const bf16*)pl[16]; bf16* st3[3] = {cqd[L], ckd[L], cvd[L]}; const float* src3[3] = {QF, KF, VF}; float so[3]; #pragma unroll for (int a = 0; a < 3; a++) { const bf16* cwp = cw + (size_t)a * HK * 4 + (size_t)c * 4; const bf16* pw = st3[a]; float val = rbf(src3[a][c]); float o = bf2f(pw[c]) * bf2f(cwp[0]) + bf2f(pw[HK + c]) * bf2f(cwp[1]) + bf2f(pw[2 * HK + c]) * bf2f(cwp[2]) + val * bf2f(cwp[3]); so[a] = rbf(silu_f(o)); } smq[tid] = so[0] * 0.08838834764831845f; smk[tid] = so[1]; smv[tid] = so[2]; float gv = rbf(GF[c]); sme[tid] = 1.f / (1.f + __expf(gv)); } __syncthreads(); if (tid < 32) { float s = 0.f; for (int i = tid; i < 128; i += 32) s += smk[i] * smq[i]; s += __shfl_down_sync(0xffffffffu, s, 16); s += __shfl_down_sync(0xffffffffu, s, 8); s += __shfl_down_sync(0xffffffffu, s, 4); s += __shfl_down_sync(0xffffffffu, s, 2); s += __shfl_down_sync(0xffffffffu, s, 1); if (tid == 0) smkap[0] = s; } float beta = sigm_f(rbf(BETAF[h])); int i0 = warp * 16; int j = jt * 32 + lane; float* Sp = Sd[L] + (size_t)h * 16384 + j; float Sv[16]; float pp = 0.f, aa = 0.f; if (warp < 8) { #pragma unroll for (int u = 0; u < 16; u++) { Sv[u] = Sp[(size_t)(i0 + u) * 128]; float se = Sv[u] * sme[i0 + u]; pp += se * smk[i0 + u]; aa += se * smq[i0 + u]; } smred[warp * 32 + lane] = pp; smred[256 + warp * 32 + lane] = aa; } __syncthreads(); float pred = 0.f, Aacc = 0.f; #pragma unroll for (int w2 = 0; w2 < 8; w2++) { pred += smred[w2 * 32 + lane]; Aacc += smred[256 + w2 * 32 + lane]; } float kappa = smkap[0]; float d = smv[j] - pred; float bk = beta * d; #pragma unroll if (warp < 8) { #pragma unroll for (int u = 0; u < 16; u++) { Sp[(size_t)(i0 + u) * 128] = Sv[u] * sme[i0 + u] + bk * smk[i0 + u]; } } if (warp == 0 && tid < 32) OB[128 * h + j] = rbf(Aacc + beta * kappa * d); } gbar(bcnt, bgen, &lgen); // ---- phase 3: o_proj (+ conv-window shift, after all conv reads) ---- if (b >= 156 && tid < 128) { int hh = b - 156; int c = 128 * hh + tid; bf16* st3[3] = {cqd[L], ckd[L], cvd[L]}; const float* src3[3] = {QF, KF, VF}; #pragma unroll for (int a = 0; a < 3; a++) { bf16* pw = st3[a]; pw[c] = pw[HK + c]; pw[HK + c] = pw[2 * HK + c]; pw[2 * HK + c] = f2bf(rbf(src3[a][c])); } } { Work w = work_all(HK); gemv_row((const uint8_t*)pl[12], (const bf16*)pl[13], (const bf16*)pl[14], OB, pslot(Acur, AcurX, b & (PSL - 1)), HID, w.k0, w.k1, 1.0f); } gbar(bcnt, bgen, &lgen); } else { // ================= MLA ================= int nrow = pos + 1; // ---- M1: q + kv_a + fresh copy + zero WNG/CVG ---- { int cj[4] = {2048, 2048, 2048, 576}; Work w = assign_work(4, cj, HID); if (w.tile < 3) { if (tid < 256) gemv_tile((const uint8_t*)pl[0], (const bf16*)pl[1], (const bf16*)pl[2], XS, QF, HQ, w.k0, w.k1, w.tile * 2048 + tid * 8, 1.0f); } else { if (tid < 72) gemv_tile((const uint8_t*)pl[3], (const bf16*)pl[4], (const bf16*)pl[5], XS, KF, KVL + KVR, w.k0, w.k1, tid * 8, 1.0f); } zfill(WNG, 32 * 512, b, tid); zfill(CVG, 32 * 512, b, tid); zfill(CVGX, (PSL - 1) * 32 * 512, b, tid); for (int i = tid; i < 32; i += TPB) { *(unsigned int*)(PMAXV + i) = fenc(-3.0e38f); PSUM[i] = 0.f; } if (fresh && b < NB) { int per = (pos + NB - 1) / NB; int r0 = b * per, r1 = min(pos, r0 + per); for (int r = r0; r < r1; r++) { for (int i = tid; i < KVL; i += TPB) ckv_buf[(size_t)r * KVL + i] = ckv_in[(size_t)r * KVL + i]; for (int i = tid; i < KVR; i += TPB) ckr_buf[(size_t)r * KVR + i] = ckr_in[(size_t)r * KVR + i]; } } } gbar(bcnt, bgen, &lgen); // ---- M2: Wn absorb (b<128) + append/rope (b==128) ---- if (b < 128 && b * TPB + tid < 32768) { int gid = b * TPB + tid; int dpart = gid & 3, mp = (gid >> 2) & 255, h = gid >> 10; const uint8_t* wrow = (const uint8_t*)pl[6] + (size_t)mp * 8192 + h * 256 + dpart * 32; int grp = mp >> 6; const bf16* srow = (const bf16*)pl[7] + (size_t)grp * 8192 + h * 256 + dpart * 32; const bf16* zrow = (const bf16*)pl[8] + (size_t)grp * 8192 + h * 256 + dpart * 32; float a0 = 0.f, a1 = 0.f; const float* qp = QF + (size_t)h * 192 + dpart * 32; for (int q = 0; q < 32; q += 16) { const uint4 wv = *(const uint4*)(wrow + q); const uint4 sv0 = *(const uint4*)(srow + q), sv1 = *(const uint4*)(srow + q + 8); const uint4 zv0 = *(const uint4*)(zrow + q), zv1 = *(const uint4*)(zrow + q + 8); const uint32_t* ww = (const uint32_t*)&wv; bf16 sb[16], zb[16]; *(uint4*)&sb[0] = sv0; *(uint4*)&sb[8] = sv1; *(uint4*)&zb[0] = zv0; *(uint4*)&zb[8] = zv1; #pragma unroll for (int j = 0; j < 4; j++) { uint32_t w32 = ww[j]; int d0 = q + 4 * j; float c0 = rbf(qp[d0]) * bf2f(sb[4 * j]); float c1 = rbf(qp[d0 + 1]) * bf2f(sb[4 * j + 1]); float c2 = rbf(qp[d0 + 2]) * bf2f(sb[4 * j + 2]); float c3 = rbf(qp[d0 + 3]) * bf2f(sb[4 * j + 3]); float zn = c0 * bf2f(zb[4 * j]) + c1 * bf2f(zb[4 * j + 1]) + c2 * bf2f(zb[4 * j + 2]) + c3 * bf2f(zb[4 * j + 3]); a0 += (float)(w32 & 0xF) * c0 + (float)((w32 >> 8) & 0xF) * c1 + (float)((w32 >> 16) & 0xF) * c2 + (float)((w32 >> 24) & 0xF) * c3 - zn; a1 += (float)((w32 >> 4) & 0xF) * c0 + (float)((w32 >> 12) & 0xF) * c1 + (float)((w32 >> 20) & 0xF) * c2 + (float)((w32 >> 28) & 0xF) * c3 - zn; } } int m0 = 2 * mp; atomicAdd(WNG + (size_t)(m0 >> 2) * 128 + h * 4 + (m0 & 3), a0); atomicAdd(WNG + (size_t)((m0 + 1) >> 2) * 128 + h * 4 + ((m0 + 1) & 3), a1); } else if (b == 128) { for (int gid = tid; gid < KVL + KVR; gid += TPB) { if (gid < KVL) { ckv_buf[(size_t)pos * KVL + gid] = f2bf(KF[gid]); } else { int d = gid - KVL, k = d >> 1; float inv = 1.f / powf(10000.f, (float)k / 32.f); float ang = (float)pos * inv; float co = cosf(ang), si = sinf(ang); float e = rbf(KF[KVL + d]), e2 = rbf(KF[KVL + (d ^ 1)]); float o = (d & 1) ? (e * co + e2 * si) : (e * co - e2 * si); ckr_buf[(size_t)pos * KVR + d] = f2bf(o); } } } gbar(bcnt, bgen, &lgen); // ---- M3: scores ---- // SC[rows, 32 heads] = ckv[rows, 512] @ Wn[512, 32] plus the rope // term ckr[rows, 64] @ qrot[64, 32]. Both are tiny-matmul shapes // (n = 32), so 16-row tiles go through four m16n8k16 bf16 tensor-core // accumulators instead of scalar dots: the phase becomes DRAM bound // on the kv cache instead of instruction bound. { // The angle only depends on k = d >> 1, so there are 32 of them: // compute the transcendentals once instead of 64*32 times. float* rco = (float*)(smem + SM_RC); float* rsi = rco + 32; if (tid < 32) { float ang = (float)pos / powf(10000.f, (float)tid / 32.f); rco[tid] = cosf(ang); rsi[tid] = sinf(ang); } __syncthreads(); // Wn as bf16 in [head][520] so a lane's two consecutive k weights // are a single 4-byte load (the mma B fragment layout); the +8 // pad keeps the head stride off a bank multiple. bf16* wnf = (bf16*)(smem + SM_WNF); { // WNG is blocked [k/8][k%8 first half?][head][4]; a float4 of it // is one head's four consecutive k, so the read is coalesced and // the store is a single 8-byte smem write. const float4* wg4 = (const float4*)WNG; for (int i = tid; i < 32 * 512 / 4; i += TPB) { const float4 v = wg4[i]; const int h = i & 31, s2 = (i >> 5) & 1, q2 = i >> 6; bf16 r4[4] = {f2bf(v.x), f2bf(v.y), f2bf(v.z), f2bf(v.w)}; *(uint2*)(wnf + h * 520 + 8 * q2 + 4 * s2) = *(const uint2*)r4; } } // rotated q_rope as bf16 in [head][64+8] bf16* qrb = (bf16*)(smem + SM_QR); if (tid < 256) { int h = tid >> 3, d0 = (tid & 7) * 8; const float* qs = QF + h * 192 + 128 + d0; const float4 va = *(const float4*)qs, vb = *(const float4*)(qs + 4); float e[8] = {rbf(va.x), rbf(va.y), rbf(va.z), rbf(va.w), rbf(vb.x), rbf(vb.y), rbf(vb.z), rbf(vb.w)}; #pragma unroll for (int j = 0; j < 8; j += 2) { const int d = d0 + j; const float co = rco[d >> 1], si = rsi[d >> 1]; qrb[h * 72 + d] = f2bf(rbf(e[j] * co - e[j + 1] * si)); qrb[h * 72 + d + 1] = f2bf(rbf(e[j + 1] * co + e[j] * si)); } } __syncthreads(); const int per = ((((nrow + NB - 1) / NB) + 15) & ~15); const int r0 = b * per, r1 = min(nrow, r0 + per); const int row = lane >> 2, kq = (lane & 3) * 2; const int hb = (lane >> 2) * 520, hq = (lane >> 2) * 72; const int hcol = (lane & 3) * 2; float lmax = -3.0e38f; for (int t0 = r0 + warp * 16; t0 < r1; t0 += 128) { float acc[4][4]; #pragma unroll for (int n = 0; n < 4; n++) #pragma unroll for (int j = 0; j < 4; j++) acc[n][j] = 0.f; const int tA = t0 + row, tB = tA + 8; const bf16* pa = ckv_buf + (size_t)(tA < r1 ? tA : r1 - 1) * KVL; const bf16* pb = ckv_buf + (size_t)(tB < r1 ? tB : r1 - 1) * KVL; for (int kt = 0; kt < 32; kt++) { const int kk = kt * 16 + kq; const uint32_t a0 = *(const uint32_t*)(pa + kk); const uint32_t a1 = *(const uint32_t*)(pb + kk); const uint32_t a2 = *(const uint32_t*)(pa + kk + 8); const uint32_t a3 = *(const uint32_t*)(pb + kk + 8); const bf16* wq = wnf + hb + kk; #pragma unroll for (int n = 0; n < 4; n++) { const uint32_t b0 = *(const uint32_t*)(wq + n * 8 * 520); const uint32_t b1 = *(const uint32_t*)(wq + n * 8 * 520 + 8); mma16816(a0, a1, a2, a3, b0, b1, acc[n]); } } const bf16* ra = ckr_buf + (size_t)(tA < r1 ? tA : r1 - 1) * KVR; const bf16* rb = ckr_buf + (size_t)(tB < r1 ? tB : r1 - 1) * KVR; for (int kt = 0; kt < 4; kt++) { const int kk = kt * 16 + kq; const uint32_t a0 = *(const uint32_t*)(ra + kk); const uint32_t a1 = *(const uint32_t*)(rb + kk); const uint32_t a2 = *(const uint32_t*)(ra + kk + 8); const uint32_t a3 = *(const uint32_t*)(rb + kk + 8); const bf16* qq = qrb + hq + kk; #pragma unroll for (int n = 0; n < 4; n++) { const uint32_t b0 = *(const uint32_t*)(qq + n * 8 * 72); const uint32_t b1 = *(const uint32_t*)(qq + n * 8 * 72 + 8); mma16816(a0, a1, a2, a3, b0, b1, acc[n]); } } // c0,c1 = C[row][hcol + n*8 .. +1]; c2,c3 = C[row+8][same] #pragma unroll for (int n = 0; n < 4; n++) { const int hd = n * 8 + hcol; if (tA < r1) { float s0 = acc[n][0] * 0.07216878364870322f; float s1 = acc[n][1] * 0.07216878364870322f; *(float2*)(SC + (size_t)tA * 32 + hd) = make_float2(s0, s1); lmax = fmaxf(lmax, fmaxf(s0, s1)); } if (tB < r1) { float s0 = acc[n][2] * 0.07216878364870322f; float s1 = acc[n][3] * 0.07216878364870322f; *(float2*)(SC + (size_t)tB * 32 + hd) = make_float2(s0, s1); lmax = fmaxf(lmax, fmaxf(s0, s1)); } } } __syncthreads(); MISC[warp * 32 + lane] = lmax; __syncthreads(); if (warp == 0) { float m = MISC[lane]; for (int w2 = 1; w2 < TPB / 32; w2++) m = fmaxf(m, MISC[w2 * 32 + lane]); atomicMax((unsigned int*)(PMAXV + lane), fenc(m)); } } gbar(bcnt, bgen, &lgen); // ---- M4: softmax + cavg (lane = head, 16 m per lane, warp = row) ---- { float pmax = fdec(*(const unsigned int*)(PMAXV + lane)); int r0 = (int)(((long long)b * nrow) / NB); int r1 = (int)(((long long)(b + 1) * nrow) / NB); int mbase = lane * 16; const bf16* cbase = ckv_buf + mbase; float pacc = 0.f; float acc[16]; #pragma unroll for (int i = 0; i < 16; i++) acc[i] = 0.f; for (int rb = r0 + warp; warp < 8 && rb < r1; rb += 32) { uint4 c0[4], c1[4]; float pv[4]; #pragma unroll for (int j = 0; j < 4; j++) { int rr = rb + 8 * j; int rc = (rr < r1) ? rr : (r1 - 1); pv[j] = (rr < r1) ? __expf(SC[(size_t)rc * 32 + lane] - pmax) : 0.f; c0[j] = *(const uint4*)(cbase + (size_t)rc * KVL); c1[j] = *(const uint4*)(cbase + (size_t)rc * KVL + 8); } #pragma unroll for (int j = 0; j < 4; j++) { float2 f0 = __bfloat1622float2(*(const bf162*)&c0[j].x); float2 f1 = __bfloat1622float2(*(const bf162*)&c0[j].y); float2 f2 = __bfloat1622float2(*(const bf162*)&c0[j].z); float2 f3 = __bfloat1622float2(*(const bf162*)&c0[j].w); float2 f4 = __bfloat1622float2(*(const bf162*)&c1[j].x); float2 f5 = __bfloat1622float2(*(const bf162*)&c1[j].y); float2 f6 = __bfloat1622float2(*(const bf162*)&c1[j].z); float2 f7 = __bfloat1622float2(*(const bf162*)&c1[j].w); float p = pv[j]; acc[0] += p * f0.x; acc[1] += p * f0.y; acc[2] += p * f1.x; acc[3] += p * f1.y; acc[4] += p * f2.x; acc[5] += p * f2.y; acc[6] += p * f3.x; acc[7] += p * f3.y; acc[8] += p * f4.x; acc[9] += p * f4.y; acc[10] += p * f5.x; acc[11] += p * f5.y; acc[12] += p * f6.x; acc[13] += p * f6.y; acc[14] += p * f7.x; acc[15] += p * f7.y; } pacc += (pv[0] + pv[1]) + (pv[2] + pv[3]); } const int sl = b & (PSL - 1); float* cp = sl ? CVGX + (sl - 1) * 32 * 512 + lane * 512 + mbase : CVG + lane * 512 + mbase; #pragma unroll for (int i = 0; i < 16; i += 4) { float4 v = make_float4(acc[i], acc[i + 1], acc[i + 2], acc[i + 3]); atomicAdd((float4*)(cp + i), v); } atomicAdd(sl ? PSUMX + (sl - 1) * 32 + lane : PSUM + lane, pacc); } gbar(bcnt, bgen, &lgen); // ---- M5: o = cavg @ kv_b[v] ---- { int gid = b * TPB + tid; if (gid < 32768) { int dg = gid & 15, ms = (gid >> 4) & 63, h = gid >> 10; float ps = PSUM[h]; #pragma unroll for (int q = 0; q < PSL - 1; q++) ps += PSUMX[q * 32 + h]; float inv = 1.f / (ps + 1e-30f); const uint8_t* wrow = (const uint8_t*)pl[6] + (size_t)(ms * 4) * 8192 + h * 256 + 128 + dg * 8; float S1[8], S0 = 0.f; #pragma unroll for (int i = 0; i < 8; i++) S1[i] = 0.f; #pragma unroll for (int t = 0; t < 4; t++) { const uint2 wv = *(const uint2*)(wrow + (size_t)t * 8192); const uint32_t* ww = (const uint32_t*)&wv; int m0 = (ms * 4 + t) * 2; const float2* q0 = (const float2*)(CVG + h * 512 + m0); float2 a0 = q0[0], a1, a2, a3; { const float2* p1 = (const float2*)(CVGX + h * 512 + m0); a1 = p1[0]; a2 = p1[8192]; a3 = p1[16384]; } float c0 = (a0.x + a1.x + a2.x + a3.x) * inv; float c1 = (a0.y + a1.y + a2.y + a3.y) * inv; S0 += c0 + c1; #pragma unroll for (int j = 0; j < 2; j++) { S1[4 * j] += c0 * (float)((ww[j] >> 0) & 0xF) + c1 * (float)((ww[j] >> 4) & 0xF); S1[4 * j + 1] += c0 * (float)((ww[j] >> 8) & 0xF) + c1 * (float)((ww[j] >> 12) & 0xF); S1[4 * j + 2] += c0 * (float)((ww[j] >> 16) & 0xF) + c1 * (float)((ww[j] >> 20) & 0xF); S1[4 * j + 3] += c0 * (float)((ww[j] >> 24) & 0xF) + c1 * (float)((ww[j] >> 28) & 0xF); } } int grp = (ms * 4) >> 6; const bf16* srow = (const bf16*)pl[7] + (size_t)grp * 8192 + h * 256 + 128 + dg * 8; const bf16* zrow = (const bf16*)pl[8] + (size_t)grp * 8192 + h * 256 + 128 + dg * 8; const uint4 sv = *(const uint4*)srow; const uint4 zv = *(const uint4*)zrow; const bf16* sb = (const bf16*)&sv; const bf16* zb = (const bf16*)&zv; #pragma unroll for (int i = 0; i < 8; i += 4) { float4 v = make_float4(0.f, 0.f, 0.f, 0.f); const bf16* sb4 = sb + i; const bf16* zb4 = zb + i; float s0 = bf2f(sb4[0]), z0 = bf2f(zb4[0]); float s1 = bf2f(sb4[1]), z1 = bf2f(zb4[1]); float s2 = bf2f(sb4[2]), z2 = bf2f(zb4[2]); float s3 = bf2f(sb4[3]), z3 = bf2f(zb4[3]); // each output element has exactly one writer, so store // (rounded to bf16, matching reference o.to(bf16)) v.x = rbf(s0 * S1[i] - (s0 * z0) * S0); v.y = rbf(s1 * S1[i + 1] - (s1 * z1) * S0); v.z = rbf(s2 * S1[i + 2] - (s2 * z2) * S0); v.w = rbf(s3 * S1[i + 3] - (s3 * z3) * S0); *(float4*)(OB + h * 128 + dg * 8 + i) = v; } } } gbar(bcnt, bgen, &lgen); // ---- M6: o_proj ---- { Work w = work_all(HK); gemv_row((const uint8_t*)pl[9], (const bf16*)pl[10], (const bf16*)pl[11], OB, pslot(Acur, AcurX, b & (PSL - 1)), HID, w.k0, w.k1, 1.0f); } gbar(bcnt, bgen, &lgen); } // ---- phase G: residual + moe norm + router ---- // the router row is loaded up front: its latency hides under prep bf16 rw[HID / TPB]; if (b < 64) { #pragma unroll for (int j = 0; j < HID / TPB; j++) rw[j] = ((const bf16*)pl[mo])[(size_t)b * HID + tid + j * TPB]; } prep(nullptr, Acur, AcurX, mnorm, 0, HB, XS, red); zfill(GU, NSLOT * 2 * MI, b, tid); // Clear the accumulators the coming layers will add into. Everything // cleared here has already been consumed (this layer's prep read the // previous moe partials, and the q/k/v/g/o buffers are dead once this // layer's attention is done), and phase G already ends in a grid // barrier, so this replaces a whole barrier per layer for free. if (L + 1 < NLAY) { zfill(AF + ((L + 1) & 1) * HID, HID, b, tid); zfill(AX + ((L + 1) & 1) * (PSL - 1) * HID, (PSL - 1) * HID, b, tid); } if (L >= 1) { zfill(YF + ((L - 1) & 1) * HID, HID, b, tid); zfill(YX + ((L - 1) & 1) * (PSL - 1) * HID, (PSL - 1) * HID, b, tid); } zfill(QF, HK, b, tid); zfill(KF, HK, b, tid); zfill(VF, HK, b, tid); zfill(GF, HK, b, tid); zfill(OB, HK, b, tid); if (b < 64) { float d = 0.f; #pragma unroll for (int j = 0; j < HID / TPB; j++) d += bf2f(rw[j]) * XS[tid + j * TPB]; d = block_reduce(d, red); if (tid == 0) LOGIT[b] = rbf(d); } gbar(bcnt, bgen, &lgen); // ---- phase H: topk + gate/up ---- // top-8 over 64 experts by rank counting: each lane counts how many // logits beat it (ties broken by lower index), so the expert with rank r // is the r-th largest. One broadcast sweep instead of eight butterfly // reductions -- the whole selection costs a couple hundred cycles. float* sl = MISC + 512; { if (tid < 64) sl[tid] = LOGIT[tid]; __syncthreads(); if (tid < 64) { float v = sl[tid]; int r = 0; #pragma unroll 4 for (int j = 0; j < 64; j++) { float u = sl[j]; r += (u > v || (u == v && j > tid)) ? 1 : 0; } if (r < 8) { sel[r] = tid; sl[64 + r] = v; } } } __syncthreads(); if (tid < 32) { float p = (tid < 8) ? __expf(sl[64 + tid] - sl[64]) : 0.f; float s8 = p + __shfl_down_sync(0xffffffffu, p, 4); s8 += __shfl_down_sync(0xffffffffu, s8, 2); s8 += __shfl_down_sync(0xffffffffu, s8, 1); s8 = __shfl_sync(0xffffffffu, s8, 0); if (tid < 8) wts[tid] = p / (s8 + 1e-9f) * 2.446f; if (tid == 8) { sel[8] = 0; wts[8] = 1.f; } } __syncthreads(); if (b < 186) { int tile = b / 62, sub = b % 62; int k0 = (int)(((long long)sub * HID) / 62) & ~1; int k1 = (sub == 61) ? HID : ((int)(((long long)(sub + 1) * HID) / 62) & ~1); // gate and up are repacked row-interleaved (2048 cols per row), so a // block streams one 2 KB row per slot with 16-byte loads. int slot = tile * 3 + (tid >> 7); int nch = (tid & 127) * 16; if (slot < NSLOT) { int e = sel[slot]; const uint8_t* w = (slot == 8) ? (const uint8_t*)pl[mo + 4] : (const uint8_t*)pl[mo + 1] + (size_t)e * 2359296; const bf16* sm = (slot == 8) ? (const bf16*)pl[mo + 5] : (const bf16*)pl[mo + 2] + (size_t)e * 36864; const bf16* zm = (slot == 8) ? (const bf16*)pl[mo + 6] : (const bf16*)pl[mo + 3] + (size_t)e * 36864; gemv_tile<16, BT>(w, sm, zm, XS, GU + slot * (2 * MI), 2 * MI, k0, k1, nch, 1.0f); } } gbar(bcnt, bgen, &lgen); // ---- phase I: down ---- { int cj[9]; for (int j = 0; j < 9; j++) cj[j] = 2304; Work w = assign_work_n(9, cj, MI, NBI); int slot = w.tile; const float* gsl = GU + slot * (2 * MI); for (int i = tid; i < MI; i += TPB) MISC[i] = silu_f(gsl[i]) * gsl[MI + i]; __syncthreads(); int e = sel[slot]; const uint8_t* wd = (slot == 8) ? (const uint8_t*)pl[mo + 16] : (const uint8_t*)pl[mo + 7] + (size_t)e * 1179648; const bf16* sd = (slot == 8) ? (const bf16*)pl[mo + 17] : (const bf16*)pl[mo + 8] + (size_t)e * 18432; const bf16* zd = (slot == 8) ? (const bf16*)pl[mo + 18] : (const bf16*)pl[mo + 9] + (size_t)e * 18432; gemv_row(wd, sd, zd, MISC, pslot(Ycur, YcurX, b & (PSL - 1)), HID, w.k0, w.k1, wts[slot]); } gbar(bcnt, bgen, &lgen); } // final hidden = HB + YF[HID..] if (b == 0) { const float* Yl = YF + ((NLAY - 1) & 1) * HID; const float* Ylx = YX + ((NLAY - 1) & 1) * (PSL - 1) * HID; for (int i = tid; i < HID; i += TPB) { float y = Yl[i]; #pragma unroll for (int q = 0; q < PSL - 1; q++) y += Ylx[q * HID + i]; hout[i] = f2bf(bf2f(HB[i]) + rbf(y)); } } } // ---------------------------------------------------------------- launcher static int g_smem_set = 0; extern "C" int mega_launch(const int64_t* plan, int pos, int fresh, const void* hin, void* hout, void* S0, void* S1, void* S2, void* cq0, void* ck0, void* cv0, void* cq1, void* ck1, void* cv1, void* cq2, void* ck2, void* cv2, const void* ckv_in, const void* ckr_in, void* ckv_buf, void* ckr_buf, void* scratch, void* bar, void* stream) { if (!g_smem_set) { cudaError_t e = cudaFuncSetAttribute((const void*)mega, cudaFuncAttributeMaxDynamicSharedMemorySize, SM_BYTES); if (e != cudaSuccess) return (int)e + 1000; g_smem_set = 1; } mega<<>>( plan, pos, fresh, (const bf16*)hin, (bf16*)hout, (float*)S0, (float*)S1, (float*)S2, (bf16*)cq0, (bf16*)ck0, (bf16*)cv0, (bf16*)cq1, (bf16*)ck1, (bf16*)cv1, (bf16*)cq2, (bf16*)ck2, (bf16*)cv2, (const bf16*)ckv_in, (const bf16*)ckr_in, (bf16*)ckv_buf, (bf16*)ckr_buf, (float*)scratch, (int*)bar); return (int)cudaGetLastError(); } extern "C" int last_err() { return (int)cudaGetLastError(); } // pin the always-read weight arena in L2 for the duration of the stream extern "C" int mega_setup(void* base, long long bytes, void* stream) { cudaError_t e = cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, (size_t)bytes); if (e != cudaSuccess) return 1; cudaStreamAttrValue av; memset(&av, 0, sizeof(av)); av.accessPolicyWindow.base_ptr = base; av.accessPolicyWindow.num_bytes = (size_t)bytes; av.accessPolicyWindow.hitRatio = 1.0f; av.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting; av.accessPolicyWindow.missProp = cudaAccessPropertyStreaming; e = cudaStreamSetAttribute((cudaStream_t)stream, cudaStreamAttributeAccessPolicyWindow, &av); return e == cudaSuccess ? 0 : 2; } '''