KernelBench cuda · RTX PRO 6000
MegaQwen Decode Gemini 4 Neon (XHigh)
7.55%geomean peak fraction across shapes
manually audited: clean
Gemini 4 Neon (XHigh) built a full-context SM120 MegaQwen decode engine combining CUDA Graphs with Programmatic Dependent Launch (griddepcontrol.wait / griddepcontrol.launch_dependents) across QKV, split-KV causal GQA, O-projection, SwiGLU MLP, and Down-projection kernels. Per-step PyTorch CPU MT19937 + Box-Muller decode noise is reproduced identically on GPU inside the timed CUDA Graph, and the entire KV cache stays in exact bf16 with zero quantization or eviction.
harnessagyagent session6h 51mtotal wall7h 4mcheck33sbenchmark12moutput tokens—gpu-lock wait1h 54mgpu-lock held1h 7mregimethroughput
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
No per-shape benchmark data archived for this run.
Kernel source (redacted)
"""Fast multi-layer Qwen3-0.6B-geometry decode engine for RTX PRO 6000 (SM120).
V11 SM120 Blackwell Direct-Pointer Single-Wave PDL Weight-Prefetch CUDA Graph Engine:
- Indirect DecodeParams CUDA Graph pre-captured with 80 MiB persisting L2 window baked into
every graph node and uploaded via cudaGraphUpload during untimed setup.
- Direct-by-value weight pointers baked into every captured kernel node (zero LayerWeights
indirection before weight prefetch).
- L1-bypassing weight prefetch (`ld.global.cg.v4.u32`) issued BEFORE `griddepcontrol.wait`
across all 4 projections: QKV (`256x512`), single-wave O_proj (`128x512`, 2 warps/row),
Gate+Up+SiLU (`192x512`), single-wave Down_proj (`128x512`, 2 warps/row) with preloaded
residuals/norms before `pdl_wait` and 1-sync Warp-0 epilogues.
- Precomputed per-step RoPE cos/sin table in prologue/aux branch (zero `sincosf` in attn,
1-sync warp-parallel Q/K RMSNorm + RoPE).
- In-warp register merge before SMEM in BOTH attention paths (2x smaller SMEM & 2x faster
block reduction in 128-bit `v4` half-warp; 4x smaller SMEM & single-pass block reduction
in 256-bit `v8` quarter-warp for >=16k contexts).
- Single-L2-RTT 64-block `reduce_pdl_kernel` overlapping `part_o` `ld.global.cg.v4` loads
with packed `float2` `part_ml` softmax scaling while triggering early `O_proj` PDL launch.
- Concurrent CUDA Graph fork/join for exact GPU MT19937 + parallel Box-Muller + RoPE table.
"""
from __future__ import annotations
from typing import List, Optional, Tuple
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
OP_TYPE = "megaqwen_decode"
SUPPORTED_PRECISIONS = ["bf16"]
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
HIDDEN = 1024
INTERMEDIATE = 3072
NUM_Q = 16
NUM_KV = 8
HEAD_DIM = 128
NUM_LAYERS = 4
EPS = 1e-6
_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <ATen/cuda/CUDAContext.h>
#include <torch/extension.h>
#include <cstdint>
#include <cmath>
#include <unordered_map>
#include <vector>
namespace {
constexpr int kHidden = 1024;
constexpr int kInter = 3072;
constexpr int kNumQ = 16;
constexpr int kNumKV = 8;
constexpr int kHeadDim = 128;
constexpr int kHalfHead = 64;
constexpr float kEps = 1e-6f;
constexpr float kAttnScaleLog2E = 0.12751742995591326f;
constexpr int kMaxChunk = 64;
constexpr int kSmemPadDim = kHeadDim + 4;
constexpr int kMaxLayers = 8;
__constant__ float c_rope_inv[kHalfHead];
struct u32x8 {
uint32_t r0, r1, r2, r3, r4, r5, r6, r7;
};
struct LayerWeights {
const __nv_bfloat16* qkv_w; // [4096, 1024]
const __nv_bfloat16* o_w; // [1024, 2048]
const __nv_bfloat16* gate_up_w; // [3072, 2, 1024]
const __nv_bfloat16* down_w; // [1024, 3072]
const __nv_bfloat16* input_ln; // [1024]
const __nv_bfloat16* q_norm; // [128]
const __nv_bfloat16* k_norm; // [128]
const __nv_bfloat16* post_ln; // [1024]
};
struct LayerKV {
__nv_bfloat16* k; // [8, max_seq, 128]
__nv_bfloat16* v; // [8, max_seq, 128]
};
struct DecodeParams {
LayerKV kvs[kMaxLayers];
const __nv_bfloat16* h_in;
__nv_bfloat16* h_out;
int64_t max_seq;
int start_pos;
int n_steps;
int init_mt;
uint64_t rng_seed;
};
struct Workspace {
__nv_bfloat16* x_curr; // [1024] exact bf16 residual1
__half* x_ln; // [1024] fp16 pre-mul x_curr * input_ln
float* h_mid; // [1024] exact fp32 residual2
__half* h_mid_ln; // [1024] fp16 pre-mul h_mid * post_ln
float* ss_x; // [1] (padded 128B)
float* ss_mid; // [1] (padded 128B)
float* q_raw; // [16, 128]
float* k_raw; // [8, 128]
__half* attn_out; // [16, 128] fp16
__half* mlp_act; // [3072] fp16
float* attn_part_o; // [16, 48, 128]
float2* attn_part_ml; // [16, 48] packed {m, l}
float* rope_cos; // [kMaxChunk, 64]
float* rope_sin; // [kMaxChunk, 64]
uint32_t* mt_state; // [624]
int* mt_cur; // [1]
uint32_t* raw_u32; // [kMaxChunk, 1024]
__nv_bfloat16* noise_buf; // [kMaxChunk, 1024]
DecodeParams* params; // [1]
};
// ---------------- Low-level SM120 PTX helpers ----------------
__device__ __forceinline__ float bf16_lo(uint32_t u) {
return __uint_as_float(u << 16);
}
__device__ __forceinline__ float bf16_hi(uint32_t u) {
return __uint_as_float(u & 0xFFFF0000u);
}
__device__ __forceinline__ float ex2_fast(float x) {
float r;
asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(r) : "f"(x));
return r;
}
__device__ __forceinline__ uint4 ld_cg_v4(const void* p) {
uint4 v;
asm volatile("ld.global.cg.v4.u32 {%0,%1,%2,%3}, [%4];"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(p));
return v;
}
__device__ __forceinline__ float4 ld_cg_f4(const void* p) {
float4 v;
asm volatile("ld.global.cg.v4.f32 {%0,%1,%2,%3}, [%4];"
: "=f"(v.x), "=f"(v.y), "=f"(v.z), "=f"(v.w)
: "l"(p));
return v;
}
__device__ __forceinline__ float2 ld_cg_f2(const void* p) {
float2 v;
asm volatile("ld.global.cg.v2.f32 {%0,%1}, [%2];"
: "=f"(v.x), "=f"(v.y)
: "l"(p));
return v;
}
__device__ __forceinline__ uint4 ld_kv_cs_v4(const void* p) {
uint4 v;
asm volatile("ld.global.cs.L2::256B.v4.u32 {%0,%1,%2,%3}, [%4];"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(p));
return v;
}
__device__ __forceinline__ u32x8 ld_kv_v8(const void* p) {
u32x8 v;
asm volatile("ld.global.L1::no_allocate.L2::evict_first.v8.b32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];"
: "=r"(v.r0), "=r"(v.r1), "=r"(v.r2), "=r"(v.r3),
"=r"(v.r4), "=r"(v.r5), "=r"(v.r6), "=r"(v.r7)
: "l"(p));
return v;
}
__device__ __forceinline__ void st_cg_f32(float* p, float v) {
asm volatile("st.global.cg.f32 [%0], %1;" :: "l"(p), "f"(v) : "memory");
}
__device__ __forceinline__ void st_cg_f2(float2* p, float2 v) {
asm volatile("st.global.cg.v2.f32 [%0], {%1,%2};" :: "l"(p), "f"(v.x), "f"(v.y) : "memory");
}
__device__ __forceinline__ void pdl_wait() {
asm volatile("griddepcontrol.wait;" ::: "memory");
}
__device__ __forceinline__ void pdl_launch() {
asm volatile("griddepcontrol.launch_dependents;" ::: "memory");
}
__device__ __forceinline__ float dot8_f16_bf16(const uint4 x, const uint4 w) {
float2 x0 = __half22float2(*reinterpret_cast<const __half2*>(&x.x));
float2 x1 = __half22float2(*reinterpret_cast<const __half2*>(&x.y));
float2 x2 = __half22float2(*reinterpret_cast<const __half2*>(&x.z));
float2 x3 = __half22float2(*reinterpret_cast<const __half2*>(&x.w));
float acc = 0.0f;
acc = fmaf(x0.x, bf16_lo(w.x), acc); acc = fmaf(x0.y, bf16_hi(w.x), acc);
acc = fmaf(x1.x, bf16_lo(w.y), acc); acc = fmaf(x1.y, bf16_hi(w.y), acc);
acc = fmaf(x2.x, bf16_lo(w.z), acc); acc = fmaf(x2.y, bf16_hi(w.z), acc);
acc = fmaf(x3.x, bf16_lo(w.w), acc); acc = fmaf(x3.y, bf16_hi(w.w), acc);
return acc;
}
__device__ __forceinline__ float dot8_f_u4(const float a[8], const uint4 w) {
float acc = 0.0f;
acc = fmaf(a[0], bf16_lo(w.x), acc); acc = fmaf(a[1], bf16_hi(w.x), acc);
acc = fmaf(a[2], bf16_lo(w.y), acc); acc = fmaf(a[3], bf16_hi(w.y), acc);
acc = fmaf(a[4], bf16_lo(w.z), acc); acc = fmaf(a[5], bf16_hi(w.z), acc);
acc = fmaf(a[6], bf16_lo(w.w), acc); acc = fmaf(a[7], bf16_hi(w.w), acc);
return acc;
}
__device__ __forceinline__ float dot16_f_u8(const float a[16], const u32x8 w) {
float acc = 0.0f;
acc = fmaf(a[0], bf16_lo(w.r0), acc); acc = fmaf(a[1], bf16_hi(w.r0), acc);
acc = fmaf(a[2], bf16_lo(w.r1), acc); acc = fmaf(a[3], bf16_hi(w.r1), acc);
acc = fmaf(a[4], bf16_lo(w.r2), acc); acc = fmaf(a[5], bf16_hi(w.r2), acc);
acc = fmaf(a[6], bf16_lo(w.r3), acc); acc = fmaf(a[7], bf16_hi(w.r3), acc);
acc = fmaf(a[8], bf16_lo(w.r4), acc); acc = fmaf(a[9], bf16_hi(w.r4), acc);
acc = fmaf(a[10], bf16_lo(w.r5), acc); acc = fmaf(a[11], bf16_hi(w.r5), acc);
acc = fmaf(a[12], bf16_lo(w.r6), acc); acc = fmaf(a[13], bf16_hi(w.r6), acc);
acc = fmaf(a[14], bf16_lo(w.r7), acc); acc = fmaf(a[15], bf16_hi(w.r7), acc);
return acc;
}
__device__ __forceinline__ void fma_v_u4(
float o0[8], float o1[8],
float a0, float a1,
float w0, float w1,
const uint4 v
) {
float lo, hi;
lo = bf16_lo(v.x); hi = bf16_hi(v.x);
o0[0] = fmaf(w0, lo, o0[0] * a0); o0[1] = fmaf(w0, hi, o0[1] * a0);
o1[0] = fmaf(w1, lo, o1[0] * a1); o1[1] = fmaf(w1, hi, o1[1] * a1);
lo = bf16_lo(v.y); hi = bf16_hi(v.y);
o0[2] = fmaf(w0, lo, o0[2] * a0); o0[3] = fmaf(w0, hi, o0[3] * a0);
o1[2] = fmaf(w1, lo, o1[2] * a1); o1[3] = fmaf(w1, hi, o1[3] * a1);
lo = bf16_lo(v.z); hi = bf16_hi(v.z);
o0[4] = fmaf(w0, lo, o0[4] * a0); o0[5] = fmaf(w0, hi, o0[5] * a0);
o1[4] = fmaf(w1, lo, o1[4] * a1); o1[5] = fmaf(w1, hi, o1[5] * a1);
lo = bf16_lo(v.w); hi = bf16_hi(v.w);
o0[6] = fmaf(w0, lo, o0[6] * a0); o0[7] = fmaf(w0, hi, o0[7] * a0);
o1[6] = fmaf(w1, lo, o1[6] * a1); o1[7] = fmaf(w1, hi, o1[7] * a1);
}
__device__ __forceinline__ void fma_v2_u4(
float o0[8], float o1[8],
float a0, float a1,
float w0A, float w0B,
float w1A, float w1B,
const uint4 vA, const uint4 vB
) {
float al, ah, bl, bh;
al = bf16_lo(vA.x); ah = bf16_hi(vA.x); bl = bf16_lo(vB.x); bh = bf16_hi(vB.x);
o0[0] = fmaf(w0B, bl, fmaf(w0A, al, o0[0] * a0));
o0[1] = fmaf(w0B, bh, fmaf(w0A, ah, o0[1] * a0));
o1[0] = fmaf(w1B, bl, fmaf(w1A, al, o1[0] * a1));
o1[1] = fmaf(w1B, bh, fmaf(w1A, ah, o1[1] * a1));
al = bf16_lo(vA.y); ah = bf16_hi(vA.y); bl = bf16_lo(vB.y); bh = bf16_hi(vB.y);
o0[2] = fmaf(w0B, bl, fmaf(w0A, al, o0[2] * a0));
o0[3] = fmaf(w0B, bh, fmaf(w0A, ah, o0[3] * a0));
o1[2] = fmaf(w1B, bl, fmaf(w1A, al, o1[2] * a1));
o1[3] = fmaf(w1B, bh, fmaf(w1A, ah, o1[3] * a1));
al = bf16_lo(vA.z); ah = bf16_hi(vA.z); bl = bf16_lo(vB.z); bh = bf16_hi(vB.z);
o0[4] = fmaf(w0B, bl, fmaf(w0A, al, o0[4] * a0));
o0[5] = fmaf(w0B, bh, fmaf(w0A, ah, o0[5] * a0));
o1[4] = fmaf(w1B, bl, fmaf(w1A, al, o1[4] * a1));
o1[5] = fmaf(w1B, bh, fmaf(w1A, ah, o1[5] * a1));
al = bf16_lo(vA.w); ah = bf16_hi(vA.w); bl = bf16_lo(vB.w); bh = bf16_hi(vB.w);
o0[6] = fmaf(w0B, bl, fmaf(w0A, al, o0[6] * a0));
o0[7] = fmaf(w0B, bh, fmaf(w0A, ah, o0[7] * a0));
o1[6] = fmaf(w1B, bl, fmaf(w1A, al, o1[6] * a1));
o1[7] = fmaf(w1B, bh, fmaf(w1A, ah, o1[7] * a1));
}
__device__ __forceinline__ void fma_v_u8(
float o0[16], float o1[16],
float a0, float a1,
float w0, float w1,
const u32x8 v
) {
const uint32_t arr[8] = {v.r0, v.r1, v.r2, v.r3, v.r4, v.r5, v.r6, v.r7};
#pragma unroll
for (int i = 0; i < 8; ++i) {
float lo = bf16_lo(arr[i]), hi = bf16_hi(arr[i]);
o0[2 * i] = fmaf(w0, lo, o0[2 * i] * a0);
o0[2 * i + 1] = fmaf(w0, hi, o0[2 * i + 1] * a0);
o1[2 * i] = fmaf(w1, lo, o1[2 * i] * a1);
o1[2 * i + 1] = fmaf(w1, hi, o1[2 * i + 1] * a1);
}
}
__device__ __forceinline__ void fma_v2_u8(
float o0[16], float o1[16],
float a0, float a1,
float w0A, float w0B,
float w1A, float w1B,
const u32x8 vA, const u32x8 vB
) {
const uint32_t aA[8] = {vA.r0, vA.r1, vA.r2, vA.r3, vA.r4, vA.r5, vA.r6, vA.r7};
const uint32_t aB[8] = {vB.r0, vB.r1, vB.r2, vB.r3, vB.r4, vB.r5, vB.r6, vB.r7};
#pragma unroll
for (int i = 0; i < 8; ++i) {
float al = bf16_lo(aA[i]), ah = bf16_hi(aA[i]);
float bl = bf16_lo(aB[i]), bh = bf16_hi(aB[i]);
o0[2 * i] = fmaf(w0B, bl, fmaf(w0A, al, o0[2 * i] * a0));
o0[2 * i + 1] = fmaf(w0B, bh, fmaf(w0A, ah, o0[2 * i + 1] * a0));
o1[2 * i] = fmaf(w1B, bl, fmaf(w1A, al, o1[2 * i] * a1));
o1[2 * i + 1] = fmaf(w1B, bh, fmaf(w1A, ah, o1[2 * i + 1] * a1));
}
}
__device__ __forceinline__ float warp_reduce_sum(float v) {
#pragma unroll
for (int m = 16; m > 0; m >>= 1) v += __shfl_xor_sync(0xffffffffu, v, m);
return v;
}
__device__ __forceinline__ float warp_reduce_max(float v) {
#pragma unroll
for (int m = 16; m > 0; m >>= 1) v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, m));
return v;
}
__device__ __forceinline__ float hw_reduce_sum(float v, unsigned mask) {
#pragma unroll
for (int off = 8; off > 0; off >>= 1) v += __shfl_xor_sync(mask, v, off, 16);
return v;
}
__device__ __forceinline__ float qw_reduce_sum(float v, unsigned mask) {
#pragma unroll
for (int off = 4; off > 0; off >>= 1) v += __shfl_xor_sync(mask, v, off, 8);
return v;
}
__device__ __forceinline__ __nv_bfloat16 mix_half_bf16(__nv_bfloat16 a, __nv_bfloat16 b) {
float ha = __bfloat162float(__float2bfloat16_rn(0.5f * __bfloat162float(a)));
float hb = __bfloat162float(__float2bfloat16_rn(0.5f * __bfloat162float(b)));
return __float2bfloat16_rn(ha + hb);
}
// ---------------- Exact PyTorch CPU MT19937 + Box-Muller + RoPE table ----------------
__device__ __forceinline__ uint32_t mt_twist_val(uint32_t u, uint32_t v) {
uint32_t m = (u & 0x80000000u) | (v & 0x7fffffffu);
return (m >> 1) ^ ((v & 1u) ? 0x9908b0dfu : 0u);
}
__device__ __forceinline__ uint32_t mt_temper(uint32_t y) {
y ^= (y >> 11);
y ^= (y << 7) & 0x9d2c5680u;
y ^= (y << 15) & 0xefc60000u;
y ^= (y >> 18);
return y;
}
__device__ __forceinline__ void mt_twist_block(uint32_t* s, int tid) {
uint32_t v1 = 0;
if (tid < 227) v1 = s[tid + 397] ^ mt_twist_val(s[tid], s[tid + 1]);
__syncthreads();
if (tid < 227) s[tid] = v1;
__syncthreads();
uint32_t v2 = 0;
if (tid < 227) {
int j = tid + 227;
v2 = s[tid] ^ mt_twist_val(s[j], s[j + 1]);
}
__syncthreads();
if (tid < 227) s[tid + 227] = v2;
__syncthreads();
uint32_t v3 = 0;
if (tid < 169) {
int j = tid + 454;
v3 = s[j - 227] ^ mt_twist_val(s[j], s[j + 1]);
}
__syncthreads();
if (tid < 169) s[tid + 454] = v3;
__syncthreads();
if (tid == 0) s[623] = s[396] ^ mt_twist_val(s[623], s[0]);
__syncthreads();
}
__device__ __forceinline__ void mt_fill_step1024(uint32_t* s, int& cur, uint32_t* sm_u, int tid) {
int dst = 0;
while (dst < kHidden) {
int avail = 624 - cur;
int rem = kHidden - dst;
int take = (rem < avail) ? rem : avail;
for (int i = tid; i < take; i += 256) {
sm_u[dst + i] = mt_temper(s[cur + i]);
}
dst += take;
cur += take;
__syncthreads();
if (cur == 624) {
mt_twist_block(s, tid);
cur = 0;
}
}
}
__global__ __launch_bounds__(256, 2)
void prologue_step0_kernel(
const __nv_bfloat16* __restrict__ iln0,
Workspace ws
) {
const int tid = threadIdx.x;
const int warp = tid >> 5;
const int lane = tid & 31;
__shared__ uint32_t mt_s[624];
__shared__ uint32_t sm_u[kHidden];
__shared__ float sm_ss[8];
const DecodeParams* dp = ws.params;
if (tid == 0) ws.ss_mid[0] = 0.0f;
if (tid < kHalfHead) {
float sn, cs;
sincosf((float)(dp->start_pos) * c_rope_inv[tid], &sn, &cs);
ws.rope_cos[tid] = cs;
ws.rope_sin[tid] = sn;
}
int cur;
if (dp->init_mt) {
if (tid == 0) {
mt_s[0] = (uint32_t)(dp->rng_seed & 0xffffffffu);
for (int j = 1; j < 624; ++j) {
mt_s[j] = 1812433253u * (mt_s[j - 1] ^ (mt_s[j - 1] >> 30)) + (uint32_t)j;
}
}
__syncthreads();
mt_twist_block(mt_s, tid);
cur = 0;
} else {
for (int j = tid; j < 624; j += 256) mt_s[j] = ws.mt_state[j];
cur = *ws.mt_cur;
__syncthreads();
}
mt_fill_step1024(mt_s, cur, sm_u, tid);
for (int j = tid; j < 624; j += 256) ws.mt_state[j] = mt_s[j];
if (tid == 0) *ws.mt_cur = cur;
constexpr uint32_t kMask = (1u << 24) - 1u;
constexpr float kDiv = 1.0f / (float)(1u << 24);
constexpr float kTwoPi = 6.28318530717958647692f;
const __nv_bfloat16* h_in = dp->h_in;
float local_ss = 0.0f;
#pragma unroll
for (int iter = 0; iter < 2; ++iter) {
int p = tid + iter * 256;
int grp = p >> 3;
int j = p & 7;
int idx0 = (grp << 4) + j;
int idx1 = idx0 + 8;
float u1 = 1.0f - (float)(sm_u[idx0] & kMask) * kDiv;
float u2 = (float)(sm_u[idx1] & kMask) * kDiv;
float radius = sqrtf(-2.0f * logf(u1));
float sn, cs;
sincosf(kTwoPi * u2, &sn, &cs);
__nv_bfloat16 n0 = __float2bfloat16_rn(radius * cs);
__nv_bfloat16 n1 = __float2bfloat16_rn(radius * sn);
__nv_bfloat16 x0 = mix_half_bf16(n0, h_in[idx0]);
__nv_bfloat16 x1 = mix_half_bf16(n1, h_in[idx1]);
ws.x_curr[idx0] = x0;
ws.x_curr[idx1] = x1;
float f0 = __bfloat162float(x0);
float f1 = __bfloat162float(x1);
ws.x_ln[idx0] = __float2half_rn(f0 * __bfloat162float(__ldg(&iln0[idx0])));
ws.x_ln[idx1] = __float2half_rn(f1 * __bfloat162float(__ldg(&iln0[idx1])));
local_ss += f0 * f0 + f1 * f1;
}
float ws_ss = warp_reduce_sum(local_ss);
if (lane == 0) sm_ss[warp] = ws_ss;
__syncthreads();
if (tid == 0) {
ws.ss_x[0] = sm_ss[0] + sm_ss[1] + sm_ss[2] + sm_ss[3]
+ sm_ss[4] + sm_ss[5] + sm_ss[6] + sm_ss[7];
}
pdl_launch();
}
__global__ __launch_bounds__(256, 2)
void mt_u32_rest_kernel(Workspace ws, int chunk_steps) {
const int tid = threadIdx.x;
__shared__ uint32_t mt_s[624];
for (int j = tid; j < 624; j += 256) mt_s[j] = ws.mt_state[j];
int cur = *ws.mt_cur;
__syncthreads();
int total = (chunk_steps - 1) * kHidden;
int dst = 0;
uint32_t* base_out = ws.raw_u32 + kHidden;
while (dst < total) {
int avail = 624 - cur;
int rem = total - dst;
int take = (rem < avail) ? rem : avail;
if (((cur | dst | take) & 3) == 0) {
int take4 = take >> 2;
uint4* out4 = reinterpret_cast<uint4*>(base_out + dst);
const uint4* in4 = reinterpret_cast<const uint4*>(mt_s + cur);
if (tid < take4) {
uint4 v = in4[tid];
out4[tid] = make_uint4(mt_temper(v.x), mt_temper(v.y), mt_temper(v.z), mt_temper(v.w));
}
} else {
for (int i = tid; i < take; i += 256) {
base_out[dst + i] = mt_temper(mt_s[cur + i]);
}
}
dst += take;
cur += take;
__syncthreads();
if (cur == 624) {
mt_twist_block(mt_s, tid);
cur = 0;
}
}
for (int j = tid; j < 624; j += 256) ws.mt_state[j] = mt_s[j];
if (tid == 0) *ws.mt_cur = cur;
}
__global__ __launch_bounds__(256, 2)
void bm_rope_rest_kernel(Workspace ws, int chunk_steps) {
constexpr uint32_t kMask = (1u << 24) - 1u;
constexpr float kDiv = 1.0f / (float)(1u << 24);
constexpr float kTwoPi = 6.28318530717958647692f;
const int gtid = blockIdx.x * 256 + threadIdx.x;
const int gstride = gridDim.x * 256;
int total_pairs = (chunk_steps - 1) * (kHidden / 2);
for (int p = gtid; p < total_pairs; p += gstride) {
int step = 1 + (p >> 9);
int p_in = p & 511;
int grp = p_in >> 3;
int j = p_in & 7;
int base = step * kHidden + (grp << 4);
float u1 = 1.0f - (float)(ws.raw_u32[base + j] & kMask) * kDiv;
float u2 = (float)(ws.raw_u32[base + j + 8] & kMask) * kDiv;
float radius = sqrtf(-2.0f * logf(u1));
float sn, cs;
sincosf(kTwoPi * u2, &sn, &cs);
ws.noise_buf[base + j] = __float2bfloat16_rn(radius * cs);
ws.noise_buf[base + j + 8] = __float2bfloat16_rn(radius * sn);
}
int total_rope = (chunk_steps - 1) * kHalfHead;
int start_pos = ws.params->start_pos;
for (int idx = gtid; idx < total_rope; idx += gstride) {
int step = 1 + (idx >> 6);
int d = idx & 63;
float sn, cs;
sincosf((float)(start_pos + step) * c_rope_inv[d], &sn, &cs);
ws.rope_cos[step * kHalfHead + d] = cs;
ws.rope_sin[step * kHalfHead + d] = sn;
}
}
// =========================================================================
// KERNEL 1: QKV Projection (256 x 512, direct weight pointer, 4-byte ss_x!)
// =========================================================================
__global__ __launch_bounds__(512, 2)
void qkv_pdl_kernel(
const __nv_bfloat16* __restrict__ qkv_w,
__half* x_ln,
float* ss_x,
float* q_raw,
float* k_raw,
DecodeParams* dp,
int layer_idx,
int step_idx
) {
const int tid = threadIdx.x;
const int warp = tid >> 5;
const int lane = tid & 31;
const int r = (blockIdx.x << 4) + warp;
const uint4* row = reinterpret_cast<const uint4*>(qkv_w + (size_t)r * kHidden);
uint4 w0 = ld_cg_v4(row + lane);
uint4 w1 = ld_cg_v4(row + 32 + lane);
uint4 w2 = ld_cg_v4(row + 64 + lane);
uint4 w3 = ld_cg_v4(row + 96 + lane);
pdl_wait();
pdl_launch();
const uint4* x4 = reinterpret_cast<const uint4*>(x_ln);
float inv_rms = rsqrtf(ss_x[0] * (1.0f / (float)kHidden) + kEps);
float acc = dot8_f16_bf16(x4[lane], w0)
+ dot8_f16_bf16(x4[32 + lane], w1)
+ dot8_f16_bf16(x4[64 + lane], w2)
+ dot8_f16_bf16(x4[96 + lane], w3);
acc = warp_reduce_sum(acc) * inv_rms;
if (lane == 0) {
if (r < 2048) {
q_raw[r] = acc;
} else if (r < 3072) {
k_raw[r - 2048] = acc;
} else {
int v_idx = r - 3072;
int pos = dp->start_pos + step_idx;
__nv_bfloat16* v_cache = dp->kvs[layer_idx].v;
v_cache[((int64_t)(v_idx >> 7) * dp->max_seq + pos) * kHeadDim + (v_idx & 127)] =
__float2bfloat16_rn(acc);
}
}
}
// =========================================================================
// KERNEL 2a: Short/Medium Context Attention (128-bit half-warp + in-warp merge!)
// Halves SMEM (10 KB) and halves block reduction (8 warps instead of 16 half-warps).
// =========================================================================
template<int SPLITS, bool USE_CS, bool UNROLL2X>
__global__ __launch_bounds__(256, 3)
void attn_pdl_v4_kernel(
const __nv_bfloat16* __restrict__ q_norm,
const __nv_bfloat16* __restrict__ k_norm,
float* q_raw,
float* k_raw,
float* rope_cos,
float* rope_sin,
float* part_o,
float2* part_ml,
DecodeParams* dp,
int layer_idx,
int step_idx
) {
const int bid = blockIdx.x;
const int tid = threadIdx.x;
const int warp = tid >> 5;
const int lane = tid & 31;
const int hw = tid >> 4;
const int lane16 = tid & 15;
const unsigned hw_mask = (tid & 16) ? 0xffff0000u : 0x0000ffffu;
const int kv_h = bid / SPLITS;
const int split_id = bid % SPLITS;
__shared__ float sm_q0[kHeadDim];
__shared__ float sm_q1[kHeadDim];
__shared__ float sm_k_last[kHeadDim];
__shared__ float sm_o0[8][kSmemPadDim]; // 2x smaller thanks to in-warp merge!
__shared__ float sm_o1[8][kSmemPadDim];
__shared__ float sm_m0[8], sm_l0[8], sm_m1[8], sm_l1[8];
__shared__ float sm_sc0[8], sm_sc1[8], sm_ml[4];
const int pos = dp->start_pos + step_idx;
const int seq_len = pos + 1;
const int active_splits = min(SPLITS, max(1, (seq_len + 15) / 16));
if (split_id >= active_splits) {
pdl_wait();
pdl_launch();
return;
}
const int64_t max_seq = dp->max_seq;
const LayerKV kv = dp->kvs[layer_idx];
const bool is_last_split = (split_id == active_splits - 1);
float rc0 = 0.0f, rs0 = 0.0f, rc1 = 0.0f, rs1 = 0.0f;
float nw0 = 0.0f, nw1 = 0.0f, nw2 = 0.0f, nw3 = 0.0f;
if (warp < 2 || (warp == 2 && is_last_split)) {
const float* rcp = rope_cos + step_idx * kHalfHead;
const float* rsp = rope_sin + step_idx * kHalfHead;
rc0 = rcp[lane]; rs0 = rsp[lane];
rc1 = rcp[lane + 32]; rs1 = rsp[lane + 32];
const __nv_bfloat16* nw = (warp < 2) ? q_norm : k_norm;
nw0 = __bfloat162float(__ldg(nw + lane));
nw1 = __bfloat162float(__ldg(nw + lane + 32));
nw2 = __bfloat162float(__ldg(nw + lane + 64));
nw3 = __bfloat162float(__ldg(nw + lane + 96));
}
const int t_start = (int)(((int64_t)seq_len * split_id) / active_splits);
const int t_end = (int)(((int64_t)seq_len * (split_id + 1)) / active_splits);
const int t_loop_end = (t_end == seq_len) ? (seq_len - 1) : t_end;
const __nv_bfloat16* kh = kv.k + (int64_t)kv_h * max_seq * kHeadDim;
const __nv_bfloat16* vh = kv.v + (int64_t)kv_h * max_seq * kHeadDim;
pdl_wait();
if (warp < 2) {
const float* qp = q_raw + (2 * kv_h + warp) * kHeadDim;
float v0 = qp[lane], v1 = qp[lane + 32], v2 = qp[lane + 64], v3 = qp[lane + 96];
float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3);
float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps);
v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3;
float* dst = (warp == 0) ? sm_q0 : sm_q1;
dst[lane] = v0 * rc0 - v2 * rs0;
dst[lane + 64] = v0 * rs0 + v2 * rc0;
dst[lane + 32] = v1 * rc1 - v3 * rs1;
dst[lane + 96] = v1 * rs1 + v3 * rc1;
} else if (warp == 2 && is_last_split) {
const float* kp = k_raw + kv_h * kHeadDim;
float v0 = kp[lane], v1 = kp[lane + 32], v2 = kp[lane + 64], v3 = kp[lane + 96];
float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3);
float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps);
v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3;
__nv_bfloat16 k0 = __float2bfloat16_rn(v0 * rc0 - v2 * rs0);
__nv_bfloat16 k2 = __float2bfloat16_rn(v0 * rs0 + v2 * rc0);
__nv_bfloat16 k1 = __float2bfloat16_rn(v1 * rc1 - v3 * rs1);
__nv_bfloat16 k3 = __float2bfloat16_rn(v1 * rs1 + v3 * rc1);
__nv_bfloat16* k_dst = kv.k + ((int64_t)kv_h * max_seq + pos) * kHeadDim;
k_dst[lane] = k0;
k_dst[lane + 32] = k1;
k_dst[lane + 64] = k2;
k_dst[lane + 96] = k3;
sm_k_last[lane] = __bfloat162float(k0);
sm_k_last[lane + 32] = __bfloat162float(k1);
sm_k_last[lane + 64] = __bfloat162float(k2);
sm_k_last[lane + 96] = __bfloat162float(k3);
}
__syncthreads();
const int d8 = lane16 * 8;
float q0[8], q1[8], o0[8], o1[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
q0[i] = sm_q0[d8 + i];
q1[i] = sm_q1[d8 + i];
o0[i] = 0.0f;
o1[i] = 0.0f;
}
float m0 = -1e30f, l0 = 0.0f;
float m1 = -1e30f, l1 = 0.0f;
int t = t_start + hw;
if constexpr (USE_CS) {
if constexpr (UNROLL2X) {
for (; t + 16 < t_loop_end; t += 32) {
uint4 kA = ld_kv_cs_v4(kh + (int64_t)t * kHeadDim + d8);
uint4 vA = ld_kv_cs_v4(vh + (int64_t)t * kHeadDim + d8);
uint4 kB = ld_kv_cs_v4(kh + (int64_t)(t + 16) * kHeadDim + d8);
uint4 vB = ld_kv_cs_v4(vh + (int64_t)(t + 16) * kHeadDim + d8);
float s0A = hw_reduce_sum(dot8_f_u4(q0, kA), hw_mask) * kAttnScaleLog2E;
float s1A = hw_reduce_sum(dot8_f_u4(q1, kA), hw_mask) * kAttnScaleLog2E;
float s0B = hw_reduce_sum(dot8_f_u4(q0, kB), hw_mask) * kAttnScaleLog2E;
float s1B = hw_reduce_sum(dot8_f_u4(q1, kB), hw_mask) * kAttnScaleLog2E;
float m0n = fmaxf(m0, fmaxf(s0A, s0B));
float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n), w0B = ex2_fast(s0B - m0n);
l0 = l0 * a0 + w0A + w0B; m0 = m0n;
float m1n = fmaxf(m1, fmaxf(s1A, s1B));
float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n), w1B = ex2_fast(s1B - m1n);
l1 = l1 * a1 + w1A + w1B; m1 = m1n;
fma_v2_u4(o0, o1, a0, a1, w0A, w0B, w1A, w1B, vA, vB);
}
}
for (; t < t_loop_end; t += 16) {
uint4 kA = ld_kv_cs_v4(kh + (int64_t)t * kHeadDim + d8);
uint4 vA = ld_kv_cs_v4(vh + (int64_t)t * kHeadDim + d8);
float s0A = hw_reduce_sum(dot8_f_u4(q0, kA), hw_mask) * kAttnScaleLog2E;
float s1A = hw_reduce_sum(dot8_f_u4(q1, kA), hw_mask) * kAttnScaleLog2E;
float m0n = fmaxf(m0, s0A);
float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n);
l0 = l0 * a0 + w0A; m0 = m0n;
float m1n = fmaxf(m1, s1A);
float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n);
l1 = l1 * a1 + w1A; m1 = m1n;
fma_v_u4(o0, o1, a0, a1, w0A, w1A, vA);
}
} else {
for (; t < t_loop_end; t += 16) {
uint4 kA = *reinterpret_cast<const uint4*>(kh + (int64_t)t * kHeadDim + d8);
uint4 vA = *reinterpret_cast<const uint4*>(vh + (int64_t)t * kHeadDim + d8);
float s0A = hw_reduce_sum(dot8_f_u4(q0, kA), hw_mask) * kAttnScaleLog2E;
float s1A = hw_reduce_sum(dot8_f_u4(q1, kA), hw_mask) * kAttnScaleLog2E;
float m0n = fmaxf(m0, s0A);
float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n);
l0 = l0 * a0 + w0A; m0 = m0n;
float m1n = fmaxf(m1, s1A);
float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n);
l1 = l1 * a1 + w1A; m1 = m1n;
fma_v_u4(o0, o1, a0, a1, w0A, w1A, vA);
}
}
if (t_end == seq_len && hw == 0) {
float d0 = 0.0f, d1 = 0.0f;
#pragma unroll
for (int i = 0; i < 8; ++i) {
float kf = sm_k_last[d8 + i];
d0 = fmaf(q0[i], kf, d0);
d1 = fmaf(q1[i], kf, d1);
}
float s0A = hw_reduce_sum(d0, hw_mask) * kAttnScaleLog2E;
float s1A = hw_reduce_sum(d1, hw_mask) * kAttnScaleLog2E;
uint4 vA = *reinterpret_cast<const uint4*>(vh + (int64_t)pos * kHeadDim + d8);
float m0n = fmaxf(m0, s0A);
float a0 = (l0 == 0.0f) ? 0.0f : ex2_fast(m0 - m0n);
float w0A = ex2_fast(s0A - m0n);
l0 = l0 * a0 + w0A; m0 = m0n;
float m1n = fmaxf(m1, s1A);
float a1 = (l1 == 0.0f) ? 0.0f : ex2_fast(m1 - m1n);
float w1A = ex2_fast(s1A - m1n);
l1 = l1 * a1 + w1A; m1 = m1n;
fma_v_u4(o0, o1, a0, a1, w0A, w1A, vA);
}
pdl_launch();
// In-warp merge of the 2 sibling half-warps IN REGISTERS before touching SMEM!
{
float m0_ot = __shfl_xor_sync(0xffffffffu, m0, 16);
float l0_ot = __shfl_xor_sync(0xffffffffu, l0, 16);
float m0_w = fmaxf((l0 > 0.0f) ? m0 : -1e30f, (l0_ot > 0.0f) ? m0_ot : -1e30f);
float s0_me = (l0 > 0.0f) ? ex2_fast(m0 - m0_w) : 0.0f;
float s0_ot = (l0_ot > 0.0f) ? ex2_fast(m0_ot - m0_w) : 0.0f;
float l0_w = l0 * s0_me + l0_ot * s0_ot;
float m1_ot = __shfl_xor_sync(0xffffffffu, m1, 16);
float l1_ot = __shfl_xor_sync(0xffffffffu, l1, 16);
float m1_w = fmaxf((l1 > 0.0f) ? m1 : -1e30f, (l1_ot > 0.0f) ? m1_ot : -1e30f);
float s1_me = (l1 > 0.0f) ? ex2_fast(m1 - m1_w) : 0.0f;
float s1_ot = (l1_ot > 0.0f) ? ex2_fast(m1_ot - m1_w) : 0.0f;
float l1_w = l1 * s1_me + l1_ot * s1_ot;
#pragma unroll
for (int i = 0; i < 8; ++i) {
float ot0 = __shfl_xor_sync(0xffffffffu, o0[i], 16);
float ot1 = __shfl_xor_sync(0xffffffffu, o1[i], 16);
o0[i] = fmaf(s0_ot, ot0, o0[i] * s0_me);
o1[i] = fmaf(s1_ot, ot1, o1[i] * s1_me);
}
if (lane < 16) {
*reinterpret_cast<float4*>(&sm_o0[warp][d8]) = make_float4(o0[0], o0[1], o0[2], o0[3]);
*reinterpret_cast<float4*>(&sm_o0[warp][d8 + 4]) = make_float4(o0[4], o0[5], o0[6], o0[7]);
*reinterpret_cast<float4*>(&sm_o1[warp][d8]) = make_float4(o1[0], o1[1], o1[2], o1[3]);
*reinterpret_cast<float4*>(&sm_o1[warp][d8 + 4]) = make_float4(o1[4], o1[5], o1[6], o1[7]);
if (lane == 0) {
sm_m0[warp] = m0_w; sm_l0[warp] = l0_w;
sm_m1[warp] = m1_w; sm_l1[warp] = l1_w;
}
}
}
__syncthreads();
if (warp < 2) {
const float* ms = (warp == 0) ? sm_m0 : sm_m1;
const float* ls = (warp == 0) ? sm_l0 : sm_l1;
float lh = (lane < 8) ? ls[lane] : 0.0f;
float mh = (lane < 8 && lh > 0.0f) ? ms[lane] : -1e30f;
float mm = mh;
#pragma unroll
for (int off = 4; off > 0; off >>= 1) mm = fmaxf(mm, __shfl_xor_sync(0xffffffffu, mm, off, 8));
float w = (lh > 0.0f) ? ex2_fast(mh - mm) : 0.0f;
float lt = lh * w;
#pragma unroll
for (int off = 4; off > 0; off >>= 1) lt += __shfl_xor_sync(0xffffffffu, lt, off, 8);
if (lane < 8) {
if (warp == 0) sm_sc0[lane] = w;
else sm_sc1[lane] = w;
}
if (lane == 0) {
sm_ml[warp * 2 + 0] = mm;
sm_ml[warp * 2 + 1] = lt;
}
}
__syncthreads();
const int qsub = tid >> 7;
const int d = tid & 127;
const float* sc = (qsub == 0) ? sm_sc0 : sm_sc1;
float o_acc = 0.0f;
if (qsub == 0) {
#pragma unroll
for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o0[w][d], o_acc);
} else {
#pragma unroll
for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o1[w][d], o_acc);
}
const int qh = kv_h * 2 + qsub;
st_cg_f32(&part_o[(qh * 48 + split_id) * kHeadDim + d], o_acc);
if (d == 0) {
st_cg_f2(&part_ml[qh * 48 + split_id], make_float2(sm_ml[qsub * 2 + 0], sm_ml[qsub * 2 + 1]));
}
}
// =========================================================================
// KERNEL 2b: Long-Context Attention (SM120 256-bit quarter-warp + 4-way in-warp merge!)
// Single-pass 8-warp SMEM reduction (10 KB total SMEM, 4x fewer SMEM reads)!
// =========================================================================
template<int SPLITS>
__global__ __launch_bounds__(256, 2)
void attn_pdl_v8_kernel(
const __nv_bfloat16* __restrict__ q_norm,
const __nv_bfloat16* __restrict__ k_norm,
float* q_raw,
float* k_raw,
float* rope_cos,
float* rope_sin,
float* part_o,
float2* part_ml,
DecodeParams* dp,
int layer_idx,
int step_idx
) {
const int bid = blockIdx.x;
const int tid = threadIdx.x;
const int warp = tid >> 5;
const int lane = tid & 31;
const int qw = tid >> 3;
const int l8 = tid & 7;
const unsigned qw_mask = 0xffu << (tid & 24);
const int kv_h = bid / SPLITS;
const int split_id = bid % SPLITS;
__shared__ float sm_q0[kHeadDim];
__shared__ float sm_q1[kHeadDim];
__shared__ float sm_k_last[kHeadDim];
__shared__ float sm_o0[8][kSmemPadDim];
__shared__ float sm_o1[8][kSmemPadDim];
__shared__ float sm_m0[8], sm_l0[8], sm_m1[8], sm_l1[8];
__shared__ float sm_sc0[8], sm_sc1[8], sm_ml[4];
const int pos = dp->start_pos + step_idx;
const int seq_len = pos + 1;
const int active_splits = min(SPLITS, max(1, (seq_len + 31) / 32));
if (split_id >= active_splits) {
pdl_wait();
pdl_launch();
return;
}
const int64_t max_seq = dp->max_seq;
const LayerKV kv = dp->kvs[layer_idx];
const bool is_last_split = (split_id == active_splits - 1);
float rc0 = 0.0f, rs0 = 0.0f, rc1 = 0.0f, rs1 = 0.0f;
float nw0 = 0.0f, nw1 = 0.0f, nw2 = 0.0f, nw3 = 0.0f;
if (warp < 2 || (warp == 2 && is_last_split)) {
const float* rcp = rope_cos + step_idx * kHalfHead;
const float* rsp = rope_sin + step_idx * kHalfHead;
rc0 = rcp[lane]; rs0 = rsp[lane];
rc1 = rcp[lane + 32]; rs1 = rsp[lane + 32];
const __nv_bfloat16* nw = (warp < 2) ? q_norm : k_norm;
nw0 = __bfloat162float(__ldg(nw + lane));
nw1 = __bfloat162float(__ldg(nw + lane + 32));
nw2 = __bfloat162float(__ldg(nw + lane + 64));
nw3 = __bfloat162float(__ldg(nw + lane + 96));
}
const int t_start = (int)(((int64_t)seq_len * split_id) / active_splits);
const int t_end = (int)(((int64_t)seq_len * (split_id + 1)) / active_splits);
const int t_loop_end = (t_end == seq_len) ? (seq_len - 1) : t_end;
const __nv_bfloat16* kh = kv.k + (int64_t)kv_h * max_seq * kHeadDim;
const __nv_bfloat16* vh = kv.v + (int64_t)kv_h * max_seq * kHeadDim;
pdl_wait();
if (warp < 2) {
const float* qp = q_raw + (2 * kv_h + warp) * kHeadDim;
float v0 = qp[lane], v1 = qp[lane + 32], v2 = qp[lane + 64], v3 = qp[lane + 96];
float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3);
float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps);
v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3;
float* dst = (warp == 0) ? sm_q0 : sm_q1;
dst[lane] = v0 * rc0 - v2 * rs0;
dst[lane + 64] = v0 * rs0 + v2 * rc0;
dst[lane + 32] = v1 * rc1 - v3 * rs1;
dst[lane + 96] = v1 * rs1 + v3 * rc1;
} else if (warp == 2 && is_last_split) {
const float* kp = k_raw + kv_h * kHeadDim;
float v0 = kp[lane], v1 = kp[lane + 32], v2 = kp[lane + 64], v3 = kp[lane + 96];
float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3);
float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps);
v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3;
__nv_bfloat16 k0 = __float2bfloat16_rn(v0 * rc0 - v2 * rs0);
__nv_bfloat16 k2 = __float2bfloat16_rn(v0 * rs0 + v2 * rc0);
__nv_bfloat16 k1 = __float2bfloat16_rn(v1 * rc1 - v3 * rs1);
__nv_bfloat16 k3 = __float2bfloat16_rn(v1 * rs1 + v3 * rc1);
__nv_bfloat16* k_dst = kv.k + ((int64_t)kv_h * max_seq + pos) * kHeadDim;
k_dst[lane] = k0;
k_dst[lane + 32] = k1;
k_dst[lane + 64] = k2;
k_dst[lane + 96] = k3;
sm_k_last[lane] = __bfloat162float(k0);
sm_k_last[lane + 32] = __bfloat162float(k1);
sm_k_last[lane + 64] = __bfloat162float(k2);
sm_k_last[lane + 96] = __bfloat162float(k3);
}
__syncthreads();
const int d16 = l8 * 16;
float q0[16], o0[16], o1[16];
#pragma unroll
for (int i = 0; i < 16; ++i) {
q0[i] = sm_q0[d16 + i];
o0[i] = 0.0f;
o1[i] = 0.0f;
}
const float* q1 = sm_q1 + d16;
float m0 = -1e30f, l0 = 0.0f;
float m1 = -1e30f, l1 = 0.0f;
int t = t_start + qw;
for (; t + 32 < t_loop_end; t += 64) {
u32x8 kA = ld_kv_v8(kh + (int64_t)t * kHeadDim + d16);
u32x8 vA = ld_kv_v8(vh + (int64_t)t * kHeadDim + d16);
u32x8 kB = ld_kv_v8(kh + (int64_t)(t + 32) * kHeadDim + d16);
u32x8 vB = ld_kv_v8(vh + (int64_t)(t + 32) * kHeadDim + d16);
float s0A = qw_reduce_sum(dot16_f_u8(q0, kA), qw_mask) * kAttnScaleLog2E;
float s1A = qw_reduce_sum(dot16_f_u8(q1, kA), qw_mask) * kAttnScaleLog2E;
float s0B = qw_reduce_sum(dot16_f_u8(q0, kB), qw_mask) * kAttnScaleLog2E;
float s1B = qw_reduce_sum(dot16_f_u8(q1, kB), qw_mask) * kAttnScaleLog2E;
float m0n = fmaxf(m0, fmaxf(s0A, s0B));
float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n), w0B = ex2_fast(s0B - m0n);
l0 = l0 * a0 + w0A + w0B; m0 = m0n;
float m1n = fmaxf(m1, fmaxf(s1A, s1B));
float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n), w1B = ex2_fast(s1B - m1n);
l1 = l1 * a1 + w1A + w1B; m1 = m1n;
fma_v2_u8(o0, o1, a0, a1, w0A, w0B, w1A, w1B, vA, vB);
}
for (; t < t_loop_end; t += 32) {
u32x8 kA = ld_kv_v8(kh + (int64_t)t * kHeadDim + d16);
u32x8 vA = ld_kv_v8(vh + (int64_t)t * kHeadDim + d16);
float s0A = qw_reduce_sum(dot16_f_u8(q0, kA), qw_mask) * kAttnScaleLog2E;
float s1A = qw_reduce_sum(dot16_f_u8(q1, kA), qw_mask) * kAttnScaleLog2E;
float m0n = fmaxf(m0, s0A);
float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n);
l0 = l0 * a0 + w0A; m0 = m0n;
float m1n = fmaxf(m1, s1A);
float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n);
l1 = l1 * a1 + w1A; m1 = m1n;
fma_v_u8(o0, o1, a0, a1, w0A, w1A, vA);
}
if (t_end == seq_len && qw == 0) {
float d0 = 0.0f, d1 = 0.0f;
#pragma unroll
for (int i = 0; i < 16; ++i) {
float kf = sm_k_last[d16 + i];
d0 = fmaf(q0[i], kf, d0);
d1 = fmaf(q1[i], kf, d1);
}
float s0A = qw_reduce_sum(d0, qw_mask) * kAttnScaleLog2E;
float s1A = qw_reduce_sum(d1, qw_mask) * kAttnScaleLog2E;
const uint4* vp = reinterpret_cast<const uint4*>(vh + (int64_t)pos * kHeadDim + d16);
uint4 v_lo = vp[0], v_hi = vp[1];
u32x8 vA{v_lo.x, v_lo.y, v_lo.z, v_lo.w, v_hi.x, v_hi.y, v_hi.z, v_hi.w};
float m0n = fmaxf(m0, s0A);
float a0 = (l0 == 0.0f) ? 0.0f : ex2_fast(m0 - m0n);
float w0A = ex2_fast(s0A - m0n);
l0 = l0 * a0 + w0A; m0 = m0n;
float m1n = fmaxf(m1, s1A);
float a1 = (l1 == 0.0f) ? 0.0f : ex2_fast(m1 - m1n);
float w1A = ex2_fast(s1A - m1n);
l1 = l1 * a1 + w1A; m1 = m1n;
fma_v_u8(o0, o1, a0, a1, w0A, w1A, vA);
}
pdl_launch();
// 4-way in-warp quarter-warp merge (stride 8 then stride 16) in registers!
#pragma unroll
for (int stride = 8; stride <= 16; stride <<= 1) {
float m0_ot = __shfl_xor_sync(0xffffffffu, m0, stride);
float l0_ot = __shfl_xor_sync(0xffffffffu, l0, stride);
float m0_w = fmaxf((l0 > 0.0f) ? m0 : -1e30f, (l0_ot > 0.0f) ? m0_ot : -1e30f);
float s0_me = (l0 > 0.0f) ? ex2_fast(m0 - m0_w) : 0.0f;
float s0_ot = (l0_ot > 0.0f) ? ex2_fast(m0_ot - m0_w) : 0.0f;
l0 = l0 * s0_me + l0_ot * s0_ot;
m0 = m0_w;
float m1_ot = __shfl_xor_sync(0xffffffffu, m1, stride);
float l1_ot = __shfl_xor_sync(0xffffffffu, l1, stride);
float m1_w = fmaxf((l1 > 0.0f) ? m1 : -1e30f, (l1_ot > 0.0f) ? m1_ot : -1e30f);
float s1_me = (l1 > 0.0f) ? ex2_fast(m1 - m1_w) : 0.0f;
float s1_ot = (l1_ot > 0.0f) ? ex2_fast(m1_ot - m1_w) : 0.0f;
l1 = l1 * s1_me + l1_ot * s1_ot;
m1 = m1_w;
#pragma unroll
for (int i = 0; i < 16; ++i) {
float ot0 = __shfl_xor_sync(0xffffffffu, o0[i], stride);
float ot1 = __shfl_xor_sync(0xffffffffu, o1[i], stride);
o0[i] = fmaf(s0_ot, ot0, o0[i] * s0_me);
o1[i] = fmaf(s1_ot, ot1, o1[i] * s1_me);
}
}
if (lane < 8) {
*reinterpret_cast<float4*>(&sm_o0[warp][d16 + 0]) = make_float4(o0[0], o0[1], o0[2], o0[3]);
*reinterpret_cast<float4*>(&sm_o0[warp][d16 + 4]) = make_float4(o0[4], o0[5], o0[6], o0[7]);
*reinterpret_cast<float4*>(&sm_o0[warp][d16 + 8]) = make_float4(o0[8], o0[9], o0[10], o0[11]);
*reinterpret_cast<float4*>(&sm_o0[warp][d16 + 12]) = make_float4(o0[12], o0[13], o0[14], o0[15]);
*reinterpret_cast<float4*>(&sm_o1[warp][d16 + 0]) = make_float4(o1[0], o1[1], o1[2], o1[3]);
*reinterpret_cast<float4*>(&sm_o1[warp][d16 + 4]) = make_float4(o1[4], o1[5], o1[6], o1[7]);
*reinterpret_cast<float4*>(&sm_o1[warp][d16 + 8]) = make_float4(o1[8], o1[9], o1[10], o1[11]);
*reinterpret_cast<float4*>(&sm_o1[warp][d16 + 12]) = make_float4(o1[12], o1[13], o1[14], o1[15]);
if (lane == 0) {
sm_m0[warp] = m0; sm_l0[warp] = l0;
sm_m1[warp] = m1; sm_l1[warp] = l1;
}
}
__syncthreads();
if (warp < 2) {
const float* ms = (warp == 0) ? sm_m0 : sm_m1;
const float* ls = (warp == 0) ? sm_l0 : sm_l1;
float lh = (lane < 8) ? ls[lane] : 0.0f;
float mh = (lane < 8 && lh > 0.0f) ? ms[lane] : -1e30f;
float mm = mh;
#pragma unroll
for (int off = 4; off > 0; off >>= 1) mm = fmaxf(mm, __shfl_xor_sync(0xffffffffu, mm, off, 8));
float w = (lh > 0.0f) ? ex2_fast(mh - mm) : 0.0f;
float lt = lh * w;
#pragma unroll
for (int off = 4; off > 0; off >>= 1) lt += __shfl_xor_sync(0xffffffffu, lt, off, 8);
if (lane < 8) {
if (warp == 0) sm_sc0[lane] = w;
else sm_sc1[lane] = w;
}
if (lane == 0) {
sm_ml[warp * 2 + 0] = mm;
sm_ml[warp * 2 + 1] = lt;
}
}
__syncthreads();
const int qsub = tid >> 7;
const int d = tid & 127;
const float* sc = (qsub == 0) ? sm_sc0 : sm_sc1;
float o_acc = 0.0f;
if (qsub == 0) {
#pragma unroll
for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o0[w][d], o_acc);
} else {
#pragma unroll
for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o1[w][d], o_acc);
}
const int qh = kv_h * 2 + qsub;
st_cg_f32(&part_o[(qh * 48 + split_id) * kHeadDim + d], o_acc);
if (d == 0) {
st_cg_f2(&part_ml[qh * 48 + split_id], make_float2(sm_ml[qsub * 2 + 0], sm_ml[qsub * 2 + 1]));
}
}
// =========================================================================
// KERNEL 2c: Single-L2-RTT 32-Block Split Reduction (2 blks/Q-head x 512 thr)
// 32 reduce blocks + 128 o_proj blocks = 160 <= 188 SMs (zero SM contention)!
// Issues part_o ld_cg_f4 CONCURRENTLY with Warp-0 part_ml softmax scaling!
// =========================================================================
template<int SPLITS, int STRIDE_TOK>
__global__ __launch_bounds__(512, 2)
void reduce_pdl_kernel(
float* part_o,
float2* part_ml,
__half* attn_out,
DecodeParams* dp,
int step_idx
) {
const int qh = blockIdx.x >> 1;
const int part = blockIdx.x & 1;
const int tid = threadIdx.x;
const int warp = tid >> 5;
const int lane = tid & 31;
const int hw = tid >> 4; // 0..31 half-warps
const int l16 = tid & 15; // 0..15 float4 lanes (64 dims/blk)
const int col4 = part * 16 + l16;
__shared__ float sm_sc[64];
__shared__ float sm_f[2048];
const int seq_len = dp->start_pos + step_idx + 1;
const int active_splits = min(SPLITS, max(1, (seq_len + STRIDE_TOK - 1) / STRIDE_TOK));
pdl_launch();
pdl_wait();
// Overlap part_o loads with Warp-0 part_ml loads -> ONE L2 round-trip instead of TWO!
const float4* ob4 = reinterpret_cast<const float4*>(part_o + (size_t)qh * 48 * kHeadDim);
float4 v0 = (hw < active_splits) ? ld_cg_f4(ob4 + hw * 32 + col4) : make_float4(0.0f, 0.0f, 0.0f, 0.0f);
float4 v1 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
if constexpr (SPLITS > 32) {
if (hw + 32 < active_splits) v1 = ld_cg_f4(ob4 + (hw + 32) * 32 + col4);
}
if (warp == 0) {
const float2* mlp = part_ml + qh * 48;
int sA = lane;
float2 mlA = (sA < active_splits) ? ld_cg_f2(mlp + sA) : make_float2(-1e30f, 0.0f);
float mA = (mlA.y > 0.0f) ? mlA.x : -1e30f;
float lA = mlA.y;
if constexpr (SPLITS <= 32) {
float mm = warp_reduce_max(mA);
float wA = (lA > 0.0f) ? ex2_fast(mA - mm) : 0.0f;
float lt = warp_reduce_sum(lA * wA);
float inv = (lt > 0.0f) ? (1.0f / lt) : 0.0f;
sm_sc[sA] = (sA < active_splits) ? (wA * inv) : 0.0f;
} else {
int sB = lane + 32;
float2 mlB = (sB < active_splits) ? ld_cg_f2(mlp + sB) : make_float2(-1e30f, 0.0f);
float mB = (mlB.y > 0.0f) ? mlB.x : -1e30f;
float lB = mlB.y;
float mm = warp_reduce_max(fmaxf(mA, mB));
float wA = (lA > 0.0f) ? ex2_fast(mA - mm) : 0.0f;
float wB = (lB > 0.0f) ? ex2_fast(mB - mm) : 0.0f;
float lt = warp_reduce_sum(lA * wA + lB * wB);
float inv = (lt > 0.0f) ? (1.0f / lt) : 0.0f;
sm_sc[sA] = (sA < active_splits) ? (wA * inv) : 0.0f;
sm_sc[sB] = (sB < active_splits) ? (wB * inv) : 0.0f;
}
}
__syncthreads();
float w0 = sm_sc[hw];
float ax = w0 * v0.x, ay = w0 * v0.y, az = w0 * v0.z, aw = w0 * v0.w;
if constexpr (SPLITS > 32) {
if (hw + 32 < active_splits) {
float w1 = sm_sc[hw + 32];
ax = fmaf(w1, v1.x, ax);
ay = fmaf(w1, v1.y, ay);
az = fmaf(w1, v1.z, az);
aw = fmaf(w1, v1.w, aw);
}
}
*reinterpret_cast<float4*>(sm_f + hw * 64 + l16 * 4) = make_float4(ax, ay, az, aw);
__syncthreads();
if (tid < 64) {
float t0 = 0.0f, t1 = 0.0f, t2 = 0.0f, t3 = 0.0f;
#pragma unroll
for (int q = 0; q < 32; q += 4) {
t0 += sm_f[(q + 0) * 64 + tid];
t1 += sm_f[(q + 1) * 64 + tid];
t2 += sm_f[(q + 2) * 64 + tid];
t3 += sm_f[(q + 3) * 64 + tid];
}
attn_out[qh * kHeadDim + part * 64 + tid] = __float2half_rn((t0 + t1) + (t2 + t3));
}
}
// =========================================================================
// KERNEL 3: Single-Wave O_proj (128 x 512, 8 rows/blk, 2 warps/row, 1 sync, 128 atomics!)
// =========================================================================
__global__ __launch_bounds__(512, 2)
void o_proj_pdl_kernel(
const __nv_bfloat16* __restrict__ o_w,
const __nv_bfloat16* __restrict__ post_ln,
__half* attn_out,
__nv_bfloat16* x_curr,
float* h_mid,
__half* h_mid_ln,
float* ss_mid,
float* ss_x
) {
const int tid = threadIdx.x;
const int grp = tid >> 6;
const int lgrp = tid & 63;
const int w_in = lgrp >> 5;
const int lane = tid & 31;
const int r = (blockIdx.x << 3) + grp;
__shared__ float sm[8][2];
const uint4* row = reinterpret_cast<const uint4*>(o_w + (size_t)r * (kNumQ * kHeadDim));
uint4 w0 = ld_cg_v4(row + lgrp);
uint4 w1 = ld_cg_v4(row + 64 + lgrp);
uint4 w2 = ld_cg_v4(row + 128 + lgrp);
uint4 w3 = ld_cg_v4(row + 192 + lgrp);
float x_res = 0.0f, pln = 0.0f;
if (tid < 8) {
int r0 = (blockIdx.x << 3) + tid;
x_res = __bfloat162float(x_curr[r0]);
pln = __bfloat162float(__ldg(post_ln + r0));
}
pdl_wait();
pdl_launch();
if (blockIdx.x == 0 && tid == 0) ss_x[0] = 0.0f;
const uint4* ao4 = reinterpret_cast<const uint4*>(attn_out);
float acc = dot8_f16_bf16(ao4[lgrp], w0)
+ dot8_f16_bf16(ao4[64 + lgrp], w1)
+ dot8_f16_bf16(ao4[128 + lgrp], w2)
+ dot8_f16_bf16(ao4[192 + lgrp], w3);
acc = warp_reduce_sum(acc);
if (lane == 0) sm[grp][w_in] = acc;
__syncthreads();
if (tid < 8) {
int r0 = (blockIdx.x << 3) + tid;
float y = x_res + sm[tid][0] + sm[tid][1];
h_mid[r0] = y;
h_mid_ln[r0] = __float2half_rn(y * pln);
float y2 = y * y;
y2 += __shfl_down_sync(0xffu, y2, 4);
y2 += __shfl_down_sync(0xffu, y2, 2);
y2 += __shfl_down_sync(0xffu, y2, 1);
if (tid == 0) atomicAdd(ss_mid, y2);
}
}
// =========================================================================
// KERNEL 4: Gate+Up+SiLU (192 x 512, direct weight pointer, 4-byte ss_mid!)
// =========================================================================
__global__ __launch_bounds__(512, 2)
void gate_up_pdl_kernel(
const __nv_bfloat16* __restrict__ gate_up_w,
__half* h_mid_ln,
float* ss_mid,
__half* mlp_act
) {
const int tid = threadIdx.x;
const int warp = tid >> 5;
const int lane = tid & 31;
const int r = (blockIdx.x << 4) + warp;
const uint4* rg = reinterpret_cast<const uint4*>(gate_up_w + ((size_t)r * 2 + 0) * kHidden);
const uint4* ru = reinterpret_cast<const uint4*>(gate_up_w + ((size_t)r * 2 + 1) * kHidden);
uint4 g0 = ld_cg_v4(rg + lane), u0 = ld_cg_v4(ru + lane);
uint4 g1 = ld_cg_v4(rg + 32 + lane), u1 = ld_cg_v4(ru + 32 + lane);
uint4 g2 = ld_cg_v4(rg + 64 + lane), u2 = ld_cg_v4(ru + 64 + lane);
uint4 g3 = ld_cg_v4(rg + 96 + lane), u3 = ld_cg_v4(ru + 96 + lane);
pdl_wait();
pdl_launch();
float inv_rms = rsqrtf(ss_mid[0] * (1.0f / (float)kHidden) + kEps);
const uint4* h4 = reinterpret_cast<const uint4*>(h_mid_ln);
uint4 x0 = h4[lane];
uint4 x1 = h4[32 + lane];
uint4 x2 = h4[64 + lane];
uint4 x3 = h4[96 + lane];
float pg = dot8_f16_bf16(x0, g0) + dot8_f16_bf16(x1, g1) + dot8_f16_bf16(x2, g2) + dot8_f16_bf16(x3, g3);
float pu = dot8_f16_bf16(x0, u0) + dot8_f16_bf16(x1, u1) + dot8_f16_bf16(x2, u2) + dot8_f16_bf16(x3, u3);
float g = warp_reduce_sum(pg) * inv_rms;
float u = warp_reduce_sum(pu) * inv_rms;
if (lane == 0) {
mlp_act[r] = __float2half_rn((g / (1.0f + expf(-g))) * u);
}
}
// =========================================================================
// KERNEL 5: Single-Wave Down_proj (128 x 512, 8 rows/blk, 2 warps/row, 1 sync, 128 atomics!)
// =========================================================================
template<int MODE> // 0=mid-layer, 1=step-boundary, 2=final-output
__global__ __launch_bounds__(512, 2)
void down_pdl_kernel(
const __nv_bfloat16* __restrict__ down_w,
const __nv_bfloat16* __restrict__ next_iln,
__half* mlp_act,
float* h_mid,
__nv_bfloat16* x_curr,
__half* x_ln,
float* ss_x,
float* ss_mid,
__nv_bfloat16* noise_next,
DecodeParams* dp
) {
const int tid = threadIdx.x;
const int grp = tid >> 6;
const int lgrp = tid & 63;
const int w_in = lgrp >> 5;
const int lane = tid & 31;
const int r = (blockIdx.x << 3) + grp;
__shared__ float sm[8][2];
const uint4* row = reinterpret_cast<const uint4*>(down_w + (size_t)r * kInter);
uint4 w0 = ld_cg_v4(row + lgrp);
uint4 w1 = ld_cg_v4(row + 64 + lgrp);
uint4 w2 = ld_cg_v4(row + 128 + lgrp);
uint4 w3 = ld_cg_v4(row + 192 + lgrp);
uint4 w4 = ld_cg_v4(row + 256 + lgrp);
uint4 w5 = ld_cg_v4(row + 320 + lgrp);
float h_res = 0.0f, niln = 0.0f;
__nv_bfloat16 n_val = __float2bfloat16_rn(0.0f);
if (tid < 8) {
int r0 = (blockIdx.x << 3) + tid;
h_res = h_mid[r0];
if constexpr (MODE != 2) niln = __bfloat162float(__ldg(next_iln + r0));
if constexpr (MODE == 1) n_val = noise_next[r0];
}
pdl_wait();
if constexpr (MODE != 2) pdl_launch();
if constexpr (MODE != 2) {
if (blockIdx.x == 0 && tid == 0) ss_mid[0] = 0.0f;
}
const uint4* ma4 = reinterpret_cast<const uint4*>(mlp_act);
float acc = dot8_f16_bf16(ma4[lgrp], w0)
+ dot8_f16_bf16(ma4[64 + lgrp], w1)
+ dot8_f16_bf16(ma4[128 + lgrp], w2)
+ dot8_f16_bf16(ma4[192 + lgrp], w3)
+ dot8_f16_bf16(ma4[256 + lgrp], w4)
+ dot8_f16_bf16(ma4[320 + lgrp], w5);
acc = warp_reduce_sum(acc);
if (lane == 0) sm[grp][w_in] = acc;
__syncthreads();
if (tid < 8) {
int r0 = (blockIdx.x << 3) + tid;
float y = h_res + sm[tid][0] + sm[tid][1];
__nv_bfloat16 y_bf = __float2bfloat16_rn(y);
if constexpr (MODE == 0) {
x_curr[r0] = y_bf;
float xf = __bfloat162float(y_bf);
x_ln[r0] = __float2half_rn(xf * niln);
float y2 = xf * xf;
y2 += __shfl_down_sync(0xffu, y2, 4);
y2 += __shfl_down_sync(0xffu, y2, 2);
y2 += __shfl_down_sync(0xffu, y2, 1);
if (tid == 0) atomicAdd(ss_x, y2);
} else if constexpr (MODE == 1) {
__nv_bfloat16 x_next = mix_half_bf16(n_val, y_bf);
x_curr[r0] = x_next;
float xf = __bfloat162float(x_next);
x_ln[r0] = __float2half_rn(xf * niln);
float y2 = xf * xf;
y2 += __shfl_down_sync(0xffu, y2, 4);
y2 += __shfl_down_sync(0xffu, y2, 2);
y2 += __shfl_down_sync(0xffu, y2, 1);
if (tid == 0) atomicAdd(ss_x, y2);
} else {
dp->h_out[r0] = y_bf;
}
}
}
// ---------------- Host helpers & PDL CUDA Graph Cache ----------------
static bool g_rope_inited = false;
static bool g_l2_limit_set = false;
static size_t g_persist_l2_max = 0;
static size_t g_access_win_max = 0;
static cudaStream_t g_cap_stream = nullptr;
static cudaStream_t g_aux_stream = nullptr;
static cudaEvent_t g_ev_mt_go = nullptr;
static cudaEvent_t g_ev_mt_done = nullptr;
static cudaStream_t g_last_l2_stream = nullptr;
static void* g_last_l2_ptr = nullptr;
static size_t g_last_l2_bytes = 0;
struct GraphKey {
uintptr_t packed_ptr;
uintptr_t ws_ptr;
int num_layers;
int chunk_steps;
int attn_mode; // 0: v4 no-cs S=32, 1: v4 cs S=32, 2: v8 S=47
bool operator==(const GraphKey& o) const {
return packed_ptr == o.packed_ptr && ws_ptr == o.ws_ptr &&
num_layers == o.num_layers && chunk_steps == o.chunk_steps &&
attn_mode == o.attn_mode;
}
};
struct GraphKeyHash {
size_t operator()(const GraphKey& k) const {
size_t h = std::hash<uintptr_t>{}(k.packed_ptr);
h ^= std::hash<uintptr_t>{}(k.ws_ptr) + 0x9e3779b97f4a7c15ull + (h << 6) + (h >> 2);
h ^= (size_t)((k.num_layers << 16) ^ (k.chunk_steps << 4) ^ k.attn_mode);
return h;
}
};
static std::unordered_map<GraphKey, cudaGraphExec_t, GraphKeyHash> g_graph_cache;
static inline size_t align128(size_t off) { return (off + 127) & ~size_t(127); }
std::vector<LayerWeights> compute_layer_weights(uint8_t* base, int num_layers) {
std::vector<LayerWeights> meta(num_layers);
size_t off = 0;
for (int l = 0; l < num_layers; ++l) {
LayerWeights lw;
off = align128(off); lw.qkv_w = reinterpret_cast<const __nv_bfloat16*>(base + off); off += (size_t)4096 * kHidden * 2;
off = align128(off); lw.o_w = reinterpret_cast<const __nv_bfloat16*>(base + off); off += (size_t)kHidden * kNumQ * kHeadDim * 2;
off = align128(off); lw.gate_up_w = reinterpret_cast<const __nv_bfloat16*>(base + off); off += (size_t)kInter * 2 * kHidden * 2;
off = align128(off); lw.down_w = reinterpret_cast<const __nv_bfloat16*>(base + off); off += (size_t)kHidden * kInter * 2;
off = align128(off); lw.input_ln = reinterpret_cast<const __nv_bfloat16*>(base + off); off += kHidden * 2;
off = align128(off); lw.q_norm = reinterpret_cast<const __nv_bfloat16*>(base + off); off += kHeadDim * 2;
off = align128(off); lw.k_norm = reinterpret_cast<const __nv_bfloat16*>(base + off); off += kHeadDim * 2;
off = align128(off); lw.post_ln = reinterpret_cast<const __nv_bfloat16*>(base + off); off += kHidden * 2;
meta[l] = lw;
}
return meta;
}
void set_l2_persist_on_stream(cudaStream_t stream, void* pptr, size_t pbytes) {
if (g_persist_l2_max == 0 || pbytes == 0) return;
size_t win_bytes = (pbytes < g_access_win_max) ? pbytes : g_access_win_max;
float hit_ratio = (win_bytes > 0) ? ((float)g_persist_l2_max / (float)win_bytes) : 1.0f;
if (hit_ratio > 1.0f) hit_ratio = 1.0f;
cudaStreamAttrValue attr = {};
attr.accessPolicyWindow.base_ptr = pptr;
attr.accessPolicyWindow.num_bytes = win_bytes;
attr.accessPolicyWindow.hitRatio = hit_ratio;
attr.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting;
attr.accessPolicyWindow.missProp = cudaAccessPropertyNormal;
cudaStreamSetAttribute(stream, cudaStreamAttributeAccessPolicyWindow, &attr);
}
void ensure_constants() {
if (!g_rope_inited) {
float inv[kHalfHead];
for (int i = 0; i < kHalfHead; ++i) {
inv[i] = (float)(1.0 / pow(10000.0, (double)i / (double)kHalfHead));
}
cudaMemcpyToSymbol(c_rope_inv, inv, sizeof(inv));
g_rope_inited = true;
}
if (!g_l2_limit_set) {
cudaDeviceProp prop;
cudaGetDeviceProperties(&prop, 0);
g_persist_l2_max = prop.persistingL2CacheMaxSize;
g_access_win_max = prop.accessPolicyMaxWindowSize;
if (g_persist_l2_max > 0) {
cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, g_persist_l2_max);
}
cudaStreamCreateWithFlags(&g_cap_stream, cudaStreamNonBlocking);
cudaStreamCreateWithFlags(&g_aux_stream, cudaStreamNonBlocking);
cudaEventCreateWithFlags(&g_ev_mt_go, cudaEventDisableTiming);
cudaEventCreateWithFlags(&g_ev_mt_done, cudaEventDisableTiming);
g_l2_limit_set = true;
}
}
Workspace make_workspace(uint8_t* wbase) {
size_t off = 0;
Workspace ws;
ws.x_curr = reinterpret_cast<__nv_bfloat16*>(wbase + off); off = align128(off + kHidden * 2);
ws.x_ln = reinterpret_cast<__half*>(wbase + off); off = align128(off + kHidden * 2);
ws.h_mid = reinterpret_cast<float*>(wbase + off); off = align128(off + kHidden * 4);
ws.h_mid_ln = reinterpret_cast<__half*>(wbase + off); off = align128(off + kHidden * 2);
ws.ss_x = reinterpret_cast<float*>(wbase + off); off = align128(off + 4);
ws.ss_mid = reinterpret_cast<float*>(wbase + off); off = align128(off + 4);
ws.q_raw = reinterpret_cast<float*>(wbase + off); off = align128(off + kNumQ * kHeadDim * 4);
ws.k_raw = reinterpret_cast<float*>(wbase + off); off = align128(off + kNumKV * kHeadDim * 4);
ws.attn_out = reinterpret_cast<__half*>(wbase + off); off = align128(off + kNumQ * kHeadDim * 2);
ws.mlp_act = reinterpret_cast<__half*>(wbase + off); off = align128(off + kInter * 2);
ws.attn_part_o = reinterpret_cast<float*>(wbase + off); off = align128(off + (size_t)kNumQ * 48 * kHeadDim * 4);
ws.attn_part_ml = reinterpret_cast<float2*>(wbase + off); off = align128(off + (size_t)kNumQ * 48 * 8);
ws.rope_cos = reinterpret_cast<float*>(wbase + off); off = align128(off + (size_t)kMaxChunk * kHalfHead * 4);
ws.rope_sin = reinterpret_cast<float*>(wbase + off); off = align128(off + (size_t)kMaxChunk * kHalfHead * 4);
ws.mt_state = reinterpret_cast<uint32_t*>(wbase + off); off = align128(off + 624 * 4);
ws.mt_cur = reinterpret_cast<int*>(wbase + off); off = align128(off + 4);
ws.raw_u32 = reinterpret_cast<uint32_t*>(wbase + off); off = align128(off + (size_t)kMaxChunk * kHidden * 4);
ws.noise_buf = reinterpret_cast<__nv_bfloat16*>(wbase + off); off = align128(off + (size_t)kMaxChunk * kHidden * 2);
ws.params = reinterpret_cast<DecodeParams*>(wbase + off);
return ws;
}
static void launch_pdl(cudaStream_t stream, void* fn, int grid, int blk, void** args) {
cudaLaunchConfig_t cfg = {};
cfg.gridDim = dim3(grid);
cfg.blockDim = dim3(blk);
cfg.stream = stream;
cudaLaunchAttribute attr[1];
attr[0].id = cudaLaunchAttributeProgrammaticStreamSerialization;
attr[0].val.programmaticStreamSerializationAllowed = 1;
cfg.attrs = attr;
cfg.numAttrs = 1;
cudaError_t err = cudaLaunchKernelExC(&cfg, fn, args);
TORCH_CHECK(err == cudaSuccess, "cudaLaunchKernelExC failed: ", cudaGetErrorString(err));
}
cudaGraphExec_t get_or_capture_graph(
void* packed_ptr,
size_t packed_bytes,
Workspace ws,
uintptr_t ws_ptr,
int num_layers,
int chunk_steps,
int attn_mode
) {
GraphKey key{reinterpret_cast<uintptr_t>(packed_ptr), ws_ptr, num_layers, chunk_steps, attn_mode};
auto it = g_graph_cache.find(key);
if (it != g_graph_cache.end()) return it->second;
if (g_graph_cache.size() >= 64) {
for (auto& kv : g_graph_cache) cudaGraphExecDestroy(kv.second);
g_graph_cache.clear();
}
auto h_layers = compute_layer_weights(reinterpret_cast<uint8_t*>(packed_ptr), num_layers);
cudaStream_t st = g_cap_stream;
cudaStream_t aux = g_aux_stream;
set_l2_persist_on_stream(st, packed_ptr, packed_bytes);
cudaError_t err = cudaStreamBeginCapture(st, cudaStreamCaptureModeThreadLocal);
TORCH_CHECK(err == cudaSuccess, "cudaStreamBeginCapture failed: ", cudaGetErrorString(err));
{
const __nv_bfloat16* iln0 = h_layers[0].input_ln;
void* a[] = {&iln0, &ws};
launch_pdl(st, (void*)prologue_step0_kernel, 1, 256, a);
}
if (chunk_steps > 1) {
cudaEventRecord(g_ev_mt_go, st);
cudaStreamWaitEvent(aux, g_ev_mt_go, 0);
mt_u32_rest_kernel<<<1, 256, 0, aux>>>(ws, chunk_steps);
bm_rope_rest_kernel<<<16, 256, 0, aux>>>(ws, chunk_steps);
cudaEventRecord(g_ev_mt_done, aux);
}
void* attn_fn = (attn_mode == 0) ? (void*)attn_pdl_v4_kernel<47, false, false>
: ((attn_mode == 1) ? (void*)attn_pdl_v4_kernel<47, true, false>
: ((attn_mode == 2) ? (void*)attn_pdl_v4_kernel<47, true, true>
: (void*)attn_pdl_v8_kernel<47>));
void* red_fn = (attn_mode < 3) ? (void*)reduce_pdl_kernel<47, 16> : (void*)reduce_pdl_kernel<47, 32>;
int attn_grid = kNumKV * 47;
DecodeParams* dp = ws.params;
for (int step = 0; step < chunk_steps; ++step) {
for (int l = 0; l < num_layers; ++l) {
const LayerWeights& lw = h_layers[l];
{
void* a[] = {(void*)&lw.qkv_w, &ws.x_ln, &ws.ss_x, &ws.q_raw, &ws.k_raw, &dp, &l, &step};
launch_pdl(st, (void*)qkv_pdl_kernel, 256, 512, a);
}
{
void* a[] = {(void*)&lw.q_norm, (void*)&lw.k_norm, &ws.q_raw, &ws.k_raw,
&ws.rope_cos, &ws.rope_sin, &ws.attn_part_o, &ws.attn_part_ml,
&dp, &l, &step};
launch_pdl(st, attn_fn, attn_grid, 256, a);
}
{
void* a[] = {&ws.attn_part_o, &ws.attn_part_ml, &ws.attn_out, &dp, &step};
launch_pdl(st, red_fn, kNumQ * 2, 512, a);
}
{
void* a[] = {(void*)&lw.o_w, (void*)&lw.post_ln, &ws.attn_out, &ws.x_curr,
&ws.h_mid, &ws.h_mid_ln, &ws.ss_mid, &ws.ss_x};
launch_pdl(st, (void*)o_proj_pdl_kernel, 128, 512, a);
}
{
void* a[] = {(void*)&lw.gate_up_w, &ws.h_mid_ln, &ws.ss_mid, &ws.mlp_act};
launch_pdl(st, (void*)gate_up_pdl_kernel, 192, 512, a);
}
if (step == 0 && l == num_layers - 1 && chunk_steps > 1) {
cudaStreamWaitEvent(st, g_ev_mt_done, 0);
}
{
const __nv_bfloat16* next_iln = (l < num_layers - 1)
? h_layers[l + 1].input_ln : h_layers[0].input_ln;
__nv_bfloat16* noise_next = ws.noise_buf + (size_t)((step + 1 < chunk_steps) ? (step + 1) : 0) * kHidden;
void* a[] = {(void*)&lw.down_w, &next_iln, &ws.mlp_act, &ws.h_mid,
&ws.x_curr, &ws.x_ln, &ws.ss_x, &ws.ss_mid, &noise_next, &dp};
void* down_fn = (l < num_layers - 1)
? (void*)down_pdl_kernel<0>
: ((step + 1 < chunk_steps) ? (void*)down_pdl_kernel<1> : (void*)down_pdl_kernel<2>);
launch_pdl(st, down_fn, 128, 512, a);
}
}
}
cudaGraph_t graph = nullptr;
err = cudaStreamEndCapture(st, &graph);
TORCH_CHECK(err == cudaSuccess, "cudaStreamEndCapture failed: ", cudaGetErrorString(err));
cudaGraphExec_t exec = nullptr;
err = cudaGraphInstantiate(&exec, graph, 0);
cudaGraphDestroy(graph);
TORCH_CHECK(err == cudaSuccess, "cudaGraphInstantiate failed: ", cudaGetErrorString(err));
cudaGraphUpload(exec, st);
cudaStreamSynchronize(st);
g_graph_cache[key] = exec;
return exec;
}
__global__ void pack_gate_up_kernel(
const __nv_bfloat16* __restrict__ gate_proj,
const __nv_bfloat16* __restrict__ up_proj,
__nv_bfloat16* __restrict__ dst
) {
size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
size_t total = (size_t)kInter * 2 * kHidden / 8;
if (idx >= total) return;
size_t col8 = idx & 127;
size_t row2 = idx >> 7;
int r = (int)(row2 >> 1);
int is_up = (int)(row2 & 1);
const __nv_bfloat16* src = (is_up ? up_proj : gate_proj) + (size_t)r * kHidden + col8 * 8;
reinterpret_cast<uint4*>(dst)[idx] = *reinterpret_cast<const uint4*>(src);
}
} // namespace
void pack_model_weights(
std::vector<torch::Tensor> q_projs,
std::vector<torch::Tensor> k_projs,
std::vector<torch::Tensor> v_projs,
std::vector<torch::Tensor> o_projs,
std::vector<torch::Tensor> gate_projs,
std::vector<torch::Tensor> up_projs,
std::vector<torch::Tensor> down_projs,
std::vector<torch::Tensor> input_lns,
std::vector<torch::Tensor> q_norms,
std::vector<torch::Tensor> k_norms,
std::vector<torch::Tensor> post_lns,
torch::Tensor packed_buf,
torch::Tensor ws_buf
) {
ensure_constants();
cudaStream_t stream = at::cuda::getCurrentCUDAStream();
int num_layers = (int)q_projs.size();
TORCH_CHECK(num_layers <= kMaxLayers, "num_layers exceeds kMaxLayers");
uint8_t* base = reinterpret_cast<uint8_t*>(packed_buf.data_ptr());
auto h_layers = compute_layer_weights(base, num_layers);
for (int l = 0; l < num_layers; ++l) {
auto* qkv_dst = const_cast<__nv_bfloat16*>(h_layers[l].qkv_w);
cudaMemcpyAsync(qkv_dst, q_projs[l].data_ptr(), (size_t)2048 * kHidden * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(qkv_dst + (size_t)2048 * kHidden, k_projs[l].data_ptr(), (size_t)1024 * kHidden * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(qkv_dst + (size_t)3072 * kHidden, v_projs[l].data_ptr(), (size_t)1024 * kHidden * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].o_w), o_projs[l].data_ptr(),
(size_t)kHidden * kNumQ * kHeadDim * 2, cudaMemcpyDeviceToDevice, stream);
{
size_t total = (size_t)kInter * 2 * kHidden / 8;
pack_gate_up_kernel<<<(total + 255) / 256, 256, 0, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(gate_projs[l].data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(up_projs[l].data_ptr()),
const_cast<__nv_bfloat16*>(h_layers[l].gate_up_w));
}
cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].down_w), down_projs[l].data_ptr(),
(size_t)kHidden * kInter * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].input_ln), input_lns[l].data_ptr(),
kHidden * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].q_norm), q_norms[l].data_ptr(),
kHeadDim * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].k_norm), k_norms[l].data_ptr(),
kHeadDim * 2, cudaMemcpyDeviceToDevice, stream);
cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].post_ln), post_lns[l].data_ptr(),
kHidden * 2, cudaMemcpyDeviceToDevice, stream);
}
void* pptr = packed_buf.data_ptr();
size_t pbytes = (size_t)packed_buf.numel() * packed_buf.element_size();
uint8_t* wbase = reinterpret_cast<uint8_t*>(ws_buf.data_ptr());
Workspace ws = make_workspace(wbase);
uintptr_t ws_ptr = reinterpret_cast<uintptr_t>(wbase);
get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 0);
get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 1);
get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 2);
get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 3);
get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 32, 3);
get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 16, 3);
}
void run_decode_steps_cuda(
torch::Tensor packed_buf,
std::vector<torch::Tensor> k_caches,
std::vector<torch::Tensor> v_caches,
torch::Tensor ws_buf,
torch::Tensor h_in,
torch::Tensor h_out,
int start_pos,
int n_steps,
int64_t rng_seed,
int init_mt,
int use_l2_persist
) {
if (n_steps <= 0) return;
ensure_constants();
cudaStream_t stream = at::cuda::getCurrentCUDAStream();
int num_layers = (int)k_caches.size();
int64_t max_seq = k_caches[0].size(1);
size_t pbytes = (size_t)packed_buf.numel() * packed_buf.element_size();
void* pptr = packed_buf.data_ptr();
if (use_l2_persist && g_persist_l2_max > 0) {
if (stream != g_last_l2_stream || pptr != g_last_l2_ptr || pbytes != g_last_l2_bytes) {
set_l2_persist_on_stream(stream, pptr, pbytes);
g_last_l2_stream = stream;
g_last_l2_ptr = pptr;
g_last_l2_bytes = pbytes;
}
}
uint8_t* wbase = reinterpret_cast<uint8_t*>(ws_buf.data_ptr());
Workspace ws = make_workspace(wbase);
uintptr_t ws_ptr = reinterpret_cast<uintptr_t>(wbase);
DecodeParams h_params = {};
for (int l = 0; l < num_layers; ++l) {
h_params.kvs[l].k = reinterpret_cast<__nv_bfloat16*>(k_caches[l].data_ptr());
h_params.kvs[l].v = reinterpret_cast<__nv_bfloat16*>(v_caches[l].data_ptr());
}
h_params.max_seq = max_seq;
h_params.rng_seed = (uint64_t)rng_seed;
const __nv_bfloat16* cur_in = reinterpret_cast<const __nv_bfloat16*>(h_in.data_ptr());
__nv_bfloat16* cur_out = reinterpret_cast<__nv_bfloat16*>(h_out.data_ptr());
int done = 0;
while (done < n_steps) {
int chunk = n_steps - done;
if (chunk > kMaxChunk) chunk = kMaxChunk;
int cur_pos = start_pos + done;
int attn_mode = (cur_pos < 512) ? 0
: ((cur_pos < 4096) ? 1
: ((cur_pos < 16384) ? 2 : 3));
h_params.h_in = (done == 0) ? cur_in : cur_out;
h_params.h_out = cur_out;
h_params.start_pos = cur_pos;
h_params.n_steps = chunk;
h_params.init_mt = (done == 0) ? init_mt : 0;
cudaMemcpyAsync(ws.params, &h_params, sizeof(DecodeParams), cudaMemcpyHostToDevice, stream);
cudaGraphExec_t exec = get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, chunk, attn_mode);
cudaError_t err = cudaGraphLaunch(exec, stream);
TORCH_CHECK(err == cudaSuccess, "cudaGraphLaunch failed: ", cudaGetErrorString(err));
done += chunk;
}
}
"""
_CPP_SRC = r"""
#include <torch/extension.h>
#include <vector>
void pack_model_weights(
std::vector<torch::Tensor> q_projs,
std::vector<torch::Tensor> k_projs,
std::vector<torch::Tensor> v_projs,
std::vector<torch::Tensor> o_projs,
std::vector<torch::Tensor> gate_projs,
std::vector<torch::Tensor> up_projs,
std::vector<torch::Tensor> down_projs,
std::vector<torch::Tensor> input_lns,
std::vector<torch::Tensor> q_norms,
std::vector<torch::Tensor> k_norms,
std::vector<torch::Tensor> post_lns,
torch::Tensor packed_buf,
torch::Tensor ws_buf
);
void run_decode_steps_cuda(
torch::Tensor packed_buf,
std::vector<torch::Tensor> k_caches,
std::vector<torch::Tensor> v_caches,
torch::Tensor ws_buf,
torch::Tensor h_in,
torch::Tensor h_out,
int start_pos,
int n_steps,
int64_t rng_seed,
int init_mt,
int use_l2_persist
);
"""
_EXT = None
def _get_ext():
global _EXT
if _EXT is None:
_EXT = load_inline(
name="megaqwen_sm120_v12",
cpp_sources=_CPP_SRC,
cuda_sources=_CUDA_SRC,
functions=["pack_model_weights", "run_decode_steps_cuda"],
extra_cuda_cflags=[
"-O3",
"-arch=sm_120",
"--expt-relaxed-constexpr",
"-lineinfo",
],
verbose=False,
)
return _EXT
class Block(nn.Module):
def __init__(self):
super().__init__()
H, I, D = HIDDEN, INTERMEDIATE, HEAD_DIM
self.input_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16))
self.q_proj = nn.Parameter(torch.empty(NUM_Q * D, H, dtype=torch.bfloat16))
self.k_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16))
self.v_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16))
self.q_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16))
self.k_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16))
self.o_proj = nn.Parameter(torch.empty(H, NUM_Q * D, dtype=torch.bfloat16))
self.post_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16))
self.gate_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16))
self.up_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16))
self.down_proj = nn.Parameter(torch.empty(H, I, dtype=torch.bfloat16))
for p in self.parameters():
if (
p is self.input_ln
or p is self.post_ln
or p is self.q_norm
or p is self.k_norm
):
continue
nn.init.normal_(p, std=0.02)
class Model(nn.Module):
def __init__(self, num_layers: int = NUM_LAYERS, max_seq: int = 131072):
super().__init__()
self.num_layers = num_layers
self.max_seq = max_seq
self.blocks = nn.ModuleList([Block() for _ in range(num_layers)])
self._packed_buf: Optional[torch.Tensor] = None
self._ws_buf: Optional[torch.Tensor] = None
self._params_tuple: Optional[Tuple[nn.Parameter, ...]] = None
self._packed_versions: Optional[Tuple[int, ...]] = None
self._packed_device: Optional[torch.device] = None
def _param_versions(self) -> Tuple[int, ...]:
if self._params_tuple is None:
self._params_tuple = tuple(self.parameters())
return tuple(p._version for p in self._params_tuple)
def _load_from_state_dict(self, *args, **kwargs):
super()._load_from_state_dict(*args, **kwargs)
self._packed_versions = None
def ensure_packed(self, device: torch.device) -> None:
vers = self._param_versions()
if (
self._packed_buf is not None
and self._packed_device == device
and self._packed_versions == vers
):
return
ext = _get_ext()
bytes_per_layer = (
(4096 * HIDDEN)
+ (HIDDEN * NUM_Q * HEAD_DIM)
+ (INTERMEDIATE * 2 * HIDDEN)
+ (HIDDEN * INTERMEDIATE)
+ (HIDDEN + HEAD_DIM + HEAD_DIM + HIDDEN)
) * 2 + 4096
total_bytes = bytes_per_layer * self.num_layers
if self._packed_buf is None or self._packed_buf.device != device or self._packed_buf.numel() < total_bytes:
self._packed_buf = torch.empty(total_bytes, dtype=torch.uint8, device=device)
if self._ws_buf is None or self._ws_buf.device != device:
self._ws_buf = torch.empty(1536 * 1024, dtype=torch.uint8, device=device)
ext.pack_model_weights(
[b.q_proj.contiguous() for b in self.blocks],
[b.k_proj.contiguous() for b in self.blocks],
[b.v_proj.contiguous() for b in self.blocks],
[b.o_proj.contiguous() for b in self.blocks],
[b.gate_proj.contiguous() for b in self.blocks],
[b.up_proj.contiguous() for b in self.blocks],
[b.down_proj.contiguous() for b in self.blocks],
[b.input_ln.contiguous() for b in self.blocks],
[b.q_norm.contiguous() for b in self.blocks],
[b.k_norm.contiguous() for b in self.blocks],
[b.post_ln.contiguous() for b in self.blocks],
self._packed_buf,
self._ws_buf,
)
self._packed_versions = vers
self._packed_device = device
def empty_caches(num_layers: int, max_seq: int, device, dtype=torch.bfloat16):
k_all = torch.empty(num_layers, NUM_KV, max_seq, HEAD_DIM, device=device, dtype=dtype)
v_all = torch.empty(num_layers, NUM_KV, max_seq, HEAD_DIM, device=device, dtype=dtype)
return [k_all[i] for i in range(num_layers)], [v_all[i] for i in range(num_layers)]
def _seeded_hidden(seed: int, device) -> torch.Tensor:
g = torch.Generator(device="cpu")
g.manual_seed(seed)
return torch.randn(HIDDEN, generator=g, dtype=torch.bfloat16).to(device)
@torch.no_grad()
def prefill(
model: Model,
ctx_len: int,
seed: int,
device: Optional[torch.device] = None,
):
"""Build KV of length ctx_len. NOT timed in benchmark."""
device = device or next(model.parameters()).device
model = model.to(device).eval()
assert ctx_len <= model.max_seq
model.ensure_packed(device)
h = _seeded_hidden(seed, device).contiguous()
k_caches, v_caches = empty_caches(model.num_layers, model.max_seq, device)
if ctx_len <= 0:
return h, k_caches, v_caches
h_out = torch.empty_like(h)
ext = _get_ext()
ext.run_decode_steps_cuda(
model._packed_buf,
k_caches,
v_caches,
model._ws_buf,
h,
h_out,
0,
ctx_len,
seed + 1,
1,
1,
)
return h_out, k_caches, v_caches
@torch.no_grad()
def decode_steps(
model: Model,
hidden: torch.Tensor,
k_caches: List[torch.Tensor],
v_caches: List[torch.Tensor],
start_pos: int,
n_steps: int,
seed: int,
):
"""Run n_steps decode steps starting at start_pos. Timed in benchmark."""
if n_steps <= 0:
return hidden, k_caches, v_caches
device = hidden.device
model.ensure_packed(device)
h_in = hidden if hidden.is_contiguous() else hidden.contiguous()
h_out = torch.empty(HIDDEN, dtype=torch.bfloat16, device=device)
ext = _get_ext()
ext.run_decode_steps_cuda(
model._packed_buf,
k_caches,
v_caches,
model._ws_buf,
h_in,
h_out,
start_pos,
n_steps,
seed + 2,
1,
1,
)
return h_out, k_caches, v_caches
def run(
ctx_len: int,
n_decode: int,
seed: int,
model: Optional[Model] = None,
max_seq: Optional[int] = None,
) -> dict:
device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
max_seq = max_seq or max(ctx_len + n_decode, 512)
if model is None:
model = Model(NUM_LAYERS, max_seq)
else:
if getattr(model, "max_seq", 0) < ctx_len + n_decode:
raise ValueError(
f"model.max_seq={getattr(model, 'max_seq', None)} too small for "
f"ctx_len={ctx_len}+n_decode={n_decode}"
)
model = model.to(device).eval()
h, k_caches, v_caches = prefill(model, ctx_len, seed, device=device)
h, k_caches, v_caches = decode_steps(
model, h, k_caches, v_caches, start_pos=ctx_len, n_steps=n_decode, seed=seed
)
return {
"last_hidden": h.detach(),
"ctx_len": ctx_len,
"decode_steps": n_decode,
}
def get_init_inputs():
return [NUM_LAYERS, 131072]
def get_inputs():
return []
20261118_150545_agy_gemini-4-neon-xhigh_03_megaqwen_decode