"""Fast multi-layer Qwen3-0.6B-geometry decode engine for RTX PRO 6000 (SM120). V11 SM120 Blackwell Direct-Pointer Single-Wave PDL Weight-Prefetch CUDA Graph Engine: - Indirect DecodeParams CUDA Graph pre-captured with 80 MiB persisting L2 window baked into every graph node and uploaded via cudaGraphUpload during untimed setup. - Direct-by-value weight pointers baked into every captured kernel node (zero LayerWeights indirection before weight prefetch). - L1-bypassing weight prefetch (`ld.global.cg.v4.u32`) issued BEFORE `griddepcontrol.wait` across all 4 projections: QKV (`256x512`), single-wave O_proj (`128x512`, 2 warps/row), Gate+Up+SiLU (`192x512`), single-wave Down_proj (`128x512`, 2 warps/row) with preloaded residuals/norms before `pdl_wait` and 1-sync Warp-0 epilogues. - Precomputed per-step RoPE cos/sin table in prologue/aux branch (zero `sincosf` in attn, 1-sync warp-parallel Q/K RMSNorm + RoPE). - In-warp register merge before SMEM in BOTH attention paths (2x smaller SMEM & 2x faster block reduction in 128-bit `v4` half-warp; 4x smaller SMEM & single-pass block reduction in 256-bit `v8` quarter-warp for >=16k contexts). - Single-L2-RTT 64-block `reduce_pdl_kernel` overlapping `part_o` `ld.global.cg.v4` loads with packed `float2` `part_ml` softmax scaling while triggering early `O_proj` PDL launch. - Concurrent CUDA Graph fork/join for exact GPU MT19937 + parallel Box-Muller + RoPE table. """ from __future__ import annotations from typing import List, Optional, Tuple import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline OP_TYPE = "megaqwen_decode" SUPPORTED_PRECISIONS = ["bf16"] HARDWARE_REQUIRED = ["RTX_PRO_6000"] HIDDEN = 1024 INTERMEDIATE = 3072 NUM_Q = 16 NUM_KV = 8 HEAD_DIM = 128 NUM_LAYERS = 4 EPS = 1e-6 _CUDA_SRC = r""" #include #include #include #include #include #include #include #include #include namespace { constexpr int kHidden = 1024; constexpr int kInter = 3072; constexpr int kNumQ = 16; constexpr int kNumKV = 8; constexpr int kHeadDim = 128; constexpr int kHalfHead = 64; constexpr float kEps = 1e-6f; constexpr float kAttnScaleLog2E = 0.12751742995591326f; constexpr int kMaxChunk = 64; constexpr int kSmemPadDim = kHeadDim + 4; constexpr int kMaxLayers = 8; __constant__ float c_rope_inv[kHalfHead]; struct u32x8 { uint32_t r0, r1, r2, r3, r4, r5, r6, r7; }; struct LayerWeights { const __nv_bfloat16* qkv_w; // [4096, 1024] const __nv_bfloat16* o_w; // [1024, 2048] const __nv_bfloat16* gate_up_w; // [3072, 2, 1024] const __nv_bfloat16* down_w; // [1024, 3072] const __nv_bfloat16* input_ln; // [1024] const __nv_bfloat16* q_norm; // [128] const __nv_bfloat16* k_norm; // [128] const __nv_bfloat16* post_ln; // [1024] }; struct LayerKV { __nv_bfloat16* k; // [8, max_seq, 128] __nv_bfloat16* v; // [8, max_seq, 128] }; struct DecodeParams { LayerKV kvs[kMaxLayers]; const __nv_bfloat16* h_in; __nv_bfloat16* h_out; int64_t max_seq; int start_pos; int n_steps; int init_mt; uint64_t rng_seed; }; struct Workspace { __nv_bfloat16* x_curr; // [1024] exact bf16 residual1 __half* x_ln; // [1024] fp16 pre-mul x_curr * input_ln float* h_mid; // [1024] exact fp32 residual2 __half* h_mid_ln; // [1024] fp16 pre-mul h_mid * post_ln float* ss_x; // [1] (padded 128B) float* ss_mid; // [1] (padded 128B) float* q_raw; // [16, 128] float* k_raw; // [8, 128] __half* attn_out; // [16, 128] fp16 __half* mlp_act; // [3072] fp16 float* attn_part_o; // [16, 48, 128] float2* attn_part_ml; // [16, 48] packed {m, l} float* rope_cos; // [kMaxChunk, 64] float* rope_sin; // [kMaxChunk, 64] uint32_t* mt_state; // [624] int* mt_cur; // [1] uint32_t* raw_u32; // [kMaxChunk, 1024] __nv_bfloat16* noise_buf; // [kMaxChunk, 1024] DecodeParams* params; // [1] }; // ---------------- Low-level SM120 PTX helpers ---------------- __device__ __forceinline__ float bf16_lo(uint32_t u) { return __uint_as_float(u << 16); } __device__ __forceinline__ float bf16_hi(uint32_t u) { return __uint_as_float(u & 0xFFFF0000u); } __device__ __forceinline__ float ex2_fast(float x) { float r; asm("ex2.approx.ftz.f32 %0, %1;" : "=f"(r) : "f"(x)); return r; } __device__ __forceinline__ uint4 ld_cg_v4(const void* p) { uint4 v; asm volatile("ld.global.cg.v4.u32 {%0,%1,%2,%3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(p)); return v; } __device__ __forceinline__ float4 ld_cg_f4(const void* p) { float4 v; asm volatile("ld.global.cg.v4.f32 {%0,%1,%2,%3}, [%4];" : "=f"(v.x), "=f"(v.y), "=f"(v.z), "=f"(v.w) : "l"(p)); return v; } __device__ __forceinline__ float2 ld_cg_f2(const void* p) { float2 v; asm volatile("ld.global.cg.v2.f32 {%0,%1}, [%2];" : "=f"(v.x), "=f"(v.y) : "l"(p)); return v; } __device__ __forceinline__ uint4 ld_kv_cs_v4(const void* p) { uint4 v; asm volatile("ld.global.cs.L2::256B.v4.u32 {%0,%1,%2,%3}, [%4];" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(p)); return v; } __device__ __forceinline__ u32x8 ld_kv_v8(const void* p) { u32x8 v; asm volatile("ld.global.L1::no_allocate.L2::evict_first.v8.b32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];" : "=r"(v.r0), "=r"(v.r1), "=r"(v.r2), "=r"(v.r3), "=r"(v.r4), "=r"(v.r5), "=r"(v.r6), "=r"(v.r7) : "l"(p)); return v; } __device__ __forceinline__ void st_cg_f32(float* p, float v) { asm volatile("st.global.cg.f32 [%0], %1;" :: "l"(p), "f"(v) : "memory"); } __device__ __forceinline__ void st_cg_f2(float2* p, float2 v) { asm volatile("st.global.cg.v2.f32 [%0], {%1,%2};" :: "l"(p), "f"(v.x), "f"(v.y) : "memory"); } __device__ __forceinline__ void pdl_wait() { asm volatile("griddepcontrol.wait;" ::: "memory"); } __device__ __forceinline__ void pdl_launch() { asm volatile("griddepcontrol.launch_dependents;" ::: "memory"); } __device__ __forceinline__ float dot8_f16_bf16(const uint4 x, const uint4 w) { float2 x0 = __half22float2(*reinterpret_cast(&x.x)); float2 x1 = __half22float2(*reinterpret_cast(&x.y)); float2 x2 = __half22float2(*reinterpret_cast(&x.z)); float2 x3 = __half22float2(*reinterpret_cast(&x.w)); float acc = 0.0f; acc = fmaf(x0.x, bf16_lo(w.x), acc); acc = fmaf(x0.y, bf16_hi(w.x), acc); acc = fmaf(x1.x, bf16_lo(w.y), acc); acc = fmaf(x1.y, bf16_hi(w.y), acc); acc = fmaf(x2.x, bf16_lo(w.z), acc); acc = fmaf(x2.y, bf16_hi(w.z), acc); acc = fmaf(x3.x, bf16_lo(w.w), acc); acc = fmaf(x3.y, bf16_hi(w.w), acc); return acc; } __device__ __forceinline__ float dot8_f_u4(const float a[8], const uint4 w) { float acc = 0.0f; acc = fmaf(a[0], bf16_lo(w.x), acc); acc = fmaf(a[1], bf16_hi(w.x), acc); acc = fmaf(a[2], bf16_lo(w.y), acc); acc = fmaf(a[3], bf16_hi(w.y), acc); acc = fmaf(a[4], bf16_lo(w.z), acc); acc = fmaf(a[5], bf16_hi(w.z), acc); acc = fmaf(a[6], bf16_lo(w.w), acc); acc = fmaf(a[7], bf16_hi(w.w), acc); return acc; } __device__ __forceinline__ float dot16_f_u8(const float a[16], const u32x8 w) { float acc = 0.0f; acc = fmaf(a[0], bf16_lo(w.r0), acc); acc = fmaf(a[1], bf16_hi(w.r0), acc); acc = fmaf(a[2], bf16_lo(w.r1), acc); acc = fmaf(a[3], bf16_hi(w.r1), acc); acc = fmaf(a[4], bf16_lo(w.r2), acc); acc = fmaf(a[5], bf16_hi(w.r2), acc); acc = fmaf(a[6], bf16_lo(w.r3), acc); acc = fmaf(a[7], bf16_hi(w.r3), acc); acc = fmaf(a[8], bf16_lo(w.r4), acc); acc = fmaf(a[9], bf16_hi(w.r4), acc); acc = fmaf(a[10], bf16_lo(w.r5), acc); acc = fmaf(a[11], bf16_hi(w.r5), acc); acc = fmaf(a[12], bf16_lo(w.r6), acc); acc = fmaf(a[13], bf16_hi(w.r6), acc); acc = fmaf(a[14], bf16_lo(w.r7), acc); acc = fmaf(a[15], bf16_hi(w.r7), acc); return acc; } __device__ __forceinline__ void fma_v_u4( float o0[8], float o1[8], float a0, float a1, float w0, float w1, const uint4 v ) { float lo, hi; lo = bf16_lo(v.x); hi = bf16_hi(v.x); o0[0] = fmaf(w0, lo, o0[0] * a0); o0[1] = fmaf(w0, hi, o0[1] * a0); o1[0] = fmaf(w1, lo, o1[0] * a1); o1[1] = fmaf(w1, hi, o1[1] * a1); lo = bf16_lo(v.y); hi = bf16_hi(v.y); o0[2] = fmaf(w0, lo, o0[2] * a0); o0[3] = fmaf(w0, hi, o0[3] * a0); o1[2] = fmaf(w1, lo, o1[2] * a1); o1[3] = fmaf(w1, hi, o1[3] * a1); lo = bf16_lo(v.z); hi = bf16_hi(v.z); o0[4] = fmaf(w0, lo, o0[4] * a0); o0[5] = fmaf(w0, hi, o0[5] * a0); o1[4] = fmaf(w1, lo, o1[4] * a1); o1[5] = fmaf(w1, hi, o1[5] * a1); lo = bf16_lo(v.w); hi = bf16_hi(v.w); o0[6] = fmaf(w0, lo, o0[6] * a0); o0[7] = fmaf(w0, hi, o0[7] * a0); o1[6] = fmaf(w1, lo, o1[6] * a1); o1[7] = fmaf(w1, hi, o1[7] * a1); } __device__ __forceinline__ void fma_v2_u4( float o0[8], float o1[8], float a0, float a1, float w0A, float w0B, float w1A, float w1B, const uint4 vA, const uint4 vB ) { float al, ah, bl, bh; al = bf16_lo(vA.x); ah = bf16_hi(vA.x); bl = bf16_lo(vB.x); bh = bf16_hi(vB.x); o0[0] = fmaf(w0B, bl, fmaf(w0A, al, o0[0] * a0)); o0[1] = fmaf(w0B, bh, fmaf(w0A, ah, o0[1] * a0)); o1[0] = fmaf(w1B, bl, fmaf(w1A, al, o1[0] * a1)); o1[1] = fmaf(w1B, bh, fmaf(w1A, ah, o1[1] * a1)); al = bf16_lo(vA.y); ah = bf16_hi(vA.y); bl = bf16_lo(vB.y); bh = bf16_hi(vB.y); o0[2] = fmaf(w0B, bl, fmaf(w0A, al, o0[2] * a0)); o0[3] = fmaf(w0B, bh, fmaf(w0A, ah, o0[3] * a0)); o1[2] = fmaf(w1B, bl, fmaf(w1A, al, o1[2] * a1)); o1[3] = fmaf(w1B, bh, fmaf(w1A, ah, o1[3] * a1)); al = bf16_lo(vA.z); ah = bf16_hi(vA.z); bl = bf16_lo(vB.z); bh = bf16_hi(vB.z); o0[4] = fmaf(w0B, bl, fmaf(w0A, al, o0[4] * a0)); o0[5] = fmaf(w0B, bh, fmaf(w0A, ah, o0[5] * a0)); o1[4] = fmaf(w1B, bl, fmaf(w1A, al, o1[4] * a1)); o1[5] = fmaf(w1B, bh, fmaf(w1A, ah, o1[5] * a1)); al = bf16_lo(vA.w); ah = bf16_hi(vA.w); bl = bf16_lo(vB.w); bh = bf16_hi(vB.w); o0[6] = fmaf(w0B, bl, fmaf(w0A, al, o0[6] * a0)); o0[7] = fmaf(w0B, bh, fmaf(w0A, ah, o0[7] * a0)); o1[6] = fmaf(w1B, bl, fmaf(w1A, al, o1[6] * a1)); o1[7] = fmaf(w1B, bh, fmaf(w1A, ah, o1[7] * a1)); } __device__ __forceinline__ void fma_v_u8( float o0[16], float o1[16], float a0, float a1, float w0, float w1, const u32x8 v ) { const uint32_t arr[8] = {v.r0, v.r1, v.r2, v.r3, v.r4, v.r5, v.r6, v.r7}; #pragma unroll for (int i = 0; i < 8; ++i) { float lo = bf16_lo(arr[i]), hi = bf16_hi(arr[i]); o0[2 * i] = fmaf(w0, lo, o0[2 * i] * a0); o0[2 * i + 1] = fmaf(w0, hi, o0[2 * i + 1] * a0); o1[2 * i] = fmaf(w1, lo, o1[2 * i] * a1); o1[2 * i + 1] = fmaf(w1, hi, o1[2 * i + 1] * a1); } } __device__ __forceinline__ void fma_v2_u8( float o0[16], float o1[16], float a0, float a1, float w0A, float w0B, float w1A, float w1B, const u32x8 vA, const u32x8 vB ) { const uint32_t aA[8] = {vA.r0, vA.r1, vA.r2, vA.r3, vA.r4, vA.r5, vA.r6, vA.r7}; const uint32_t aB[8] = {vB.r0, vB.r1, vB.r2, vB.r3, vB.r4, vB.r5, vB.r6, vB.r7}; #pragma unroll for (int i = 0; i < 8; ++i) { float al = bf16_lo(aA[i]), ah = bf16_hi(aA[i]); float bl = bf16_lo(aB[i]), bh = bf16_hi(aB[i]); o0[2 * i] = fmaf(w0B, bl, fmaf(w0A, al, o0[2 * i] * a0)); o0[2 * i + 1] = fmaf(w0B, bh, fmaf(w0A, ah, o0[2 * i + 1] * a0)); o1[2 * i] = fmaf(w1B, bl, fmaf(w1A, al, o1[2 * i] * a1)); o1[2 * i + 1] = fmaf(w1B, bh, fmaf(w1A, ah, o1[2 * i + 1] * a1)); } } __device__ __forceinline__ float warp_reduce_sum(float v) { #pragma unroll for (int m = 16; m > 0; m >>= 1) v += __shfl_xor_sync(0xffffffffu, v, m); return v; } __device__ __forceinline__ float warp_reduce_max(float v) { #pragma unroll for (int m = 16; m > 0; m >>= 1) v = fmaxf(v, __shfl_xor_sync(0xffffffffu, v, m)); return v; } __device__ __forceinline__ float hw_reduce_sum(float v, unsigned mask) { #pragma unroll for (int off = 8; off > 0; off >>= 1) v += __shfl_xor_sync(mask, v, off, 16); return v; } __device__ __forceinline__ float qw_reduce_sum(float v, unsigned mask) { #pragma unroll for (int off = 4; off > 0; off >>= 1) v += __shfl_xor_sync(mask, v, off, 8); return v; } __device__ __forceinline__ __nv_bfloat16 mix_half_bf16(__nv_bfloat16 a, __nv_bfloat16 b) { float ha = __bfloat162float(__float2bfloat16_rn(0.5f * __bfloat162float(a))); float hb = __bfloat162float(__float2bfloat16_rn(0.5f * __bfloat162float(b))); return __float2bfloat16_rn(ha + hb); } // ---------------- Exact PyTorch CPU MT19937 + Box-Muller + RoPE table ---------------- __device__ __forceinline__ uint32_t mt_twist_val(uint32_t u, uint32_t v) { uint32_t m = (u & 0x80000000u) | (v & 0x7fffffffu); return (m >> 1) ^ ((v & 1u) ? 0x9908b0dfu : 0u); } __device__ __forceinline__ uint32_t mt_temper(uint32_t y) { y ^= (y >> 11); y ^= (y << 7) & 0x9d2c5680u; y ^= (y << 15) & 0xefc60000u; y ^= (y >> 18); return y; } __device__ __forceinline__ void mt_twist_block(uint32_t* s, int tid) { uint32_t v1 = 0; if (tid < 227) v1 = s[tid + 397] ^ mt_twist_val(s[tid], s[tid + 1]); __syncthreads(); if (tid < 227) s[tid] = v1; __syncthreads(); uint32_t v2 = 0; if (tid < 227) { int j = tid + 227; v2 = s[tid] ^ mt_twist_val(s[j], s[j + 1]); } __syncthreads(); if (tid < 227) s[tid + 227] = v2; __syncthreads(); uint32_t v3 = 0; if (tid < 169) { int j = tid + 454; v3 = s[j - 227] ^ mt_twist_val(s[j], s[j + 1]); } __syncthreads(); if (tid < 169) s[tid + 454] = v3; __syncthreads(); if (tid == 0) s[623] = s[396] ^ mt_twist_val(s[623], s[0]); __syncthreads(); } __device__ __forceinline__ void mt_fill_step1024(uint32_t* s, int& cur, uint32_t* sm_u, int tid) { int dst = 0; while (dst < kHidden) { int avail = 624 - cur; int rem = kHidden - dst; int take = (rem < avail) ? rem : avail; for (int i = tid; i < take; i += 256) { sm_u[dst + i] = mt_temper(s[cur + i]); } dst += take; cur += take; __syncthreads(); if (cur == 624) { mt_twist_block(s, tid); cur = 0; } } } __global__ __launch_bounds__(256, 2) void prologue_step0_kernel( const __nv_bfloat16* __restrict__ iln0, Workspace ws ) { const int tid = threadIdx.x; const int warp = tid >> 5; const int lane = tid & 31; __shared__ uint32_t mt_s[624]; __shared__ uint32_t sm_u[kHidden]; __shared__ float sm_ss[8]; const DecodeParams* dp = ws.params; if (tid == 0) ws.ss_mid[0] = 0.0f; if (tid < kHalfHead) { float sn, cs; sincosf((float)(dp->start_pos) * c_rope_inv[tid], &sn, &cs); ws.rope_cos[tid] = cs; ws.rope_sin[tid] = sn; } int cur; if (dp->init_mt) { if (tid == 0) { mt_s[0] = (uint32_t)(dp->rng_seed & 0xffffffffu); for (int j = 1; j < 624; ++j) { mt_s[j] = 1812433253u * (mt_s[j - 1] ^ (mt_s[j - 1] >> 30)) + (uint32_t)j; } } __syncthreads(); mt_twist_block(mt_s, tid); cur = 0; } else { for (int j = tid; j < 624; j += 256) mt_s[j] = ws.mt_state[j]; cur = *ws.mt_cur; __syncthreads(); } mt_fill_step1024(mt_s, cur, sm_u, tid); for (int j = tid; j < 624; j += 256) ws.mt_state[j] = mt_s[j]; if (tid == 0) *ws.mt_cur = cur; constexpr uint32_t kMask = (1u << 24) - 1u; constexpr float kDiv = 1.0f / (float)(1u << 24); constexpr float kTwoPi = 6.28318530717958647692f; const __nv_bfloat16* h_in = dp->h_in; float local_ss = 0.0f; #pragma unroll for (int iter = 0; iter < 2; ++iter) { int p = tid + iter * 256; int grp = p >> 3; int j = p & 7; int idx0 = (grp << 4) + j; int idx1 = idx0 + 8; float u1 = 1.0f - (float)(sm_u[idx0] & kMask) * kDiv; float u2 = (float)(sm_u[idx1] & kMask) * kDiv; float radius = sqrtf(-2.0f * logf(u1)); float sn, cs; sincosf(kTwoPi * u2, &sn, &cs); __nv_bfloat16 n0 = __float2bfloat16_rn(radius * cs); __nv_bfloat16 n1 = __float2bfloat16_rn(radius * sn); __nv_bfloat16 x0 = mix_half_bf16(n0, h_in[idx0]); __nv_bfloat16 x1 = mix_half_bf16(n1, h_in[idx1]); ws.x_curr[idx0] = x0; ws.x_curr[idx1] = x1; float f0 = __bfloat162float(x0); float f1 = __bfloat162float(x1); ws.x_ln[idx0] = __float2half_rn(f0 * __bfloat162float(__ldg(&iln0[idx0]))); ws.x_ln[idx1] = __float2half_rn(f1 * __bfloat162float(__ldg(&iln0[idx1]))); local_ss += f0 * f0 + f1 * f1; } float ws_ss = warp_reduce_sum(local_ss); if (lane == 0) sm_ss[warp] = ws_ss; __syncthreads(); if (tid == 0) { ws.ss_x[0] = sm_ss[0] + sm_ss[1] + sm_ss[2] + sm_ss[3] + sm_ss[4] + sm_ss[5] + sm_ss[6] + sm_ss[7]; } pdl_launch(); } __global__ __launch_bounds__(256, 2) void mt_u32_rest_kernel(Workspace ws, int chunk_steps) { const int tid = threadIdx.x; __shared__ uint32_t mt_s[624]; for (int j = tid; j < 624; j += 256) mt_s[j] = ws.mt_state[j]; int cur = *ws.mt_cur; __syncthreads(); int total = (chunk_steps - 1) * kHidden; int dst = 0; uint32_t* base_out = ws.raw_u32 + kHidden; while (dst < total) { int avail = 624 - cur; int rem = total - dst; int take = (rem < avail) ? rem : avail; if (((cur | dst | take) & 3) == 0) { int take4 = take >> 2; uint4* out4 = reinterpret_cast(base_out + dst); const uint4* in4 = reinterpret_cast(mt_s + cur); if (tid < take4) { uint4 v = in4[tid]; out4[tid] = make_uint4(mt_temper(v.x), mt_temper(v.y), mt_temper(v.z), mt_temper(v.w)); } } else { for (int i = tid; i < take; i += 256) { base_out[dst + i] = mt_temper(mt_s[cur + i]); } } dst += take; cur += take; __syncthreads(); if (cur == 624) { mt_twist_block(mt_s, tid); cur = 0; } } for (int j = tid; j < 624; j += 256) ws.mt_state[j] = mt_s[j]; if (tid == 0) *ws.mt_cur = cur; } __global__ __launch_bounds__(256, 2) void bm_rope_rest_kernel(Workspace ws, int chunk_steps) { constexpr uint32_t kMask = (1u << 24) - 1u; constexpr float kDiv = 1.0f / (float)(1u << 24); constexpr float kTwoPi = 6.28318530717958647692f; const int gtid = blockIdx.x * 256 + threadIdx.x; const int gstride = gridDim.x * 256; int total_pairs = (chunk_steps - 1) * (kHidden / 2); for (int p = gtid; p < total_pairs; p += gstride) { int step = 1 + (p >> 9); int p_in = p & 511; int grp = p_in >> 3; int j = p_in & 7; int base = step * kHidden + (grp << 4); float u1 = 1.0f - (float)(ws.raw_u32[base + j] & kMask) * kDiv; float u2 = (float)(ws.raw_u32[base + j + 8] & kMask) * kDiv; float radius = sqrtf(-2.0f * logf(u1)); float sn, cs; sincosf(kTwoPi * u2, &sn, &cs); ws.noise_buf[base + j] = __float2bfloat16_rn(radius * cs); ws.noise_buf[base + j + 8] = __float2bfloat16_rn(radius * sn); } int total_rope = (chunk_steps - 1) * kHalfHead; int start_pos = ws.params->start_pos; for (int idx = gtid; idx < total_rope; idx += gstride) { int step = 1 + (idx >> 6); int d = idx & 63; float sn, cs; sincosf((float)(start_pos + step) * c_rope_inv[d], &sn, &cs); ws.rope_cos[step * kHalfHead + d] = cs; ws.rope_sin[step * kHalfHead + d] = sn; } } // ========================================================================= // KERNEL 1: QKV Projection (256 x 512, direct weight pointer, 4-byte ss_x!) // ========================================================================= __global__ __launch_bounds__(512, 2) void qkv_pdl_kernel( const __nv_bfloat16* __restrict__ qkv_w, __half* x_ln, float* ss_x, float* q_raw, float* k_raw, DecodeParams* dp, int layer_idx, int step_idx ) { const int tid = threadIdx.x; const int warp = tid >> 5; const int lane = tid & 31; const int r = (blockIdx.x << 4) + warp; const uint4* row = reinterpret_cast(qkv_w + (size_t)r * kHidden); uint4 w0 = ld_cg_v4(row + lane); uint4 w1 = ld_cg_v4(row + 32 + lane); uint4 w2 = ld_cg_v4(row + 64 + lane); uint4 w3 = ld_cg_v4(row + 96 + lane); pdl_wait(); pdl_launch(); const uint4* x4 = reinterpret_cast(x_ln); float inv_rms = rsqrtf(ss_x[0] * (1.0f / (float)kHidden) + kEps); float acc = dot8_f16_bf16(x4[lane], w0) + dot8_f16_bf16(x4[32 + lane], w1) + dot8_f16_bf16(x4[64 + lane], w2) + dot8_f16_bf16(x4[96 + lane], w3); acc = warp_reduce_sum(acc) * inv_rms; if (lane == 0) { if (r < 2048) { q_raw[r] = acc; } else if (r < 3072) { k_raw[r - 2048] = acc; } else { int v_idx = r - 3072; int pos = dp->start_pos + step_idx; __nv_bfloat16* v_cache = dp->kvs[layer_idx].v; v_cache[((int64_t)(v_idx >> 7) * dp->max_seq + pos) * kHeadDim + (v_idx & 127)] = __float2bfloat16_rn(acc); } } } // ========================================================================= // KERNEL 2a: Short/Medium Context Attention (128-bit half-warp + in-warp merge!) // Halves SMEM (10 KB) and halves block reduction (8 warps instead of 16 half-warps). // ========================================================================= template __global__ __launch_bounds__(256, 3) void attn_pdl_v4_kernel( const __nv_bfloat16* __restrict__ q_norm, const __nv_bfloat16* __restrict__ k_norm, float* q_raw, float* k_raw, float* rope_cos, float* rope_sin, float* part_o, float2* part_ml, DecodeParams* dp, int layer_idx, int step_idx ) { const int bid = blockIdx.x; const int tid = threadIdx.x; const int warp = tid >> 5; const int lane = tid & 31; const int hw = tid >> 4; const int lane16 = tid & 15; const unsigned hw_mask = (tid & 16) ? 0xffff0000u : 0x0000ffffu; const int kv_h = bid / SPLITS; const int split_id = bid % SPLITS; __shared__ float sm_q0[kHeadDim]; __shared__ float sm_q1[kHeadDim]; __shared__ float sm_k_last[kHeadDim]; __shared__ float sm_o0[8][kSmemPadDim]; // 2x smaller thanks to in-warp merge! __shared__ float sm_o1[8][kSmemPadDim]; __shared__ float sm_m0[8], sm_l0[8], sm_m1[8], sm_l1[8]; __shared__ float sm_sc0[8], sm_sc1[8], sm_ml[4]; const int pos = dp->start_pos + step_idx; const int seq_len = pos + 1; const int active_splits = min(SPLITS, max(1, (seq_len + 15) / 16)); if (split_id >= active_splits) { pdl_wait(); pdl_launch(); return; } const int64_t max_seq = dp->max_seq; const LayerKV kv = dp->kvs[layer_idx]; const bool is_last_split = (split_id == active_splits - 1); float rc0 = 0.0f, rs0 = 0.0f, rc1 = 0.0f, rs1 = 0.0f; float nw0 = 0.0f, nw1 = 0.0f, nw2 = 0.0f, nw3 = 0.0f; if (warp < 2 || (warp == 2 && is_last_split)) { const float* rcp = rope_cos + step_idx * kHalfHead; const float* rsp = rope_sin + step_idx * kHalfHead; rc0 = rcp[lane]; rs0 = rsp[lane]; rc1 = rcp[lane + 32]; rs1 = rsp[lane + 32]; const __nv_bfloat16* nw = (warp < 2) ? q_norm : k_norm; nw0 = __bfloat162float(__ldg(nw + lane)); nw1 = __bfloat162float(__ldg(nw + lane + 32)); nw2 = __bfloat162float(__ldg(nw + lane + 64)); nw3 = __bfloat162float(__ldg(nw + lane + 96)); } const int t_start = (int)(((int64_t)seq_len * split_id) / active_splits); const int t_end = (int)(((int64_t)seq_len * (split_id + 1)) / active_splits); const int t_loop_end = (t_end == seq_len) ? (seq_len - 1) : t_end; const __nv_bfloat16* kh = kv.k + (int64_t)kv_h * max_seq * kHeadDim; const __nv_bfloat16* vh = kv.v + (int64_t)kv_h * max_seq * kHeadDim; pdl_wait(); if (warp < 2) { const float* qp = q_raw + (2 * kv_h + warp) * kHeadDim; float v0 = qp[lane], v1 = qp[lane + 32], v2 = qp[lane + 64], v3 = qp[lane + 96]; float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3); float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps); v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3; float* dst = (warp == 0) ? sm_q0 : sm_q1; dst[lane] = v0 * rc0 - v2 * rs0; dst[lane + 64] = v0 * rs0 + v2 * rc0; dst[lane + 32] = v1 * rc1 - v3 * rs1; dst[lane + 96] = v1 * rs1 + v3 * rc1; } else if (warp == 2 && is_last_split) { const float* kp = k_raw + kv_h * kHeadDim; float v0 = kp[lane], v1 = kp[lane + 32], v2 = kp[lane + 64], v3 = kp[lane + 96]; float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3); float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps); v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3; __nv_bfloat16 k0 = __float2bfloat16_rn(v0 * rc0 - v2 * rs0); __nv_bfloat16 k2 = __float2bfloat16_rn(v0 * rs0 + v2 * rc0); __nv_bfloat16 k1 = __float2bfloat16_rn(v1 * rc1 - v3 * rs1); __nv_bfloat16 k3 = __float2bfloat16_rn(v1 * rs1 + v3 * rc1); __nv_bfloat16* k_dst = kv.k + ((int64_t)kv_h * max_seq + pos) * kHeadDim; k_dst[lane] = k0; k_dst[lane + 32] = k1; k_dst[lane + 64] = k2; k_dst[lane + 96] = k3; sm_k_last[lane] = __bfloat162float(k0); sm_k_last[lane + 32] = __bfloat162float(k1); sm_k_last[lane + 64] = __bfloat162float(k2); sm_k_last[lane + 96] = __bfloat162float(k3); } __syncthreads(); const int d8 = lane16 * 8; float q0[8], q1[8], o0[8], o1[8]; #pragma unroll for (int i = 0; i < 8; ++i) { q0[i] = sm_q0[d8 + i]; q1[i] = sm_q1[d8 + i]; o0[i] = 0.0f; o1[i] = 0.0f; } float m0 = -1e30f, l0 = 0.0f; float m1 = -1e30f, l1 = 0.0f; int t = t_start + hw; if constexpr (USE_CS) { if constexpr (UNROLL2X) { for (; t + 16 < t_loop_end; t += 32) { uint4 kA = ld_kv_cs_v4(kh + (int64_t)t * kHeadDim + d8); uint4 vA = ld_kv_cs_v4(vh + (int64_t)t * kHeadDim + d8); uint4 kB = ld_kv_cs_v4(kh + (int64_t)(t + 16) * kHeadDim + d8); uint4 vB = ld_kv_cs_v4(vh + (int64_t)(t + 16) * kHeadDim + d8); float s0A = hw_reduce_sum(dot8_f_u4(q0, kA), hw_mask) * kAttnScaleLog2E; float s1A = hw_reduce_sum(dot8_f_u4(q1, kA), hw_mask) * kAttnScaleLog2E; float s0B = hw_reduce_sum(dot8_f_u4(q0, kB), hw_mask) * kAttnScaleLog2E; float s1B = hw_reduce_sum(dot8_f_u4(q1, kB), hw_mask) * kAttnScaleLog2E; float m0n = fmaxf(m0, fmaxf(s0A, s0B)); float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n), w0B = ex2_fast(s0B - m0n); l0 = l0 * a0 + w0A + w0B; m0 = m0n; float m1n = fmaxf(m1, fmaxf(s1A, s1B)); float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n), w1B = ex2_fast(s1B - m1n); l1 = l1 * a1 + w1A + w1B; m1 = m1n; fma_v2_u4(o0, o1, a0, a1, w0A, w0B, w1A, w1B, vA, vB); } } for (; t < t_loop_end; t += 16) { uint4 kA = ld_kv_cs_v4(kh + (int64_t)t * kHeadDim + d8); uint4 vA = ld_kv_cs_v4(vh + (int64_t)t * kHeadDim + d8); float s0A = hw_reduce_sum(dot8_f_u4(q0, kA), hw_mask) * kAttnScaleLog2E; float s1A = hw_reduce_sum(dot8_f_u4(q1, kA), hw_mask) * kAttnScaleLog2E; float m0n = fmaxf(m0, s0A); float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n); l0 = l0 * a0 + w0A; m0 = m0n; float m1n = fmaxf(m1, s1A); float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n); l1 = l1 * a1 + w1A; m1 = m1n; fma_v_u4(o0, o1, a0, a1, w0A, w1A, vA); } } else { for (; t < t_loop_end; t += 16) { uint4 kA = *reinterpret_cast(kh + (int64_t)t * kHeadDim + d8); uint4 vA = *reinterpret_cast(vh + (int64_t)t * kHeadDim + d8); float s0A = hw_reduce_sum(dot8_f_u4(q0, kA), hw_mask) * kAttnScaleLog2E; float s1A = hw_reduce_sum(dot8_f_u4(q1, kA), hw_mask) * kAttnScaleLog2E; float m0n = fmaxf(m0, s0A); float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n); l0 = l0 * a0 + w0A; m0 = m0n; float m1n = fmaxf(m1, s1A); float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n); l1 = l1 * a1 + w1A; m1 = m1n; fma_v_u4(o0, o1, a0, a1, w0A, w1A, vA); } } if (t_end == seq_len && hw == 0) { float d0 = 0.0f, d1 = 0.0f; #pragma unroll for (int i = 0; i < 8; ++i) { float kf = sm_k_last[d8 + i]; d0 = fmaf(q0[i], kf, d0); d1 = fmaf(q1[i], kf, d1); } float s0A = hw_reduce_sum(d0, hw_mask) * kAttnScaleLog2E; float s1A = hw_reduce_sum(d1, hw_mask) * kAttnScaleLog2E; uint4 vA = *reinterpret_cast(vh + (int64_t)pos * kHeadDim + d8); float m0n = fmaxf(m0, s0A); float a0 = (l0 == 0.0f) ? 0.0f : ex2_fast(m0 - m0n); float w0A = ex2_fast(s0A - m0n); l0 = l0 * a0 + w0A; m0 = m0n; float m1n = fmaxf(m1, s1A); float a1 = (l1 == 0.0f) ? 0.0f : ex2_fast(m1 - m1n); float w1A = ex2_fast(s1A - m1n); l1 = l1 * a1 + w1A; m1 = m1n; fma_v_u4(o0, o1, a0, a1, w0A, w1A, vA); } pdl_launch(); // In-warp merge of the 2 sibling half-warps IN REGISTERS before touching SMEM! { float m0_ot = __shfl_xor_sync(0xffffffffu, m0, 16); float l0_ot = __shfl_xor_sync(0xffffffffu, l0, 16); float m0_w = fmaxf((l0 > 0.0f) ? m0 : -1e30f, (l0_ot > 0.0f) ? m0_ot : -1e30f); float s0_me = (l0 > 0.0f) ? ex2_fast(m0 - m0_w) : 0.0f; float s0_ot = (l0_ot > 0.0f) ? ex2_fast(m0_ot - m0_w) : 0.0f; float l0_w = l0 * s0_me + l0_ot * s0_ot; float m1_ot = __shfl_xor_sync(0xffffffffu, m1, 16); float l1_ot = __shfl_xor_sync(0xffffffffu, l1, 16); float m1_w = fmaxf((l1 > 0.0f) ? m1 : -1e30f, (l1_ot > 0.0f) ? m1_ot : -1e30f); float s1_me = (l1 > 0.0f) ? ex2_fast(m1 - m1_w) : 0.0f; float s1_ot = (l1_ot > 0.0f) ? ex2_fast(m1_ot - m1_w) : 0.0f; float l1_w = l1 * s1_me + l1_ot * s1_ot; #pragma unroll for (int i = 0; i < 8; ++i) { float ot0 = __shfl_xor_sync(0xffffffffu, o0[i], 16); float ot1 = __shfl_xor_sync(0xffffffffu, o1[i], 16); o0[i] = fmaf(s0_ot, ot0, o0[i] * s0_me); o1[i] = fmaf(s1_ot, ot1, o1[i] * s1_me); } if (lane < 16) { *reinterpret_cast(&sm_o0[warp][d8]) = make_float4(o0[0], o0[1], o0[2], o0[3]); *reinterpret_cast(&sm_o0[warp][d8 + 4]) = make_float4(o0[4], o0[5], o0[6], o0[7]); *reinterpret_cast(&sm_o1[warp][d8]) = make_float4(o1[0], o1[1], o1[2], o1[3]); *reinterpret_cast(&sm_o1[warp][d8 + 4]) = make_float4(o1[4], o1[5], o1[6], o1[7]); if (lane == 0) { sm_m0[warp] = m0_w; sm_l0[warp] = l0_w; sm_m1[warp] = m1_w; sm_l1[warp] = l1_w; } } } __syncthreads(); if (warp < 2) { const float* ms = (warp == 0) ? sm_m0 : sm_m1; const float* ls = (warp == 0) ? sm_l0 : sm_l1; float lh = (lane < 8) ? ls[lane] : 0.0f; float mh = (lane < 8 && lh > 0.0f) ? ms[lane] : -1e30f; float mm = mh; #pragma unroll for (int off = 4; off > 0; off >>= 1) mm = fmaxf(mm, __shfl_xor_sync(0xffffffffu, mm, off, 8)); float w = (lh > 0.0f) ? ex2_fast(mh - mm) : 0.0f; float lt = lh * w; #pragma unroll for (int off = 4; off > 0; off >>= 1) lt += __shfl_xor_sync(0xffffffffu, lt, off, 8); if (lane < 8) { if (warp == 0) sm_sc0[lane] = w; else sm_sc1[lane] = w; } if (lane == 0) { sm_ml[warp * 2 + 0] = mm; sm_ml[warp * 2 + 1] = lt; } } __syncthreads(); const int qsub = tid >> 7; const int d = tid & 127; const float* sc = (qsub == 0) ? sm_sc0 : sm_sc1; float o_acc = 0.0f; if (qsub == 0) { #pragma unroll for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o0[w][d], o_acc); } else { #pragma unroll for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o1[w][d], o_acc); } const int qh = kv_h * 2 + qsub; st_cg_f32(&part_o[(qh * 48 + split_id) * kHeadDim + d], o_acc); if (d == 0) { st_cg_f2(&part_ml[qh * 48 + split_id], make_float2(sm_ml[qsub * 2 + 0], sm_ml[qsub * 2 + 1])); } } // ========================================================================= // KERNEL 2b: Long-Context Attention (SM120 256-bit quarter-warp + 4-way in-warp merge!) // Single-pass 8-warp SMEM reduction (10 KB total SMEM, 4x fewer SMEM reads)! // ========================================================================= template __global__ __launch_bounds__(256, 2) void attn_pdl_v8_kernel( const __nv_bfloat16* __restrict__ q_norm, const __nv_bfloat16* __restrict__ k_norm, float* q_raw, float* k_raw, float* rope_cos, float* rope_sin, float* part_o, float2* part_ml, DecodeParams* dp, int layer_idx, int step_idx ) { const int bid = blockIdx.x; const int tid = threadIdx.x; const int warp = tid >> 5; const int lane = tid & 31; const int qw = tid >> 3; const int l8 = tid & 7; const unsigned qw_mask = 0xffu << (tid & 24); const int kv_h = bid / SPLITS; const int split_id = bid % SPLITS; __shared__ float sm_q0[kHeadDim]; __shared__ float sm_q1[kHeadDim]; __shared__ float sm_k_last[kHeadDim]; __shared__ float sm_o0[8][kSmemPadDim]; __shared__ float sm_o1[8][kSmemPadDim]; __shared__ float sm_m0[8], sm_l0[8], sm_m1[8], sm_l1[8]; __shared__ float sm_sc0[8], sm_sc1[8], sm_ml[4]; const int pos = dp->start_pos + step_idx; const int seq_len = pos + 1; const int active_splits = min(SPLITS, max(1, (seq_len + 31) / 32)); if (split_id >= active_splits) { pdl_wait(); pdl_launch(); return; } const int64_t max_seq = dp->max_seq; const LayerKV kv = dp->kvs[layer_idx]; const bool is_last_split = (split_id == active_splits - 1); float rc0 = 0.0f, rs0 = 0.0f, rc1 = 0.0f, rs1 = 0.0f; float nw0 = 0.0f, nw1 = 0.0f, nw2 = 0.0f, nw3 = 0.0f; if (warp < 2 || (warp == 2 && is_last_split)) { const float* rcp = rope_cos + step_idx * kHalfHead; const float* rsp = rope_sin + step_idx * kHalfHead; rc0 = rcp[lane]; rs0 = rsp[lane]; rc1 = rcp[lane + 32]; rs1 = rsp[lane + 32]; const __nv_bfloat16* nw = (warp < 2) ? q_norm : k_norm; nw0 = __bfloat162float(__ldg(nw + lane)); nw1 = __bfloat162float(__ldg(nw + lane + 32)); nw2 = __bfloat162float(__ldg(nw + lane + 64)); nw3 = __bfloat162float(__ldg(nw + lane + 96)); } const int t_start = (int)(((int64_t)seq_len * split_id) / active_splits); const int t_end = (int)(((int64_t)seq_len * (split_id + 1)) / active_splits); const int t_loop_end = (t_end == seq_len) ? (seq_len - 1) : t_end; const __nv_bfloat16* kh = kv.k + (int64_t)kv_h * max_seq * kHeadDim; const __nv_bfloat16* vh = kv.v + (int64_t)kv_h * max_seq * kHeadDim; pdl_wait(); if (warp < 2) { const float* qp = q_raw + (2 * kv_h + warp) * kHeadDim; float v0 = qp[lane], v1 = qp[lane + 32], v2 = qp[lane + 64], v3 = qp[lane + 96]; float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3); float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps); v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3; float* dst = (warp == 0) ? sm_q0 : sm_q1; dst[lane] = v0 * rc0 - v2 * rs0; dst[lane + 64] = v0 * rs0 + v2 * rc0; dst[lane + 32] = v1 * rc1 - v3 * rs1; dst[lane + 96] = v1 * rs1 + v3 * rc1; } else if (warp == 2 && is_last_split) { const float* kp = k_raw + kv_h * kHeadDim; float v0 = kp[lane], v1 = kp[lane + 32], v2 = kp[lane + 64], v3 = kp[lane + 96]; float ss = warp_reduce_sum(v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3); float inv = rsqrtf(ss * (1.0f / (float)kHeadDim) + kEps); v0 *= inv * nw0; v1 *= inv * nw1; v2 *= inv * nw2; v3 *= inv * nw3; __nv_bfloat16 k0 = __float2bfloat16_rn(v0 * rc0 - v2 * rs0); __nv_bfloat16 k2 = __float2bfloat16_rn(v0 * rs0 + v2 * rc0); __nv_bfloat16 k1 = __float2bfloat16_rn(v1 * rc1 - v3 * rs1); __nv_bfloat16 k3 = __float2bfloat16_rn(v1 * rs1 + v3 * rc1); __nv_bfloat16* k_dst = kv.k + ((int64_t)kv_h * max_seq + pos) * kHeadDim; k_dst[lane] = k0; k_dst[lane + 32] = k1; k_dst[lane + 64] = k2; k_dst[lane + 96] = k3; sm_k_last[lane] = __bfloat162float(k0); sm_k_last[lane + 32] = __bfloat162float(k1); sm_k_last[lane + 64] = __bfloat162float(k2); sm_k_last[lane + 96] = __bfloat162float(k3); } __syncthreads(); const int d16 = l8 * 16; float q0[16], o0[16], o1[16]; #pragma unroll for (int i = 0; i < 16; ++i) { q0[i] = sm_q0[d16 + i]; o0[i] = 0.0f; o1[i] = 0.0f; } const float* q1 = sm_q1 + d16; float m0 = -1e30f, l0 = 0.0f; float m1 = -1e30f, l1 = 0.0f; int t = t_start + qw; for (; t + 32 < t_loop_end; t += 64) { u32x8 kA = ld_kv_v8(kh + (int64_t)t * kHeadDim + d16); u32x8 vA = ld_kv_v8(vh + (int64_t)t * kHeadDim + d16); u32x8 kB = ld_kv_v8(kh + (int64_t)(t + 32) * kHeadDim + d16); u32x8 vB = ld_kv_v8(vh + (int64_t)(t + 32) * kHeadDim + d16); float s0A = qw_reduce_sum(dot16_f_u8(q0, kA), qw_mask) * kAttnScaleLog2E; float s1A = qw_reduce_sum(dot16_f_u8(q1, kA), qw_mask) * kAttnScaleLog2E; float s0B = qw_reduce_sum(dot16_f_u8(q0, kB), qw_mask) * kAttnScaleLog2E; float s1B = qw_reduce_sum(dot16_f_u8(q1, kB), qw_mask) * kAttnScaleLog2E; float m0n = fmaxf(m0, fmaxf(s0A, s0B)); float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n), w0B = ex2_fast(s0B - m0n); l0 = l0 * a0 + w0A + w0B; m0 = m0n; float m1n = fmaxf(m1, fmaxf(s1A, s1B)); float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n), w1B = ex2_fast(s1B - m1n); l1 = l1 * a1 + w1A + w1B; m1 = m1n; fma_v2_u8(o0, o1, a0, a1, w0A, w0B, w1A, w1B, vA, vB); } for (; t < t_loop_end; t += 32) { u32x8 kA = ld_kv_v8(kh + (int64_t)t * kHeadDim + d16); u32x8 vA = ld_kv_v8(vh + (int64_t)t * kHeadDim + d16); float s0A = qw_reduce_sum(dot16_f_u8(q0, kA), qw_mask) * kAttnScaleLog2E; float s1A = qw_reduce_sum(dot16_f_u8(q1, kA), qw_mask) * kAttnScaleLog2E; float m0n = fmaxf(m0, s0A); float a0 = ex2_fast(m0 - m0n), w0A = ex2_fast(s0A - m0n); l0 = l0 * a0 + w0A; m0 = m0n; float m1n = fmaxf(m1, s1A); float a1 = ex2_fast(m1 - m1n), w1A = ex2_fast(s1A - m1n); l1 = l1 * a1 + w1A; m1 = m1n; fma_v_u8(o0, o1, a0, a1, w0A, w1A, vA); } if (t_end == seq_len && qw == 0) { float d0 = 0.0f, d1 = 0.0f; #pragma unroll for (int i = 0; i < 16; ++i) { float kf = sm_k_last[d16 + i]; d0 = fmaf(q0[i], kf, d0); d1 = fmaf(q1[i], kf, d1); } float s0A = qw_reduce_sum(d0, qw_mask) * kAttnScaleLog2E; float s1A = qw_reduce_sum(d1, qw_mask) * kAttnScaleLog2E; const uint4* vp = reinterpret_cast(vh + (int64_t)pos * kHeadDim + d16); uint4 v_lo = vp[0], v_hi = vp[1]; u32x8 vA{v_lo.x, v_lo.y, v_lo.z, v_lo.w, v_hi.x, v_hi.y, v_hi.z, v_hi.w}; float m0n = fmaxf(m0, s0A); float a0 = (l0 == 0.0f) ? 0.0f : ex2_fast(m0 - m0n); float w0A = ex2_fast(s0A - m0n); l0 = l0 * a0 + w0A; m0 = m0n; float m1n = fmaxf(m1, s1A); float a1 = (l1 == 0.0f) ? 0.0f : ex2_fast(m1 - m1n); float w1A = ex2_fast(s1A - m1n); l1 = l1 * a1 + w1A; m1 = m1n; fma_v_u8(o0, o1, a0, a1, w0A, w1A, vA); } pdl_launch(); // 4-way in-warp quarter-warp merge (stride 8 then stride 16) in registers! #pragma unroll for (int stride = 8; stride <= 16; stride <<= 1) { float m0_ot = __shfl_xor_sync(0xffffffffu, m0, stride); float l0_ot = __shfl_xor_sync(0xffffffffu, l0, stride); float m0_w = fmaxf((l0 > 0.0f) ? m0 : -1e30f, (l0_ot > 0.0f) ? m0_ot : -1e30f); float s0_me = (l0 > 0.0f) ? ex2_fast(m0 - m0_w) : 0.0f; float s0_ot = (l0_ot > 0.0f) ? ex2_fast(m0_ot - m0_w) : 0.0f; l0 = l0 * s0_me + l0_ot * s0_ot; m0 = m0_w; float m1_ot = __shfl_xor_sync(0xffffffffu, m1, stride); float l1_ot = __shfl_xor_sync(0xffffffffu, l1, stride); float m1_w = fmaxf((l1 > 0.0f) ? m1 : -1e30f, (l1_ot > 0.0f) ? m1_ot : -1e30f); float s1_me = (l1 > 0.0f) ? ex2_fast(m1 - m1_w) : 0.0f; float s1_ot = (l1_ot > 0.0f) ? ex2_fast(m1_ot - m1_w) : 0.0f; l1 = l1 * s1_me + l1_ot * s1_ot; m1 = m1_w; #pragma unroll for (int i = 0; i < 16; ++i) { float ot0 = __shfl_xor_sync(0xffffffffu, o0[i], stride); float ot1 = __shfl_xor_sync(0xffffffffu, o1[i], stride); o0[i] = fmaf(s0_ot, ot0, o0[i] * s0_me); o1[i] = fmaf(s1_ot, ot1, o1[i] * s1_me); } } if (lane < 8) { *reinterpret_cast(&sm_o0[warp][d16 + 0]) = make_float4(o0[0], o0[1], o0[2], o0[3]); *reinterpret_cast(&sm_o0[warp][d16 + 4]) = make_float4(o0[4], o0[5], o0[6], o0[7]); *reinterpret_cast(&sm_o0[warp][d16 + 8]) = make_float4(o0[8], o0[9], o0[10], o0[11]); *reinterpret_cast(&sm_o0[warp][d16 + 12]) = make_float4(o0[12], o0[13], o0[14], o0[15]); *reinterpret_cast(&sm_o1[warp][d16 + 0]) = make_float4(o1[0], o1[1], o1[2], o1[3]); *reinterpret_cast(&sm_o1[warp][d16 + 4]) = make_float4(o1[4], o1[5], o1[6], o1[7]); *reinterpret_cast(&sm_o1[warp][d16 + 8]) = make_float4(o1[8], o1[9], o1[10], o1[11]); *reinterpret_cast(&sm_o1[warp][d16 + 12]) = make_float4(o1[12], o1[13], o1[14], o1[15]); if (lane == 0) { sm_m0[warp] = m0; sm_l0[warp] = l0; sm_m1[warp] = m1; sm_l1[warp] = l1; } } __syncthreads(); if (warp < 2) { const float* ms = (warp == 0) ? sm_m0 : sm_m1; const float* ls = (warp == 0) ? sm_l0 : sm_l1; float lh = (lane < 8) ? ls[lane] : 0.0f; float mh = (lane < 8 && lh > 0.0f) ? ms[lane] : -1e30f; float mm = mh; #pragma unroll for (int off = 4; off > 0; off >>= 1) mm = fmaxf(mm, __shfl_xor_sync(0xffffffffu, mm, off, 8)); float w = (lh > 0.0f) ? ex2_fast(mh - mm) : 0.0f; float lt = lh * w; #pragma unroll for (int off = 4; off > 0; off >>= 1) lt += __shfl_xor_sync(0xffffffffu, lt, off, 8); if (lane < 8) { if (warp == 0) sm_sc0[lane] = w; else sm_sc1[lane] = w; } if (lane == 0) { sm_ml[warp * 2 + 0] = mm; sm_ml[warp * 2 + 1] = lt; } } __syncthreads(); const int qsub = tid >> 7; const int d = tid & 127; const float* sc = (qsub == 0) ? sm_sc0 : sm_sc1; float o_acc = 0.0f; if (qsub == 0) { #pragma unroll for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o0[w][d], o_acc); } else { #pragma unroll for (int w = 0; w < 8; ++w) o_acc = fmaf(sc[w], sm_o1[w][d], o_acc); } const int qh = kv_h * 2 + qsub; st_cg_f32(&part_o[(qh * 48 + split_id) * kHeadDim + d], o_acc); if (d == 0) { st_cg_f2(&part_ml[qh * 48 + split_id], make_float2(sm_ml[qsub * 2 + 0], sm_ml[qsub * 2 + 1])); } } // ========================================================================= // KERNEL 2c: Single-L2-RTT 32-Block Split Reduction (2 blks/Q-head x 512 thr) // 32 reduce blocks + 128 o_proj blocks = 160 <= 188 SMs (zero SM contention)! // Issues part_o ld_cg_f4 CONCURRENTLY with Warp-0 part_ml softmax scaling! // ========================================================================= template __global__ __launch_bounds__(512, 2) void reduce_pdl_kernel( float* part_o, float2* part_ml, __half* attn_out, DecodeParams* dp, int step_idx ) { const int qh = blockIdx.x >> 1; const int part = blockIdx.x & 1; const int tid = threadIdx.x; const int warp = tid >> 5; const int lane = tid & 31; const int hw = tid >> 4; // 0..31 half-warps const int l16 = tid & 15; // 0..15 float4 lanes (64 dims/blk) const int col4 = part * 16 + l16; __shared__ float sm_sc[64]; __shared__ float sm_f[2048]; const int seq_len = dp->start_pos + step_idx + 1; const int active_splits = min(SPLITS, max(1, (seq_len + STRIDE_TOK - 1) / STRIDE_TOK)); pdl_launch(); pdl_wait(); // Overlap part_o loads with Warp-0 part_ml loads -> ONE L2 round-trip instead of TWO! const float4* ob4 = reinterpret_cast(part_o + (size_t)qh * 48 * kHeadDim); float4 v0 = (hw < active_splits) ? ld_cg_f4(ob4 + hw * 32 + col4) : make_float4(0.0f, 0.0f, 0.0f, 0.0f); float4 v1 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); if constexpr (SPLITS > 32) { if (hw + 32 < active_splits) v1 = ld_cg_f4(ob4 + (hw + 32) * 32 + col4); } if (warp == 0) { const float2* mlp = part_ml + qh * 48; int sA = lane; float2 mlA = (sA < active_splits) ? ld_cg_f2(mlp + sA) : make_float2(-1e30f, 0.0f); float mA = (mlA.y > 0.0f) ? mlA.x : -1e30f; float lA = mlA.y; if constexpr (SPLITS <= 32) { float mm = warp_reduce_max(mA); float wA = (lA > 0.0f) ? ex2_fast(mA - mm) : 0.0f; float lt = warp_reduce_sum(lA * wA); float inv = (lt > 0.0f) ? (1.0f / lt) : 0.0f; sm_sc[sA] = (sA < active_splits) ? (wA * inv) : 0.0f; } else { int sB = lane + 32; float2 mlB = (sB < active_splits) ? ld_cg_f2(mlp + sB) : make_float2(-1e30f, 0.0f); float mB = (mlB.y > 0.0f) ? mlB.x : -1e30f; float lB = mlB.y; float mm = warp_reduce_max(fmaxf(mA, mB)); float wA = (lA > 0.0f) ? ex2_fast(mA - mm) : 0.0f; float wB = (lB > 0.0f) ? ex2_fast(mB - mm) : 0.0f; float lt = warp_reduce_sum(lA * wA + lB * wB); float inv = (lt > 0.0f) ? (1.0f / lt) : 0.0f; sm_sc[sA] = (sA < active_splits) ? (wA * inv) : 0.0f; sm_sc[sB] = (sB < active_splits) ? (wB * inv) : 0.0f; } } __syncthreads(); float w0 = sm_sc[hw]; float ax = w0 * v0.x, ay = w0 * v0.y, az = w0 * v0.z, aw = w0 * v0.w; if constexpr (SPLITS > 32) { if (hw + 32 < active_splits) { float w1 = sm_sc[hw + 32]; ax = fmaf(w1, v1.x, ax); ay = fmaf(w1, v1.y, ay); az = fmaf(w1, v1.z, az); aw = fmaf(w1, v1.w, aw); } } *reinterpret_cast(sm_f + hw * 64 + l16 * 4) = make_float4(ax, ay, az, aw); __syncthreads(); if (tid < 64) { float t0 = 0.0f, t1 = 0.0f, t2 = 0.0f, t3 = 0.0f; #pragma unroll for (int q = 0; q < 32; q += 4) { t0 += sm_f[(q + 0) * 64 + tid]; t1 += sm_f[(q + 1) * 64 + tid]; t2 += sm_f[(q + 2) * 64 + tid]; t3 += sm_f[(q + 3) * 64 + tid]; } attn_out[qh * kHeadDim + part * 64 + tid] = __float2half_rn((t0 + t1) + (t2 + t3)); } } // ========================================================================= // KERNEL 3: Single-Wave O_proj (128 x 512, 8 rows/blk, 2 warps/row, 1 sync, 128 atomics!) // ========================================================================= __global__ __launch_bounds__(512, 2) void o_proj_pdl_kernel( const __nv_bfloat16* __restrict__ o_w, const __nv_bfloat16* __restrict__ post_ln, __half* attn_out, __nv_bfloat16* x_curr, float* h_mid, __half* h_mid_ln, float* ss_mid, float* ss_x ) { const int tid = threadIdx.x; const int grp = tid >> 6; const int lgrp = tid & 63; const int w_in = lgrp >> 5; const int lane = tid & 31; const int r = (blockIdx.x << 3) + grp; __shared__ float sm[8][2]; const uint4* row = reinterpret_cast(o_w + (size_t)r * (kNumQ * kHeadDim)); uint4 w0 = ld_cg_v4(row + lgrp); uint4 w1 = ld_cg_v4(row + 64 + lgrp); uint4 w2 = ld_cg_v4(row + 128 + lgrp); uint4 w3 = ld_cg_v4(row + 192 + lgrp); float x_res = 0.0f, pln = 0.0f; if (tid < 8) { int r0 = (blockIdx.x << 3) + tid; x_res = __bfloat162float(x_curr[r0]); pln = __bfloat162float(__ldg(post_ln + r0)); } pdl_wait(); pdl_launch(); if (blockIdx.x == 0 && tid == 0) ss_x[0] = 0.0f; const uint4* ao4 = reinterpret_cast(attn_out); float acc = dot8_f16_bf16(ao4[lgrp], w0) + dot8_f16_bf16(ao4[64 + lgrp], w1) + dot8_f16_bf16(ao4[128 + lgrp], w2) + dot8_f16_bf16(ao4[192 + lgrp], w3); acc = warp_reduce_sum(acc); if (lane == 0) sm[grp][w_in] = acc; __syncthreads(); if (tid < 8) { int r0 = (blockIdx.x << 3) + tid; float y = x_res + sm[tid][0] + sm[tid][1]; h_mid[r0] = y; h_mid_ln[r0] = __float2half_rn(y * pln); float y2 = y * y; y2 += __shfl_down_sync(0xffu, y2, 4); y2 += __shfl_down_sync(0xffu, y2, 2); y2 += __shfl_down_sync(0xffu, y2, 1); if (tid == 0) atomicAdd(ss_mid, y2); } } // ========================================================================= // KERNEL 4: Gate+Up+SiLU (192 x 512, direct weight pointer, 4-byte ss_mid!) // ========================================================================= __global__ __launch_bounds__(512, 2) void gate_up_pdl_kernel( const __nv_bfloat16* __restrict__ gate_up_w, __half* h_mid_ln, float* ss_mid, __half* mlp_act ) { const int tid = threadIdx.x; const int warp = tid >> 5; const int lane = tid & 31; const int r = (blockIdx.x << 4) + warp; const uint4* rg = reinterpret_cast(gate_up_w + ((size_t)r * 2 + 0) * kHidden); const uint4* ru = reinterpret_cast(gate_up_w + ((size_t)r * 2 + 1) * kHidden); uint4 g0 = ld_cg_v4(rg + lane), u0 = ld_cg_v4(ru + lane); uint4 g1 = ld_cg_v4(rg + 32 + lane), u1 = ld_cg_v4(ru + 32 + lane); uint4 g2 = ld_cg_v4(rg + 64 + lane), u2 = ld_cg_v4(ru + 64 + lane); uint4 g3 = ld_cg_v4(rg + 96 + lane), u3 = ld_cg_v4(ru + 96 + lane); pdl_wait(); pdl_launch(); float inv_rms = rsqrtf(ss_mid[0] * (1.0f / (float)kHidden) + kEps); const uint4* h4 = reinterpret_cast(h_mid_ln); uint4 x0 = h4[lane]; uint4 x1 = h4[32 + lane]; uint4 x2 = h4[64 + lane]; uint4 x3 = h4[96 + lane]; float pg = dot8_f16_bf16(x0, g0) + dot8_f16_bf16(x1, g1) + dot8_f16_bf16(x2, g2) + dot8_f16_bf16(x3, g3); float pu = dot8_f16_bf16(x0, u0) + dot8_f16_bf16(x1, u1) + dot8_f16_bf16(x2, u2) + dot8_f16_bf16(x3, u3); float g = warp_reduce_sum(pg) * inv_rms; float u = warp_reduce_sum(pu) * inv_rms; if (lane == 0) { mlp_act[r] = __float2half_rn((g / (1.0f + expf(-g))) * u); } } // ========================================================================= // KERNEL 5: Single-Wave Down_proj (128 x 512, 8 rows/blk, 2 warps/row, 1 sync, 128 atomics!) // ========================================================================= template // 0=mid-layer, 1=step-boundary, 2=final-output __global__ __launch_bounds__(512, 2) void down_pdl_kernel( const __nv_bfloat16* __restrict__ down_w, const __nv_bfloat16* __restrict__ next_iln, __half* mlp_act, float* h_mid, __nv_bfloat16* x_curr, __half* x_ln, float* ss_x, float* ss_mid, __nv_bfloat16* noise_next, DecodeParams* dp ) { const int tid = threadIdx.x; const int grp = tid >> 6; const int lgrp = tid & 63; const int w_in = lgrp >> 5; const int lane = tid & 31; const int r = (blockIdx.x << 3) + grp; __shared__ float sm[8][2]; const uint4* row = reinterpret_cast(down_w + (size_t)r * kInter); uint4 w0 = ld_cg_v4(row + lgrp); uint4 w1 = ld_cg_v4(row + 64 + lgrp); uint4 w2 = ld_cg_v4(row + 128 + lgrp); uint4 w3 = ld_cg_v4(row + 192 + lgrp); uint4 w4 = ld_cg_v4(row + 256 + lgrp); uint4 w5 = ld_cg_v4(row + 320 + lgrp); float h_res = 0.0f, niln = 0.0f; __nv_bfloat16 n_val = __float2bfloat16_rn(0.0f); if (tid < 8) { int r0 = (blockIdx.x << 3) + tid; h_res = h_mid[r0]; if constexpr (MODE != 2) niln = __bfloat162float(__ldg(next_iln + r0)); if constexpr (MODE == 1) n_val = noise_next[r0]; } pdl_wait(); if constexpr (MODE != 2) pdl_launch(); if constexpr (MODE != 2) { if (blockIdx.x == 0 && tid == 0) ss_mid[0] = 0.0f; } const uint4* ma4 = reinterpret_cast(mlp_act); float acc = dot8_f16_bf16(ma4[lgrp], w0) + dot8_f16_bf16(ma4[64 + lgrp], w1) + dot8_f16_bf16(ma4[128 + lgrp], w2) + dot8_f16_bf16(ma4[192 + lgrp], w3) + dot8_f16_bf16(ma4[256 + lgrp], w4) + dot8_f16_bf16(ma4[320 + lgrp], w5); acc = warp_reduce_sum(acc); if (lane == 0) sm[grp][w_in] = acc; __syncthreads(); if (tid < 8) { int r0 = (blockIdx.x << 3) + tid; float y = h_res + sm[tid][0] + sm[tid][1]; __nv_bfloat16 y_bf = __float2bfloat16_rn(y); if constexpr (MODE == 0) { x_curr[r0] = y_bf; float xf = __bfloat162float(y_bf); x_ln[r0] = __float2half_rn(xf * niln); float y2 = xf * xf; y2 += __shfl_down_sync(0xffu, y2, 4); y2 += __shfl_down_sync(0xffu, y2, 2); y2 += __shfl_down_sync(0xffu, y2, 1); if (tid == 0) atomicAdd(ss_x, y2); } else if constexpr (MODE == 1) { __nv_bfloat16 x_next = mix_half_bf16(n_val, y_bf); x_curr[r0] = x_next; float xf = __bfloat162float(x_next); x_ln[r0] = __float2half_rn(xf * niln); float y2 = xf * xf; y2 += __shfl_down_sync(0xffu, y2, 4); y2 += __shfl_down_sync(0xffu, y2, 2); y2 += __shfl_down_sync(0xffu, y2, 1); if (tid == 0) atomicAdd(ss_x, y2); } else { dp->h_out[r0] = y_bf; } } } // ---------------- Host helpers & PDL CUDA Graph Cache ---------------- static bool g_rope_inited = false; static bool g_l2_limit_set = false; static size_t g_persist_l2_max = 0; static size_t g_access_win_max = 0; static cudaStream_t g_cap_stream = nullptr; static cudaStream_t g_aux_stream = nullptr; static cudaEvent_t g_ev_mt_go = nullptr; static cudaEvent_t g_ev_mt_done = nullptr; static cudaStream_t g_last_l2_stream = nullptr; static void* g_last_l2_ptr = nullptr; static size_t g_last_l2_bytes = 0; struct GraphKey { uintptr_t packed_ptr; uintptr_t ws_ptr; int num_layers; int chunk_steps; int attn_mode; // 0: v4 no-cs S=32, 1: v4 cs S=32, 2: v8 S=47 bool operator==(const GraphKey& o) const { return packed_ptr == o.packed_ptr && ws_ptr == o.ws_ptr && num_layers == o.num_layers && chunk_steps == o.chunk_steps && attn_mode == o.attn_mode; } }; struct GraphKeyHash { size_t operator()(const GraphKey& k) const { size_t h = std::hash{}(k.packed_ptr); h ^= std::hash{}(k.ws_ptr) + 0x9e3779b97f4a7c15ull + (h << 6) + (h >> 2); h ^= (size_t)((k.num_layers << 16) ^ (k.chunk_steps << 4) ^ k.attn_mode); return h; } }; static std::unordered_map g_graph_cache; static inline size_t align128(size_t off) { return (off + 127) & ~size_t(127); } std::vector compute_layer_weights(uint8_t* base, int num_layers) { std::vector meta(num_layers); size_t off = 0; for (int l = 0; l < num_layers; ++l) { LayerWeights lw; off = align128(off); lw.qkv_w = reinterpret_cast(base + off); off += (size_t)4096 * kHidden * 2; off = align128(off); lw.o_w = reinterpret_cast(base + off); off += (size_t)kHidden * kNumQ * kHeadDim * 2; off = align128(off); lw.gate_up_w = reinterpret_cast(base + off); off += (size_t)kInter * 2 * kHidden * 2; off = align128(off); lw.down_w = reinterpret_cast(base + off); off += (size_t)kHidden * kInter * 2; off = align128(off); lw.input_ln = reinterpret_cast(base + off); off += kHidden * 2; off = align128(off); lw.q_norm = reinterpret_cast(base + off); off += kHeadDim * 2; off = align128(off); lw.k_norm = reinterpret_cast(base + off); off += kHeadDim * 2; off = align128(off); lw.post_ln = reinterpret_cast(base + off); off += kHidden * 2; meta[l] = lw; } return meta; } void set_l2_persist_on_stream(cudaStream_t stream, void* pptr, size_t pbytes) { if (g_persist_l2_max == 0 || pbytes == 0) return; size_t win_bytes = (pbytes < g_access_win_max) ? pbytes : g_access_win_max; float hit_ratio = (win_bytes > 0) ? ((float)g_persist_l2_max / (float)win_bytes) : 1.0f; if (hit_ratio > 1.0f) hit_ratio = 1.0f; cudaStreamAttrValue attr = {}; attr.accessPolicyWindow.base_ptr = pptr; attr.accessPolicyWindow.num_bytes = win_bytes; attr.accessPolicyWindow.hitRatio = hit_ratio; attr.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting; attr.accessPolicyWindow.missProp = cudaAccessPropertyNormal; cudaStreamSetAttribute(stream, cudaStreamAttributeAccessPolicyWindow, &attr); } void ensure_constants() { if (!g_rope_inited) { float inv[kHalfHead]; for (int i = 0; i < kHalfHead; ++i) { inv[i] = (float)(1.0 / pow(10000.0, (double)i / (double)kHalfHead)); } cudaMemcpyToSymbol(c_rope_inv, inv, sizeof(inv)); g_rope_inited = true; } if (!g_l2_limit_set) { cudaDeviceProp prop; cudaGetDeviceProperties(&prop, 0); g_persist_l2_max = prop.persistingL2CacheMaxSize; g_access_win_max = prop.accessPolicyMaxWindowSize; if (g_persist_l2_max > 0) { cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, g_persist_l2_max); } cudaStreamCreateWithFlags(&g_cap_stream, cudaStreamNonBlocking); cudaStreamCreateWithFlags(&g_aux_stream, cudaStreamNonBlocking); cudaEventCreateWithFlags(&g_ev_mt_go, cudaEventDisableTiming); cudaEventCreateWithFlags(&g_ev_mt_done, cudaEventDisableTiming); g_l2_limit_set = true; } } Workspace make_workspace(uint8_t* wbase) { size_t off = 0; Workspace ws; ws.x_curr = reinterpret_cast<__nv_bfloat16*>(wbase + off); off = align128(off + kHidden * 2); ws.x_ln = reinterpret_cast<__half*>(wbase + off); off = align128(off + kHidden * 2); ws.h_mid = reinterpret_cast(wbase + off); off = align128(off + kHidden * 4); ws.h_mid_ln = reinterpret_cast<__half*>(wbase + off); off = align128(off + kHidden * 2); ws.ss_x = reinterpret_cast(wbase + off); off = align128(off + 4); ws.ss_mid = reinterpret_cast(wbase + off); off = align128(off + 4); ws.q_raw = reinterpret_cast(wbase + off); off = align128(off + kNumQ * kHeadDim * 4); ws.k_raw = reinterpret_cast(wbase + off); off = align128(off + kNumKV * kHeadDim * 4); ws.attn_out = reinterpret_cast<__half*>(wbase + off); off = align128(off + kNumQ * kHeadDim * 2); ws.mlp_act = reinterpret_cast<__half*>(wbase + off); off = align128(off + kInter * 2); ws.attn_part_o = reinterpret_cast(wbase + off); off = align128(off + (size_t)kNumQ * 48 * kHeadDim * 4); ws.attn_part_ml = reinterpret_cast(wbase + off); off = align128(off + (size_t)kNumQ * 48 * 8); ws.rope_cos = reinterpret_cast(wbase + off); off = align128(off + (size_t)kMaxChunk * kHalfHead * 4); ws.rope_sin = reinterpret_cast(wbase + off); off = align128(off + (size_t)kMaxChunk * kHalfHead * 4); ws.mt_state = reinterpret_cast(wbase + off); off = align128(off + 624 * 4); ws.mt_cur = reinterpret_cast(wbase + off); off = align128(off + 4); ws.raw_u32 = reinterpret_cast(wbase + off); off = align128(off + (size_t)kMaxChunk * kHidden * 4); ws.noise_buf = reinterpret_cast<__nv_bfloat16*>(wbase + off); off = align128(off + (size_t)kMaxChunk * kHidden * 2); ws.params = reinterpret_cast(wbase + off); return ws; } static void launch_pdl(cudaStream_t stream, void* fn, int grid, int blk, void** args) { cudaLaunchConfig_t cfg = {}; cfg.gridDim = dim3(grid); cfg.blockDim = dim3(blk); cfg.stream = stream; cudaLaunchAttribute attr[1]; attr[0].id = cudaLaunchAttributeProgrammaticStreamSerialization; attr[0].val.programmaticStreamSerializationAllowed = 1; cfg.attrs = attr; cfg.numAttrs = 1; cudaError_t err = cudaLaunchKernelExC(&cfg, fn, args); TORCH_CHECK(err == cudaSuccess, "cudaLaunchKernelExC failed: ", cudaGetErrorString(err)); } cudaGraphExec_t get_or_capture_graph( void* packed_ptr, size_t packed_bytes, Workspace ws, uintptr_t ws_ptr, int num_layers, int chunk_steps, int attn_mode ) { GraphKey key{reinterpret_cast(packed_ptr), ws_ptr, num_layers, chunk_steps, attn_mode}; auto it = g_graph_cache.find(key); if (it != g_graph_cache.end()) return it->second; if (g_graph_cache.size() >= 64) { for (auto& kv : g_graph_cache) cudaGraphExecDestroy(kv.second); g_graph_cache.clear(); } auto h_layers = compute_layer_weights(reinterpret_cast(packed_ptr), num_layers); cudaStream_t st = g_cap_stream; cudaStream_t aux = g_aux_stream; set_l2_persist_on_stream(st, packed_ptr, packed_bytes); cudaError_t err = cudaStreamBeginCapture(st, cudaStreamCaptureModeThreadLocal); TORCH_CHECK(err == cudaSuccess, "cudaStreamBeginCapture failed: ", cudaGetErrorString(err)); { const __nv_bfloat16* iln0 = h_layers[0].input_ln; void* a[] = {&iln0, &ws}; launch_pdl(st, (void*)prologue_step0_kernel, 1, 256, a); } if (chunk_steps > 1) { cudaEventRecord(g_ev_mt_go, st); cudaStreamWaitEvent(aux, g_ev_mt_go, 0); mt_u32_rest_kernel<<<1, 256, 0, aux>>>(ws, chunk_steps); bm_rope_rest_kernel<<<16, 256, 0, aux>>>(ws, chunk_steps); cudaEventRecord(g_ev_mt_done, aux); } void* attn_fn = (attn_mode == 0) ? (void*)attn_pdl_v4_kernel<47, false, false> : ((attn_mode == 1) ? (void*)attn_pdl_v4_kernel<47, true, false> : ((attn_mode == 2) ? (void*)attn_pdl_v4_kernel<47, true, true> : (void*)attn_pdl_v8_kernel<47>)); void* red_fn = (attn_mode < 3) ? (void*)reduce_pdl_kernel<47, 16> : (void*)reduce_pdl_kernel<47, 32>; int attn_grid = kNumKV * 47; DecodeParams* dp = ws.params; for (int step = 0; step < chunk_steps; ++step) { for (int l = 0; l < num_layers; ++l) { const LayerWeights& lw = h_layers[l]; { void* a[] = {(void*)&lw.qkv_w, &ws.x_ln, &ws.ss_x, &ws.q_raw, &ws.k_raw, &dp, &l, &step}; launch_pdl(st, (void*)qkv_pdl_kernel, 256, 512, a); } { void* a[] = {(void*)&lw.q_norm, (void*)&lw.k_norm, &ws.q_raw, &ws.k_raw, &ws.rope_cos, &ws.rope_sin, &ws.attn_part_o, &ws.attn_part_ml, &dp, &l, &step}; launch_pdl(st, attn_fn, attn_grid, 256, a); } { void* a[] = {&ws.attn_part_o, &ws.attn_part_ml, &ws.attn_out, &dp, &step}; launch_pdl(st, red_fn, kNumQ * 2, 512, a); } { void* a[] = {(void*)&lw.o_w, (void*)&lw.post_ln, &ws.attn_out, &ws.x_curr, &ws.h_mid, &ws.h_mid_ln, &ws.ss_mid, &ws.ss_x}; launch_pdl(st, (void*)o_proj_pdl_kernel, 128, 512, a); } { void* a[] = {(void*)&lw.gate_up_w, &ws.h_mid_ln, &ws.ss_mid, &ws.mlp_act}; launch_pdl(st, (void*)gate_up_pdl_kernel, 192, 512, a); } if (step == 0 && l == num_layers - 1 && chunk_steps > 1) { cudaStreamWaitEvent(st, g_ev_mt_done, 0); } { const __nv_bfloat16* next_iln = (l < num_layers - 1) ? h_layers[l + 1].input_ln : h_layers[0].input_ln; __nv_bfloat16* noise_next = ws.noise_buf + (size_t)((step + 1 < chunk_steps) ? (step + 1) : 0) * kHidden; void* a[] = {(void*)&lw.down_w, &next_iln, &ws.mlp_act, &ws.h_mid, &ws.x_curr, &ws.x_ln, &ws.ss_x, &ws.ss_mid, &noise_next, &dp}; void* down_fn = (l < num_layers - 1) ? (void*)down_pdl_kernel<0> : ((step + 1 < chunk_steps) ? (void*)down_pdl_kernel<1> : (void*)down_pdl_kernel<2>); launch_pdl(st, down_fn, 128, 512, a); } } } cudaGraph_t graph = nullptr; err = cudaStreamEndCapture(st, &graph); TORCH_CHECK(err == cudaSuccess, "cudaStreamEndCapture failed: ", cudaGetErrorString(err)); cudaGraphExec_t exec = nullptr; err = cudaGraphInstantiate(&exec, graph, 0); cudaGraphDestroy(graph); TORCH_CHECK(err == cudaSuccess, "cudaGraphInstantiate failed: ", cudaGetErrorString(err)); cudaGraphUpload(exec, st); cudaStreamSynchronize(st); g_graph_cache[key] = exec; return exec; } __global__ void pack_gate_up_kernel( const __nv_bfloat16* __restrict__ gate_proj, const __nv_bfloat16* __restrict__ up_proj, __nv_bfloat16* __restrict__ dst ) { size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; size_t total = (size_t)kInter * 2 * kHidden / 8; if (idx >= total) return; size_t col8 = idx & 127; size_t row2 = idx >> 7; int r = (int)(row2 >> 1); int is_up = (int)(row2 & 1); const __nv_bfloat16* src = (is_up ? up_proj : gate_proj) + (size_t)r * kHidden + col8 * 8; reinterpret_cast(dst)[idx] = *reinterpret_cast(src); } } // namespace void pack_model_weights( std::vector q_projs, std::vector k_projs, std::vector v_projs, std::vector o_projs, std::vector gate_projs, std::vector up_projs, std::vector down_projs, std::vector input_lns, std::vector q_norms, std::vector k_norms, std::vector post_lns, torch::Tensor packed_buf, torch::Tensor ws_buf ) { ensure_constants(); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); int num_layers = (int)q_projs.size(); TORCH_CHECK(num_layers <= kMaxLayers, "num_layers exceeds kMaxLayers"); uint8_t* base = reinterpret_cast(packed_buf.data_ptr()); auto h_layers = compute_layer_weights(base, num_layers); for (int l = 0; l < num_layers; ++l) { auto* qkv_dst = const_cast<__nv_bfloat16*>(h_layers[l].qkv_w); cudaMemcpyAsync(qkv_dst, q_projs[l].data_ptr(), (size_t)2048 * kHidden * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(qkv_dst + (size_t)2048 * kHidden, k_projs[l].data_ptr(), (size_t)1024 * kHidden * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(qkv_dst + (size_t)3072 * kHidden, v_projs[l].data_ptr(), (size_t)1024 * kHidden * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].o_w), o_projs[l].data_ptr(), (size_t)kHidden * kNumQ * kHeadDim * 2, cudaMemcpyDeviceToDevice, stream); { size_t total = (size_t)kInter * 2 * kHidden / 8; pack_gate_up_kernel<<<(total + 255) / 256, 256, 0, stream>>>( reinterpret_cast(gate_projs[l].data_ptr()), reinterpret_cast(up_projs[l].data_ptr()), const_cast<__nv_bfloat16*>(h_layers[l].gate_up_w)); } cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].down_w), down_projs[l].data_ptr(), (size_t)kHidden * kInter * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].input_ln), input_lns[l].data_ptr(), kHidden * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].q_norm), q_norms[l].data_ptr(), kHeadDim * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].k_norm), k_norms[l].data_ptr(), kHeadDim * 2, cudaMemcpyDeviceToDevice, stream); cudaMemcpyAsync(const_cast<__nv_bfloat16*>(h_layers[l].post_ln), post_lns[l].data_ptr(), kHidden * 2, cudaMemcpyDeviceToDevice, stream); } void* pptr = packed_buf.data_ptr(); size_t pbytes = (size_t)packed_buf.numel() * packed_buf.element_size(); uint8_t* wbase = reinterpret_cast(ws_buf.data_ptr()); Workspace ws = make_workspace(wbase); uintptr_t ws_ptr = reinterpret_cast(wbase); get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 0); get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 1); get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 2); get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 64, 3); get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 32, 3); get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, 16, 3); } void run_decode_steps_cuda( torch::Tensor packed_buf, std::vector k_caches, std::vector v_caches, torch::Tensor ws_buf, torch::Tensor h_in, torch::Tensor h_out, int start_pos, int n_steps, int64_t rng_seed, int init_mt, int use_l2_persist ) { if (n_steps <= 0) return; ensure_constants(); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); int num_layers = (int)k_caches.size(); int64_t max_seq = k_caches[0].size(1); size_t pbytes = (size_t)packed_buf.numel() * packed_buf.element_size(); void* pptr = packed_buf.data_ptr(); if (use_l2_persist && g_persist_l2_max > 0) { if (stream != g_last_l2_stream || pptr != g_last_l2_ptr || pbytes != g_last_l2_bytes) { set_l2_persist_on_stream(stream, pptr, pbytes); g_last_l2_stream = stream; g_last_l2_ptr = pptr; g_last_l2_bytes = pbytes; } } uint8_t* wbase = reinterpret_cast(ws_buf.data_ptr()); Workspace ws = make_workspace(wbase); uintptr_t ws_ptr = reinterpret_cast(wbase); DecodeParams h_params = {}; for (int l = 0; l < num_layers; ++l) { h_params.kvs[l].k = reinterpret_cast<__nv_bfloat16*>(k_caches[l].data_ptr()); h_params.kvs[l].v = reinterpret_cast<__nv_bfloat16*>(v_caches[l].data_ptr()); } h_params.max_seq = max_seq; h_params.rng_seed = (uint64_t)rng_seed; const __nv_bfloat16* cur_in = reinterpret_cast(h_in.data_ptr()); __nv_bfloat16* cur_out = reinterpret_cast<__nv_bfloat16*>(h_out.data_ptr()); int done = 0; while (done < n_steps) { int chunk = n_steps - done; if (chunk > kMaxChunk) chunk = kMaxChunk; int cur_pos = start_pos + done; int attn_mode = (cur_pos < 512) ? 0 : ((cur_pos < 4096) ? 1 : ((cur_pos < 16384) ? 2 : 3)); h_params.h_in = (done == 0) ? cur_in : cur_out; h_params.h_out = cur_out; h_params.start_pos = cur_pos; h_params.n_steps = chunk; h_params.init_mt = (done == 0) ? init_mt : 0; cudaMemcpyAsync(ws.params, &h_params, sizeof(DecodeParams), cudaMemcpyHostToDevice, stream); cudaGraphExec_t exec = get_or_capture_graph(pptr, pbytes, ws, ws_ptr, num_layers, chunk, attn_mode); cudaError_t err = cudaGraphLaunch(exec, stream); TORCH_CHECK(err == cudaSuccess, "cudaGraphLaunch failed: ", cudaGetErrorString(err)); done += chunk; } } """ _CPP_SRC = r""" #include #include void pack_model_weights( std::vector q_projs, std::vector k_projs, std::vector v_projs, std::vector o_projs, std::vector gate_projs, std::vector up_projs, std::vector down_projs, std::vector input_lns, std::vector q_norms, std::vector k_norms, std::vector post_lns, torch::Tensor packed_buf, torch::Tensor ws_buf ); void run_decode_steps_cuda( torch::Tensor packed_buf, std::vector k_caches, std::vector v_caches, torch::Tensor ws_buf, torch::Tensor h_in, torch::Tensor h_out, int start_pos, int n_steps, int64_t rng_seed, int init_mt, int use_l2_persist ); """ _EXT = None def _get_ext(): global _EXT if _EXT is None: _EXT = load_inline( name="megaqwen_sm120_v12", cpp_sources=_CPP_SRC, cuda_sources=_CUDA_SRC, functions=["pack_model_weights", "run_decode_steps_cuda"], extra_cuda_cflags=[ "-O3", "-arch=sm_120", "--expt-relaxed-constexpr", "-lineinfo", ], verbose=False, ) return _EXT class Block(nn.Module): def __init__(self): super().__init__() H, I, D = HIDDEN, INTERMEDIATE, HEAD_DIM self.input_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16)) self.q_proj = nn.Parameter(torch.empty(NUM_Q * D, H, dtype=torch.bfloat16)) self.k_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16)) self.v_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16)) self.q_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16)) self.k_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16)) self.o_proj = nn.Parameter(torch.empty(H, NUM_Q * D, dtype=torch.bfloat16)) self.post_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16)) self.gate_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16)) self.up_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16)) self.down_proj = nn.Parameter(torch.empty(H, I, dtype=torch.bfloat16)) for p in self.parameters(): if ( p is self.input_ln or p is self.post_ln or p is self.q_norm or p is self.k_norm ): continue nn.init.normal_(p, std=0.02) class Model(nn.Module): def __init__(self, num_layers: int = NUM_LAYERS, max_seq: int = 131072): super().__init__() self.num_layers = num_layers self.max_seq = max_seq self.blocks = nn.ModuleList([Block() for _ in range(num_layers)]) self._packed_buf: Optional[torch.Tensor] = None self._ws_buf: Optional[torch.Tensor] = None self._params_tuple: Optional[Tuple[nn.Parameter, ...]] = None self._packed_versions: Optional[Tuple[int, ...]] = None self._packed_device: Optional[torch.device] = None def _param_versions(self) -> Tuple[int, ...]: if self._params_tuple is None: self._params_tuple = tuple(self.parameters()) return tuple(p._version for p in self._params_tuple) def _load_from_state_dict(self, *args, **kwargs): super()._load_from_state_dict(*args, **kwargs) self._packed_versions = None def ensure_packed(self, device: torch.device) -> None: vers = self._param_versions() if ( self._packed_buf is not None and self._packed_device == device and self._packed_versions == vers ): return ext = _get_ext() bytes_per_layer = ( (4096 * HIDDEN) + (HIDDEN * NUM_Q * HEAD_DIM) + (INTERMEDIATE * 2 * HIDDEN) + (HIDDEN * INTERMEDIATE) + (HIDDEN + HEAD_DIM + HEAD_DIM + HIDDEN) ) * 2 + 4096 total_bytes = bytes_per_layer * self.num_layers if self._packed_buf is None or self._packed_buf.device != device or self._packed_buf.numel() < total_bytes: self._packed_buf = torch.empty(total_bytes, dtype=torch.uint8, device=device) if self._ws_buf is None or self._ws_buf.device != device: self._ws_buf = torch.empty(1536 * 1024, dtype=torch.uint8, device=device) ext.pack_model_weights( [b.q_proj.contiguous() for b in self.blocks], [b.k_proj.contiguous() for b in self.blocks], [b.v_proj.contiguous() for b in self.blocks], [b.o_proj.contiguous() for b in self.blocks], [b.gate_proj.contiguous() for b in self.blocks], [b.up_proj.contiguous() for b in self.blocks], [b.down_proj.contiguous() for b in self.blocks], [b.input_ln.contiguous() for b in self.blocks], [b.q_norm.contiguous() for b in self.blocks], [b.k_norm.contiguous() for b in self.blocks], [b.post_ln.contiguous() for b in self.blocks], self._packed_buf, self._ws_buf, ) self._packed_versions = vers self._packed_device = device def empty_caches(num_layers: int, max_seq: int, device, dtype=torch.bfloat16): k_all = torch.empty(num_layers, NUM_KV, max_seq, HEAD_DIM, device=device, dtype=dtype) v_all = torch.empty(num_layers, NUM_KV, max_seq, HEAD_DIM, device=device, dtype=dtype) return [k_all[i] for i in range(num_layers)], [v_all[i] for i in range(num_layers)] def _seeded_hidden(seed: int, device) -> torch.Tensor: g = torch.Generator(device="cpu") g.manual_seed(seed) return torch.randn(HIDDEN, generator=g, dtype=torch.bfloat16).to(device) @torch.no_grad() def prefill( model: Model, ctx_len: int, seed: int, device: Optional[torch.device] = None, ): """Build KV of length ctx_len. NOT timed in benchmark.""" device = device or next(model.parameters()).device model = model.to(device).eval() assert ctx_len <= model.max_seq model.ensure_packed(device) h = _seeded_hidden(seed, device).contiguous() k_caches, v_caches = empty_caches(model.num_layers, model.max_seq, device) if ctx_len <= 0: return h, k_caches, v_caches h_out = torch.empty_like(h) ext = _get_ext() ext.run_decode_steps_cuda( model._packed_buf, k_caches, v_caches, model._ws_buf, h, h_out, 0, ctx_len, seed + 1, 1, 1, ) return h_out, k_caches, v_caches @torch.no_grad() def decode_steps( model: Model, hidden: torch.Tensor, k_caches: List[torch.Tensor], v_caches: List[torch.Tensor], start_pos: int, n_steps: int, seed: int, ): """Run n_steps decode steps starting at start_pos. Timed in benchmark.""" if n_steps <= 0: return hidden, k_caches, v_caches device = hidden.device model.ensure_packed(device) h_in = hidden if hidden.is_contiguous() else hidden.contiguous() h_out = torch.empty(HIDDEN, dtype=torch.bfloat16, device=device) ext = _get_ext() ext.run_decode_steps_cuda( model._packed_buf, k_caches, v_caches, model._ws_buf, h_in, h_out, start_pos, n_steps, seed + 2, 1, 1, ) return h_out, k_caches, v_caches def run( ctx_len: int, n_decode: int, seed: int, model: Optional[Model] = None, max_seq: Optional[int] = None, ) -> dict: device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") max_seq = max_seq or max(ctx_len + n_decode, 512) if model is None: model = Model(NUM_LAYERS, max_seq) else: if getattr(model, "max_seq", 0) < ctx_len + n_decode: raise ValueError( f"model.max_seq={getattr(model, 'max_seq', None)} too small for " f"ctx_len={ctx_len}+n_decode={n_decode}" ) model = model.to(device).eval() h, k_caches, v_caches = prefill(model, ctx_len, seed, device=device) h, k_caches, v_caches = decode_steps( model, h, k_caches, v_caches, start_pos=ctx_len, n_steps=n_decode, seed=seed ) return { "last_hidden": h.detach(), "ctx_len": ctx_len, "decode_steps": n_decode, } def get_init_inputs(): return [NUM_LAYERS, 131072] def get_inputs(): return []