from __future__ import annotations import math import os import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline OP_TYPE = "megaqwen_decode" SUPPORTED_PRECISIONS = ["bf16"] HARDWARE_REQUIRED = ["RTX_PRO_6000"] HIDDEN = 1024 INTERMEDIATE = 3072 NUM_Q = 16 NUM_KV = 8 HEAD_DIM = 128 NUM_LAYERS = 4 EPS = 1e-6 _CUDA_SRC = r""" #include #include #include #include #include #include #include #include #include namespace cg = cooperative_groups; constexpr int WARP_SIZE = 32; constexpr int HIDDEN_SIZE = 1024; constexpr int INTERMEDIATE_SIZE = 3072; constexpr int NUM_Q_HEADS = 16; constexpr int NUM_KV_HEADS = 8; constexpr int HEAD_DIM = 128; constexpr int Q_SIZE = NUM_Q_HEADS * HEAD_DIM; // 2048 constexpr int KV_SIZE = NUM_KV_HEADS * HEAD_DIM; // 1024 constexpr int QKV_SIZE = Q_SIZE + 2 * KV_SIZE; // 4096 constexpr float RMS_EPS = 1e-6f; constexpr float LOG2E = 1.44269504088896340736f; constexpr float ATTN_SCALE_LOG2E = 0.088388347648318447f * LOG2E; // (1/sqrt(128)) * log2(e) constexpr int MK_NUM_BLOCKS = 188; constexpr int MK_BLOCK_SIZE = 512; constexpr int MK_NUM_WARPS = MK_BLOCK_SIZE / WARP_SIZE; // 16 constexpr int MK_QWARPS = MK_NUM_WARPS * 4; // 64 constexpr int MAX_SPLITS_PER_HEAD = 32; constexpr int MAX_LAYERS = 8; constexpr int MAX_STREAMED_STEPS = 256; constexpr int NUM_RING_SLOTS = 8; struct __align__(32) u32x8 { uint32_t v[8]; }; struct LayerDesc { const __nv_bfloat16* input_ln; const __nv_bfloat16* qkv_weight; // [4096, 1024] const __nv_bfloat16* q_norm; // [128] const __nv_bfloat16* k_norm; // [128] const __nv_bfloat16* o_weight; // [1024, 2048] const __nv_bfloat16* post_ln; // [1024] const __nv_bfloat16* gate_weight; // [3072, 1024] const __nv_bfloat16* up_weight; // [3072, 1024] const __nv_bfloat16* down_weight; // [1024, 3072] __nv_bfloat16* k_cache; // [8, max_seq, 128] __nv_bfloat16* v_cache; // [8, max_seq, 128] }; struct KernelParams { LayerDesc layers[MAX_LAYERS]; const __nv_bfloat16* persist_end; float inv_freq[HEAD_DIM / 2]; __nv_bfloat16 first_x_rand[HIDDEN_SIZE]; // 2 KB in __grid_constant__ for zero-latency step 0 const __nv_bfloat16* d_host_x_rand; // mapped pinned host buffer for steps 1..n-1 (or nullptr) const int* d_host_ready; // mapped pinned host ready counter (or nullptr) int num_layers; int max_seq; }; struct SharedStorage { union { float s_vec[INTERMEDIATE_SIZE]; // 12 KB struct { float s_acc0[MK_NUM_WARPS][HEAD_DIM]; // 8 KB float s_acc1[MK_NUM_WARPS][HEAD_DIM]; // 8 KB float s_m0[MK_NUM_WARPS]; float s_l0[MK_NUM_WARPS]; float s_m1[MK_NUM_WARPS]; float s_l1[MK_NUM_WARPS]; } attn; }; float s_q0[HEAD_DIM]; float s_q1[HEAD_DIM]; float s_cos[HEAD_DIM / 2]; float s_sin[HEAD_DIM / 2]; __align__(32) __nv_bfloat16 s_k_cur[HEAD_DIM]; float s_reduce[64]; int is_last_kv; }; __device__ __forceinline__ float fast_exp2(float x) { float y; asm volatile("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); return y; } __device__ __forceinline__ float warp_reduce_sum(float val) { #pragma unroll for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) { val += __shfl_xor_sync(0xffffffff, val, offset); } return val; } __device__ __forceinline__ u32x8 ld_v8_evict_last(const void* ptr) { u32x8 out; asm( "ld.global.L1::no_allocate.L2::evict_last.v8.b32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];" : "=r"(out.v[0]), "=r"(out.v[1]), "=r"(out.v[2]), "=r"(out.v[3]), "=r"(out.v[4]), "=r"(out.v[5]), "=r"(out.v[6]), "=r"(out.v[7]) : "l"(ptr) ); return out; } __device__ __forceinline__ u32x8 ld_v8_evict_first(const void* ptr) { u32x8 out; asm( "ld.global.L1::no_allocate.L2::evict_first.v8.b32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];" : "=r"(out.v[0]), "=r"(out.v[1]), "=r"(out.v[2]), "=r"(out.v[3]), "=r"(out.v[4]), "=r"(out.v[5]), "=r"(out.v[6]), "=r"(out.v[7]) : "l"(ptr) ); return out; } __device__ __forceinline__ int ld_cv_s32(const int* ptr) { int r; asm volatile("ld.global.cv.s32 %0, [%1];" : "=r"(r) : "l"(ptr)); return r; } __device__ __forceinline__ uint2 ld_cv_u64(const void* ptr) { uint2 r; asm volatile("ld.global.cv.v2.u32 {%0, %1}, [%2];" : "=r"(r.x), "=r"(r.y) : "l"(ptr)); return r; } template __device__ __forceinline__ u32x8 ld_kv_v8(const void* ptr) { if constexpr (STREAM_KV) { return ld_v8_evict_first(ptr); } else { return ld_v8_evict_last(ptr); } } __device__ __forceinline__ u32x8 ld_weight_v8(const __nv_bfloat16* ptr, const __nv_bfloat16* persist_end) { return (ptr < persist_end) ? ld_v8_evict_last(ptr) : ld_v8_evict_first(ptr); } __device__ __forceinline__ float2 bf16x2_to_f32x2(uint32_t u) { __nv_bfloat162_raw r{static_cast(u), static_cast(u >> 16)}; return __bfloat1622float2(__nv_bfloat162(r)); } __device__ __forceinline__ float dot16_bf16_f32(const u32x8& w, float4 a0, float4 a1, float4 a2, float4 a3) { float2 f0 = bf16x2_to_f32x2(w.v[0]); float2 f1 = bf16x2_to_f32x2(w.v[1]); float2 f2 = bf16x2_to_f32x2(w.v[2]); float2 f3 = bf16x2_to_f32x2(w.v[3]); float2 f4 = bf16x2_to_f32x2(w.v[4]); float2 f5 = bf16x2_to_f32x2(w.v[5]); float2 f6 = bf16x2_to_f32x2(w.v[6]); float2 f7 = bf16x2_to_f32x2(w.v[7]); return ((f0.x * a0.x + f0.y * a0.y) + (f1.x * a0.z + f1.y * a0.w)) + ((f2.x * a1.x + f2.y * a1.y) + (f3.x * a1.z + f3.y * a1.w)) + ((f4.x * a2.x + f4.y * a2.y) + (f5.x * a2.z + f5.y * a2.w)) + ((f6.x * a3.x + f6.y * a3.y) + (f7.x * a3.z + f7.y * a3.w)); } __device__ __forceinline__ void apply_head_norm_rope( const float* __restrict__ src_head, const __nv_bfloat16* __restrict__ norm_w, const float* __restrict__ s_cos, const float* __restrict__ s_sin, int lane_id, float post_scale, float* __restrict__ dst_f32, __nv_bfloat16* __restrict__ dst_bf16, __nv_bfloat16* __restrict__ dst_bf16_2 = nullptr ) { int i0 = lane_id; int i1 = lane_id + 32; int i2 = lane_id + 64; int i3 = lane_id + 96; float v0 = src_head[i0]; float v1 = src_head[i1]; float v2 = src_head[i2]; float v3 = src_head[i3]; float sq = v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3; sq = warp_reduce_sum(sq); float rstd = rsqrtf(sq * (1.0f / float(HEAD_DIM)) + RMS_EPS); v0 = v0 * rstd * __bfloat162float(__ldg(norm_w + i0)); v1 = v1 * rstd * __bfloat162float(__ldg(norm_w + i1)); v2 = v2 * rstd * __bfloat162float(__ldg(norm_w + i2)); v3 = v3 * rstd * __bfloat162float(__ldg(norm_w + i3)); float c0 = s_cos[i0], s0 = s_sin[i0]; float c1 = s_cos[i1], s1 = s_sin[i1]; float r0 = (v0 * c0 - v2 * s0) * post_scale; float r2 = (v0 * s0 + v2 * c0) * post_scale; float r1 = (v1 * c1 - v3 * s1) * post_scale; float r3 = (v1 * s1 + v3 * c1) * post_scale; if (dst_f32 != nullptr) { dst_f32[i0] = r0; dst_f32[i1] = r1; dst_f32[i2] = r2; dst_f32[i3] = r3; } if (dst_bf16 != nullptr) { __nv_bfloat16 b0 = __float2bfloat16(r0); __nv_bfloat16 b1 = __float2bfloat16(r1); __nv_bfloat16 b2 = __float2bfloat16(r2); __nv_bfloat16 b3 = __float2bfloat16(r3); dst_bf16[i0] = b0; dst_bf16[i1] = b1; dst_bf16[i2] = b2; dst_bf16[i3] = b3; if (dst_bf16_2 != nullptr) { dst_bf16_2[i0] = b0; dst_bf16_2[i1] = b1; dst_bf16_2[i2] = b2; dst_bf16_2[i3] = b3; } } } __device__ __forceinline__ void attn_accum_step( const u32x8& k_v8, const u32x8& v_v8, const float q0[16], const float q1[16], uint32_t qw_mask, float& m0, float& l0, float acc0[16], float& m1, float& l1, float acc1[16] ) { float s0 = 0.0f; float s1 = 0.0f; #pragma unroll for (int j = 0; j < 8; ++j) { float2 kj = bf16x2_to_f32x2(k_v8.v[j]); s0 += q0[2 * j] * kj.x + q0[2 * j + 1] * kj.y; s1 += q1[2 * j] * kj.x + q1[2 * j + 1] * kj.y; } #pragma unroll for (int mask = 4; mask >= 1; mask >>= 1) { s0 += __shfl_xor_sync(qw_mask, s0, mask); s1 += __shfl_xor_sync(qw_mask, s1, mask); } float old_m0 = m0; m0 = fmaxf(m0, s0); float alpha0 = (l0 > 0.0f) ? fast_exp2(old_m0 - m0) : 0.0f; float p0 = fast_exp2(s0 - m0); l0 = l0 * alpha0 + p0; float old_m1 = m1; m1 = fmaxf(m1, s1); float alpha1 = (l1 > 0.0f) ? fast_exp2(old_m1 - m1) : 0.0f; float p1 = fast_exp2(s1 - m1); l1 = l1 * alpha1 + p1; #pragma unroll for (int j = 0; j < 8; ++j) { float2 vj = bf16x2_to_f32x2(v_v8.v[j]); acc0[2 * j] = acc0[2 * j] * alpha0 + p0 * vj.x; acc0[2 * j + 1] = acc0[2 * j + 1] * alpha0 + p0 * vj.y; acc1[2 * j] = acc1[2 * j] * alpha1 + p1 * vj.x; acc1[2 * j + 1] = acc1[2 * j + 1] * alpha1 + p1 * vj.y; } } template __device__ __forceinline__ void run_kv_range( const __nv_bfloat16* __restrict__ k_head_base, const __nv_bfloat16* __restrict__ v_head_base, const __nv_bfloat16* __restrict__ s_k_cur_d, int p_init, int fast_end, int pos, bool has_cur_pos, const float q0[16], const float q1[16], uint32_t qw_mask, float& m0, float& l0, float acc0[16], float& m1, float& l1, float acc1[16] ) { int p = p_init; if (p < fast_end) { u32x8 cur_k = ld_kv_v8(k_head_base + (int64_t)p * HEAD_DIM); u32x8 cur_v = ld_kv_v8(v_head_base + (int64_t)p * HEAD_DIM); for (; p + MK_QWARPS < fast_end; p += MK_QWARPS) { int p_nxt = p + MK_QWARPS; u32x8 nxt_k = ld_kv_v8(k_head_base + (int64_t)p_nxt * HEAD_DIM); u32x8 nxt_v = ld_kv_v8(v_head_base + (int64_t)p_nxt * HEAD_DIM); attn_accum_step(cur_k, cur_v, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1); cur_k = nxt_k; cur_v = nxt_v; } attn_accum_step(cur_k, cur_v, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1); p += MK_QWARPS; } if (has_cur_pos && p == pos) { u32x8 ka = *reinterpret_cast(s_k_cur_d); u32x8 va = ld_kv_v8(v_head_base + (int64_t)pos * HEAD_DIM); attn_accum_step(ka, va, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1); } } // MODE 0: cache_len <= 2560 (L2-resident KV with .L2::evict_last + 1-head-per-block Split-KV) // MODE 1: 2560 < cache_len <= 49152 (Streaming KV with .L2::evict_first + 1-head-per-block Split-KV) // MODE 2: cache_len > 49152 (Streaming KV with .L2::evict_first + flat 1/188 Split-KV) template __global__ void __launch_bounds__(MK_BLOCK_SIZE, 1) megaqwen_multistep_decode_kernel( const __grid_constant__ KernelParams params, const __nv_bfloat16* __restrict__ x_rand_gmem, // [n_steps, 1024] when not host-streamed const __nv_bfloat16* __restrict__ g_hidden_in, // [1024] __nv_bfloat16* __restrict__ g_hidden, // [1024] __nv_bfloat16* __restrict__ g_x_stage, // [1024] GPU L2 staging buffer for host-streamed x_rand float* __restrict__ g_residual, // [1024] float* __restrict__ g_post_attn, // [1024] float* __restrict__ g_q, // [2048] float* __restrict__ g_k, // [1024] float* __restrict__ g_attn_out, // [2048] float* __restrict__ g_mlp_inter, // [3072] float2* __restrict__ g_part_ml, // [16 * 32] float* __restrict__ g_part_out, // [16 * 32 * 128] unsigned int* __restrict__ g_kv_done, // [8] int start_pos, int n_steps ) { constexpr bool STREAM_KV = (MODE != 0); constexpr bool ONE_HEAD = (MODE != 2); cg::grid_group grid = cg::this_grid(); __shared__ SharedStorage smem; const int tid = threadIdx.x; const int block_id = blockIdx.x; const int warp_id = tid / WARP_SIZE; const int lane_id = tid % WARP_SIZE; const int hw_in_warp = lane_id >> 4; const int hw_lane = lane_id & 15; const uint32_t hw_mask = hw_in_warp ? 0xffff0000u : 0x0000ffffu; const int hw_id = warp_id * 2 + hw_in_warp; const int swiz_1024 = (block_id * 32) & 1023; const int swiz_2048 = (block_id * 32) & 2047; const int oh_kv_h = (block_id < 96) ? (block_id / 24) : (4 + (block_id - 96) / 23); const int oh_n_b = (oh_kv_h < 4) ? 24 : 23; const int oh_slot = (oh_kv_h < 4) ? (block_id - oh_kv_h * 24) : (block_id - 96 - (oh_kv_h - 4) * 23); const bool host_streamed = (params.d_host_ready != nullptr); for (int step = 0; step < n_steps; ++step) { const int pos = start_pos + step; const int cache_len = pos + 1; const __nv_bfloat16* step_x_rand = host_streamed ? ((step == 0) ? params.first_x_rand : g_x_stage) : (x_rand_gmem + (int64_t)step * HIDDEN_SIZE); for (int layer = 0; layer < params.num_layers; ++layer) { const LayerDesc& ld = params.layers[layer]; // ================================================================= // Phase 1: Input Mix (layer 0) + Input RMSNorm + QKV + V-Cache // ================================================================= { if (layer == 0 && tid < (HEAD_DIM / 2)) { constexpr double INV_2PI = 0.15915494309189533577; constexpr double TWO_PI = 6.28318530717958647692; float a = float(pos) * params.inv_freq[tid]; float r = (float)((double)a - nearbyint((double)a * INV_2PI) * TWO_PI); smem.s_sin[tid] = __sinf(r); smem.s_cos[tid] = __cosf(r); } float local_sq = 0.0f; int idx = ((tid * 2) + swiz_1024) & 1023; #pragma unroll for (int c = 0; c < 2; ++c) { int i = idx + c; float v; if (layer == 0) { float xr = __bfloat162float(step_x_rand[i]); float hp = (step == 0) ? __bfloat162float(g_hidden_in[i]) : __bfloat162float(__float2bfloat16(g_post_attn[i])); v = __bfloat162float(__float2bfloat16(0.5f * xr + 0.5f * hp)); } else { v = __bfloat162float(__float2bfloat16(g_post_attn[i])); } smem.s_vec[i] = v; if (block_id == 0) { g_residual[i] = v; } local_sq += v * v; } local_sq = warp_reduce_sum(local_sq); if (lane_id == 0) { smem.s_reduce[warp_id] = local_sq; } __syncthreads(); if (warp_id == 0) { float s = (lane_id < MK_NUM_WARPS) ? smem.s_reduce[lane_id] : 0.0f; s = warp_reduce_sum(s); if (lane_id == 0) { smem.s_reduce[0] = rsqrtf(s * (1.0f / float(HIDDEN_SIZE)) + RMS_EPS); } } __syncthreads(); float rstd = smem.s_reduce[0]; { uint32_t w_pair = *reinterpret_cast(ld.input_ln + idx); float2 wf = bf16x2_to_f32x2(w_pair); smem.s_vec[idx + 0] = smem.s_vec[idx + 0] * rstd * wf.x; smem.s_vec[idx + 1] = smem.s_vec[idx + 1] * rstd * wf.y; } __syncthreads(); // Full-warp v8.b32 QKV GEMV across all 16 warps/block (persisted L2) { const int k_start = lane_id * 16; float4 a0 = *reinterpret_cast(smem.s_vec + k_start + 0); float4 a1 = *reinterpret_cast(smem.s_vec + k_start + 4); float4 a2 = *reinterpret_cast(smem.s_vec + k_start + 8); float4 a3 = *reinterpret_cast(smem.s_vec + k_start + 12); float4 b0 = *reinterpret_cast(smem.s_vec + k_start + 512); float4 b1 = *reinterpret_cast(smem.s_vec + k_start + 516); float4 b2 = *reinterpret_cast(smem.s_vec + k_start + 520); float4 b3 = *reinterpret_cast(smem.s_vec + k_start + 524); for (int m = warp_id * MK_NUM_BLOCKS + block_id; m < QKV_SIZE; m += MK_NUM_WARPS * MK_NUM_BLOCKS) { const __nv_bfloat16* w_row = ld.qkv_weight + (int64_t)m * HIDDEN_SIZE; u32x8 w0 = ld_weight_v8(w_row + k_start + 0, params.persist_end); u32x8 w1 = ld_weight_v8(w_row + k_start + 512, params.persist_end); float sum = dot16_bf16_f32(w0, a0, a1, a2, a3) + dot16_bf16_f32(w1, b0, b1, b2, b3); sum = warp_reduce_sum(sum); if (lane_id == 0) { if (m < Q_SIZE) { g_q[m] = sum; } else if (m < Q_SIZE + KV_SIZE) { g_k[m - Q_SIZE] = sum; } else { int vm = m - (Q_SIZE + KV_SIZE); ld.v_cache[(int64_t)(vm >> 7) * params.max_seq * HEAD_DIM + (int64_t)pos * HEAD_DIM + (vm & 127)] = __float2bfloat16(sum); } } } } } grid.sync(); // ================================================================= // Phase 2: Q/K Norm + RoPE + Compile-Time Specialized Split-KV Attn // ================================================================= { const int kv_h_begin = ONE_HEAD ? oh_kv_h : 0; const int kv_h_end = ONE_HEAD ? (oh_kv_h + 1) : NUM_KV_HEADS; const int64_t T = (int64_t)NUM_KV_HEADS * cache_len; const int64_t flat_s = ONE_HEAD ? 0 : (((int64_t)block_id * T) / MK_NUM_BLOCKS); const int64_t flat_e = ONE_HEAD ? 0 : (((int64_t)(block_id + 1) * T) / MK_NUM_BLOCKS); for (int kv_h = kv_h_begin; kv_h < kv_h_end; ++kv_h) { int n_b, slot_idx, pos_start, pos_end; if constexpr (ONE_HEAD) { n_b = oh_n_b; slot_idx = oh_slot; pos_start = (int)(((int64_t)slot_idx * cache_len) / n_b); pos_end = (int)(((int64_t)(slot_idx + 1) * cache_len) / n_b); } else { const int64_t A = (int64_t)kv_h * cache_len; const int64_t B = (int64_t)(kv_h + 1) * cache_len; const int b_lo = (int)(((A + 1) * MK_NUM_BLOCKS + T - 1) / T) - 1; const int b_hi = (int)((B * MK_NUM_BLOCKS - 1) / T); if (block_id < b_lo || block_id > b_hi) { continue; } n_b = b_hi - b_lo + 1; slot_idx = block_id - b_lo; pos_start = (int)((flat_s > A ? flat_s : A) - A); pos_end = (int)((flat_e < B ? flat_e : B) - A); } if (pos_start < pos_end) { const bool has_cur_pos = (pos_start <= pos && pos < pos_end); if (warp_id == 0) { apply_head_norm_rope( g_q + (2 * kv_h) * HEAD_DIM, ld.q_norm, smem.s_cos, smem.s_sin, lane_id, ATTN_SCALE_LOG2E, smem.s_q0, nullptr ); } else if (warp_id == 1) { apply_head_norm_rope( g_q + (2 * kv_h + 1) * HEAD_DIM, ld.q_norm, smem.s_cos, smem.s_sin, lane_id, ATTN_SCALE_LOG2E, smem.s_q1, nullptr ); } else if (warp_id == 2 && has_cur_pos) { __nv_bfloat16* k_dst = ld.k_cache + (int64_t)kv_h * params.max_seq * HEAD_DIM + (int64_t)pos * HEAD_DIM; apply_head_norm_rope( g_k + kv_h * HEAD_DIM, ld.k_norm, smem.s_cos, smem.s_sin, lane_id, 1.0f, nullptr, smem.s_k_cur, k_dst ); } __syncthreads(); const int qw_in_warp = lane_id >> 3; const int qw_lane = lane_id & 7; const uint32_t qw_mask = 0xffu << (qw_in_warp * 8); const int qw_id = warp_id * 4 + qw_in_warp; const int d_base = qw_lane * 16; float q0[16], q1[16]; #pragma unroll for (int c = 0; c < 4; ++c) { float4 fq0 = *reinterpret_cast(smem.s_q0 + d_base + c * 4); q0[c * 4 + 0] = fq0.x; q0[c * 4 + 1] = fq0.y; q0[c * 4 + 2] = fq0.z; q0[c * 4 + 3] = fq0.w; float4 fq1 = *reinterpret_cast(smem.s_q1 + d_base + c * 4); q1[c * 4 + 0] = fq1.x; q1[c * 4 + 1] = fq1.y; q1[c * 4 + 2] = fq1.z; q1[c * 4 + 3] = fq1.w; } float m0 = -INFINITY, l0 = 0.0f; float m1 = -INFINITY, l1 = 0.0f; float acc0[16] = {0}; float acc1[16] = {0}; const __nv_bfloat16* k_head_base = ld.k_cache + (int64_t)kv_h * params.max_seq * HEAD_DIM + d_base; const __nv_bfloat16* v_head_base = ld.v_cache + (int64_t)kv_h * params.max_seq * HEAD_DIM + d_base; const int fast_end = (pos_end < pos) ? pos_end : pos; run_kv_range(k_head_base, v_head_base, smem.s_k_cur + d_base, pos_start + qw_id, fast_end, pos, has_cur_pos, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1); #pragma unroll for (int xor_off = 16; xor_off >= 8; xor_off >>= 1) { float m0_o = __shfl_xor_sync(0xffffffff, m0, xor_off); float l0_o = __shfl_xor_sync(0xffffffff, l0, xor_off); float mn0 = fmaxf(m0, m0_o); float sa0 = (l0 > 0.0f) ? fast_exp2(m0 - mn0) : 0.0f; float sb0 = (l0_o > 0.0f) ? fast_exp2(m0_o - mn0) : 0.0f; #pragma unroll for (int j = 0; j < 16; ++j) { float ao = __shfl_xor_sync(0xffffffff, acc0[j], xor_off); acc0[j] = acc0[j] * sa0 + ao * sb0; } m0 = mn0; l0 = l0 * sa0 + l0_o * sb0; float m1_o = __shfl_xor_sync(0xffffffff, m1, xor_off); float l1_o = __shfl_xor_sync(0xffffffff, l1, xor_off); float mn1 = fmaxf(m1, m1_o); float sa1 = (l1 > 0.0f) ? fast_exp2(m1 - mn1) : 0.0f; float sb1 = (l1_o > 0.0f) ? fast_exp2(m1_o - mn1) : 0.0f; #pragma unroll for (int j = 0; j < 16; ++j) { float ao = __shfl_xor_sync(0xffffffff, acc1[j], xor_off); acc1[j] = acc1[j] * sa1 + ao * sb1; } m1 = mn1; l1 = l1 * sa1 + l1_o * sb1; } if (qw_in_warp == 0) { if (qw_lane == 0) { smem.attn.s_m0[warp_id] = m0; smem.attn.s_l0[warp_id] = l0; smem.attn.s_m1[warp_id] = m1; smem.attn.s_l1[warp_id] = l1; } #pragma unroll for (int c = 0; c < 4; ++c) { *reinterpret_cast(&smem.attn.s_acc0[warp_id][d_base + c * 4]) = make_float4(acc0[c * 4 + 0], acc0[c * 4 + 1], acc0[c * 4 + 2], acc0[c * 4 + 3]); *reinterpret_cast(&smem.attn.s_acc1[warp_id][d_base + c * 4]) = make_float4(acc1[c * 4 + 0], acc1[c * 4 + 1], acc1[c * 4 + 2], acc1[c * 4 + 3]); } } __syncthreads(); { const int q_local = hw_id >> 4; const int d_grp = (hw_id & 15) * 8; const float* sm = (q_local == 0) ? smem.attn.s_m0 : smem.attn.s_m1; const float* sl = (q_local == 0) ? smem.attn.s_l0 : smem.attn.s_l1; const auto& sacc = (q_local == 0) ? smem.attn.s_acc0 : smem.attn.s_acc1; float mw = sm[hw_lane]; float lw = sl[hw_lane]; float m_max = mw; #pragma unroll for (int mask = 8; mask >= 1; mask >>= 1) { m_max = fmaxf(m_max, __shfl_xor_sync(hw_mask, m_max, mask)); } float sw = (lw > 0.0f) ? fast_exp2(mw - m_max) : 0.0f; float l_sum = lw * sw; float4 va = make_float4(0.0f, 0.0f, 0.0f, 0.0f); float4 vb = make_float4(0.0f, 0.0f, 0.0f, 0.0f); if (lw > 0.0f) { float4 ra = *reinterpret_cast(&sacc[hw_lane][d_grp + 0]); float4 rb = *reinterpret_cast(&sacc[hw_lane][d_grp + 4]); va = make_float4(ra.x * sw, ra.y * sw, ra.z * sw, ra.w * sw); vb = make_float4(rb.x * sw, rb.y * sw, rb.z * sw, rb.w * sw); } #pragma unroll for (int mask = 8; mask >= 1; mask >>= 1) { l_sum += __shfl_xor_sync(hw_mask, l_sum, mask); va.x += __shfl_xor_sync(hw_mask, va.x, mask); va.y += __shfl_xor_sync(hw_mask, va.y, mask); va.z += __shfl_xor_sync(hw_mask, va.z, mask); va.w += __shfl_xor_sync(hw_mask, va.w, mask); vb.x += __shfl_xor_sync(hw_mask, vb.x, mask); vb.y += __shfl_xor_sync(hw_mask, vb.y, mask); vb.z += __shfl_xor_sync(hw_mask, vb.z, mask); vb.w += __shfl_xor_sync(hw_mask, vb.w, mask); } if (hw_lane == 0) { const int qh = 2 * kv_h + q_local; const int slot = qh * MAX_SPLITS_PER_HEAD + slot_idx; if (d_grp == 0) { g_part_ml[slot] = make_float2(m_max, l_sum); } *reinterpret_cast(g_part_out + slot * HEAD_DIM + d_grp + 0) = va; *reinterpret_cast(g_part_out + slot * HEAD_DIM + d_grp + 4) = vb; } } } else { if (warp_id < 2 && lane_id == 0) { g_part_ml[(2 * kv_h + warp_id) * MAX_SPLITS_PER_HEAD + slot_idx] = make_float2(-INFINITY, 0.0f); } } __syncthreads(); if (tid == 0) { unsigned int prev; asm volatile("atom.global.acq_rel.gpu.add.u32 %0, [%1], 1;" : "=r"(prev) : "l"(&g_kv_done[kv_h]) : "memory"); if (prev == (unsigned int)(n_b - 1)) { g_kv_done[kv_h] = 0u; smem.is_last_kv = 1; } else { smem.is_last_kv = 0; } } __syncthreads(); if (smem.is_last_kv) { const int q_local = warp_id >> 3; const int d_grp = (warp_id & 7) * 16; const int qh = 2 * kv_h + q_local; const int s = lane_id; const int slot = qh * MAX_SPLITS_PER_HEAD + s; float2 ml = make_float2(-INFINITY, 0.0f); float4 r0 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); float4 r1 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); float4 r2 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); float4 r3 = make_float4(0.0f, 0.0f, 0.0f, 0.0f); if (s < n_b) { ml = g_part_ml[slot]; const float* p_src = g_part_out + slot * HEAD_DIM + d_grp; r0 = *reinterpret_cast(p_src + 0); r1 = *reinterpret_cast(p_src + 4); r2 = *reinterpret_cast(p_src + 8); r3 = *reinterpret_cast(p_src + 12); } float m_max = ml.x; #pragma unroll for (int offset = 16; offset >= 1; offset >>= 1) { m_max = fmaxf(m_max, __shfl_xor_sync(0xffffffff, m_max, offset)); } float sw = (ml.y > 0.0f) ? fast_exp2(ml.x - m_max) : 0.0f; float l_tot = warp_reduce_sum(ml.y * sw); r0.x = warp_reduce_sum(r0.x * sw); r0.y = warp_reduce_sum(r0.y * sw); r0.z = warp_reduce_sum(r0.z * sw); r0.w = warp_reduce_sum(r0.w * sw); r1.x = warp_reduce_sum(r1.x * sw); r1.y = warp_reduce_sum(r1.y * sw); r1.z = warp_reduce_sum(r1.z * sw); r1.w = warp_reduce_sum(r1.w * sw); r2.x = warp_reduce_sum(r2.x * sw); r2.y = warp_reduce_sum(r2.y * sw); r2.z = warp_reduce_sum(r2.z * sw); r2.w = warp_reduce_sum(r2.w * sw); r3.x = warp_reduce_sum(r3.x * sw); r3.y = warp_reduce_sum(r3.y * sw); r3.z = warp_reduce_sum(r3.z * sw); r3.w = warp_reduce_sum(r3.w * sw); if (lane_id == 0) { float inv_l = 1.0f / l_tot; float* dst = g_attn_out + qh * HEAD_DIM + d_grp; *reinterpret_cast(dst + 0) = make_float4(r0.x * inv_l, r0.y * inv_l, r0.z * inv_l, r0.w * inv_l); *reinterpret_cast(dst + 4) = make_float4(r1.x * inv_l, r1.y * inv_l, r1.z * inv_l, r1.w * inv_l); *reinterpret_cast(dst + 8) = make_float4(r2.x * inv_l, r2.y * inv_l, r2.z * inv_l, r2.w * inv_l); *reinterpret_cast(dst + 12) = make_float4(r3.x * inv_l, r3.y * inv_l, r3.z * inv_l, r3.w * inv_l); } if constexpr (!ONE_HEAD) { __syncthreads(); } } } } grid.sync(); // ================================================================= // Phase 3: Full 16-Warp O-Projection (Persisted L2) + L2 Atomic Add // ================================================================= { int idx = ((tid * 4) + swiz_2048) & 2047; *reinterpret_cast(smem.s_vec + idx) = *reinterpret_cast(g_attn_out + idx); __syncthreads(); const int c0 = warp_id * MK_NUM_BLOCKS + block_id; const int c1 = c0 + MK_NUM_WARPS * MK_NUM_BLOCKS; const int m0 = c0 >> 2; const int ks0 = (c0 & 3) * 512 + lane_id * 16; if (c1 < 4096) { const int m1 = c1 >> 2; const int ks1 = (c1 & 3) * 512 + lane_id * 16; u32x8 w0 = ld_weight_v8(ld.o_weight + (int64_t)m0 * Q_SIZE + ks0, params.persist_end); u32x8 w1 = ld_weight_v8(ld.o_weight + (int64_t)m1 * Q_SIZE + ks1, params.persist_end); float s0 = dot16_bf16_f32(w0, *reinterpret_cast(smem.s_vec + ks0 + 0), *reinterpret_cast(smem.s_vec + ks0 + 4), *reinterpret_cast(smem.s_vec + ks0 + 8), *reinterpret_cast(smem.s_vec + ks0 + 12)); float s1 = dot16_bf16_f32(w1, *reinterpret_cast(smem.s_vec + ks1 + 0), *reinterpret_cast(smem.s_vec + ks1 + 4), *reinterpret_cast(smem.s_vec + ks1 + 8), *reinterpret_cast(smem.s_vec + ks1 + 12)); s0 = warp_reduce_sum(s0); s1 = warp_reduce_sum(s1); if (lane_id == 0) { atomicAdd(&g_residual[m0], s0); atomicAdd(&g_residual[m1], s1); } } else { u32x8 w0 = ld_weight_v8(ld.o_weight + (int64_t)m0 * Q_SIZE + ks0, params.persist_end); float s0 = dot16_bf16_f32(w0, *reinterpret_cast(smem.s_vec + ks0 + 0), *reinterpret_cast(smem.s_vec + ks0 + 4), *reinterpret_cast(smem.s_vec + ks0 + 8), *reinterpret_cast(smem.s_vec + ks0 + 12)); s0 = warp_reduce_sum(s0); if (lane_id == 0) { atomicAdd(&g_residual[m0], s0); } } } grid.sync(); // ================================================================= // Phase 4: Post-Attention RMSNorm + Overlapped 16-Warp Gate & Up + SwiGLU // ================================================================= { int idx = ((tid * 2) + swiz_1024) & 1023; float2 v2 = *reinterpret_cast(g_residual + idx); smem.s_vec[idx + 0] = v2.x; smem.s_vec[idx + 1] = v2.y; if (block_id == 0) { *reinterpret_cast(g_post_attn + idx) = v2; } float local_sq = warp_reduce_sum(v2.x * v2.x + v2.y * v2.y); if (lane_id == 0) { smem.s_reduce[warp_id] = local_sq; } __syncthreads(); if (warp_id == 0) { float s = warp_reduce_sum((lane_id < MK_NUM_WARPS) ? smem.s_reduce[lane_id] : 0.0f); if (lane_id == 0) { smem.s_reduce[0] = rsqrtf(s * (1.0f / float(HIDDEN_SIZE)) + RMS_EPS); } } __syncthreads(); float rstd = smem.s_reduce[0]; { float2 wf = bf16x2_to_f32x2(*reinterpret_cast(ld.post_ln + idx)); smem.s_vec[idx + 0] = v2.x * rstd * wf.x; smem.s_vec[idx + 1] = v2.y * rstd * wf.y; } __syncthreads(); { const int k_start = lane_id * 16; const int m0 = warp_id * MK_NUM_BLOCKS + block_id; const __nv_bfloat16* g_row = ld.gate_weight + (int64_t)m0 * HIDDEN_SIZE; const __nv_bfloat16* u_row = ld.up_weight + (int64_t)m0 * HIDDEN_SIZE; if (warp_id < 2 && block_id >= 64 && block_id < 128) { const int m_rem = MK_NUM_WARPS * MK_NUM_BLOCKS + (block_id - 64); const __nv_bfloat16* r_row = ((warp_id == 0) ? ld.gate_weight : ld.up_weight) + (int64_t)m_rem * HIDDEN_SIZE; u32x8 gw0 = ld_weight_v8(g_row + k_start + 0, params.persist_end); u32x8 gw1 = ld_weight_v8(g_row + k_start + 512, params.persist_end); u32x8 uw0 = ld_weight_v8(u_row + k_start + 0, params.persist_end); u32x8 uw1 = ld_weight_v8(u_row + k_start + 512, params.persist_end); u32x8 rw0 = ld_weight_v8(r_row + k_start + 0, params.persist_end); u32x8 rw1 = ld_weight_v8(r_row + k_start + 512, params.persist_end); float4 a0 = *reinterpret_cast(smem.s_vec + k_start + 0); float4 a1 = *reinterpret_cast(smem.s_vec + k_start + 4); float4 a2 = *reinterpret_cast(smem.s_vec + k_start + 8); float4 a3 = *reinterpret_cast(smem.s_vec + k_start + 12); float g_sum = dot16_bf16_f32(gw0, a0, a1, a2, a3); float u_sum = dot16_bf16_f32(uw0, a0, a1, a2, a3); float r_sum = dot16_bf16_f32(rw0, a0, a1, a2, a3); float4 b0 = *reinterpret_cast(smem.s_vec + k_start + 512); float4 b1 = *reinterpret_cast(smem.s_vec + k_start + 516); float4 b2 = *reinterpret_cast(smem.s_vec + k_start + 520); float4 b3 = *reinterpret_cast(smem.s_vec + k_start + 524); g_sum += dot16_bf16_f32(gw1, b0, b1, b2, b3); u_sum += dot16_bf16_f32(uw1, b0, b1, b2, b3); r_sum += dot16_bf16_f32(rw1, b0, b1, b2, b3); g_sum = warp_reduce_sum(g_sum); u_sum = warp_reduce_sum(u_sum); r_sum = warp_reduce_sum(r_sum); if (lane_id == 0) { g_mlp_inter[m0] = (g_sum / (1.0f + fast_exp2(-g_sum * LOG2E))) * u_sum; smem.s_reduce[warp_id] = r_sum; } asm volatile("bar.sync 1, 64;"); if (tid == 0) { float gs = smem.s_reduce[0], us = smem.s_reduce[1]; g_mlp_inter[m_rem] = (gs / (1.0f + fast_exp2(-gs * LOG2E))) * us; } } else { float4 a0 = *reinterpret_cast(smem.s_vec + k_start + 0); float4 a1 = *reinterpret_cast(smem.s_vec + k_start + 4); float4 a2 = *reinterpret_cast(smem.s_vec + k_start + 8); float4 a3 = *reinterpret_cast(smem.s_vec + k_start + 12); float4 b0 = *reinterpret_cast(smem.s_vec + k_start + 512); float4 b1 = *reinterpret_cast(smem.s_vec + k_start + 516); float4 b2 = *reinterpret_cast(smem.s_vec + k_start + 520); float4 b3 = *reinterpret_cast(smem.s_vec + k_start + 524); u32x8 gw0 = ld_weight_v8(g_row + k_start + 0, params.persist_end); u32x8 gw1 = ld_weight_v8(g_row + k_start + 512, params.persist_end); u32x8 uw0 = ld_weight_v8(u_row + k_start + 0, params.persist_end); u32x8 uw1 = ld_weight_v8(u_row + k_start + 512, params.persist_end); float g_sum = warp_reduce_sum(dot16_bf16_f32(gw0, a0, a1, a2, a3) + dot16_bf16_f32(gw1, b0, b1, b2, b3)); float u_sum = warp_reduce_sum(dot16_bf16_f32(uw0, a0, a1, a2, a3) + dot16_bf16_f32(uw1, b0, b1, b2, b3)); if (lane_id == 0) { g_mlp_inter[m0] = (g_sum / (1.0f + fast_exp2(-g_sum * LOG2E))) * u_sum; } } } } grid.sync(); // ================================================================= // Phase 5: Full 16-Warp Down Projection (Persisted L2) + Next-Step Host Fetch // ================================================================= { const bool fetch_next_x = (host_streamed && layer == params.num_layers - 1 && (step + 1) < n_steps && block_id == 1); int idx0 = ((tid * 4) + swiz_2048) & 2047; *reinterpret_cast(smem.s_vec + idx0) = *reinterpret_cast(g_mlp_inter + idx0); if (tid < 256) { int idx1 = 2048 + (((tid * 4) + swiz_1024) & 1023); *reinterpret_cast(smem.s_vec + idx1) = *reinterpret_cast(g_mlp_inter + idx1); } else if (fetch_next_x && tid == 256) { while (ld_cv_s32(params.d_host_ready) <= (step + 1)) {} } __syncthreads(); if (fetch_next_x && tid >= 256) { int x_off = (tid - 256) * 4; uint2 raw = ld_cv_u64(params.d_host_x_rand + (int64_t)(step + 1) * HIDDEN_SIZE + x_off); *reinterpret_cast(g_x_stage + x_off) = raw; } const int dc = warp_id * MK_NUM_BLOCKS + block_id; const int m0 = dc / 3; const int ks0 = (dc - m0 * 3) * 1024 + lane_id * 16; const __nv_bfloat16* d_row0 = ld.down_weight + (int64_t)m0 * INTERMEDIATE_SIZE; if (block_id >= 60 && warp_id == 0) { const int rem = block_id - 60; const int dc_r = 3008 + (rem >> 1); const int m_rem = dc_r / 3; const int ks_rem = (dc_r - m_rem * 3) * 1024 + (rem & 1) * 512 + lane_id * 16; u32x8 d0 = ld_weight_v8(d_row0 + ks0 + 0, params.persist_end); u32x8 d1 = ld_weight_v8(d_row0 + ks0 + 512, params.persist_end); u32x8 dr = ld_weight_v8(ld.down_weight + (int64_t)m_rem * INTERMEDIATE_SIZE + ks_rem, params.persist_end); float sum = dot16_bf16_f32(d0, *reinterpret_cast(smem.s_vec + ks0 + 0), *reinterpret_cast(smem.s_vec + ks0 + 4), *reinterpret_cast(smem.s_vec + ks0 + 8), *reinterpret_cast(smem.s_vec + ks0 + 12)); sum += dot16_bf16_f32(d1, *reinterpret_cast(smem.s_vec + ks0 + 512), *reinterpret_cast(smem.s_vec + ks0 + 516), *reinterpret_cast(smem.s_vec + ks0 + 520), *reinterpret_cast(smem.s_vec + ks0 + 524)); float sr = dot16_bf16_f32(dr, *reinterpret_cast(smem.s_vec + ks_rem + 0), *reinterpret_cast(smem.s_vec + ks_rem + 4), *reinterpret_cast(smem.s_vec + ks_rem + 8), *reinterpret_cast(smem.s_vec + ks_rem + 12)); sum = warp_reduce_sum(sum); sr = warp_reduce_sum(sr); if (lane_id == 0) { atomicAdd(&g_post_attn[m0], sum); atomicAdd(&g_post_attn[m_rem], sr); } } else { u32x8 d0 = ld_weight_v8(d_row0 + ks0 + 0, params.persist_end); u32x8 d1 = ld_weight_v8(d_row0 + ks0 + 512, params.persist_end); float sum = dot16_bf16_f32(d0, *reinterpret_cast(smem.s_vec + ks0 + 0), *reinterpret_cast(smem.s_vec + ks0 + 4), *reinterpret_cast(smem.s_vec + ks0 + 8), *reinterpret_cast(smem.s_vec + ks0 + 12)); sum += dot16_bf16_f32(d1, *reinterpret_cast(smem.s_vec + ks0 + 512), *reinterpret_cast(smem.s_vec + ks0 + 516), *reinterpret_cast(smem.s_vec + ks0 + 520), *reinterpret_cast(smem.s_vec + ks0 + 524)); sum = warp_reduce_sum(sum); if (lane_id == 0) { atomicAdd(&g_post_attn[m0], sum); } } } grid.sync(); } } if (n_steps > 0 && block_id == 0) { int idx = tid * 2; g_hidden[idx + 0] = __float2bfloat16(g_post_attn[idx + 0]); g_hidden[idx + 1] = __float2bfloat16(g_post_attn[idx + 1]); } } static bool g_kernel_configured = false; static size_t g_persisting_l2_max = 0; static const void* g_last_persisted_ptr = nullptr; static size_t g_last_persisted_bytes = 0; static __nv_bfloat16* g_h_x_pinned = nullptr; static __nv_bfloat16* g_d_x_pinned = nullptr; static int* g_h_ready_pinned = nullptr; static int* g_d_ready_pinned = nullptr; static cudaEvent_t g_slot_events[NUM_RING_SLOTS]; static bool g_slot_used[NUM_RING_SLOTS] = {false}; static uint64_t g_ring_idx = 0; static at::Generator g_cpu_gen; static void ensure_kernel_configured() { if (!g_kernel_configured) { cudaFuncSetAttribute(megaqwen_multistep_decode_kernel<0>, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1); cudaFuncSetAttribute(megaqwen_multistep_decode_kernel<1>, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1); cudaFuncSetAttribute(megaqwen_multistep_decode_kernel<2>, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1); cudaDeviceProp prop; int dev = 0; cudaGetDevice(&dev); cudaGetDeviceProperties(&prop, dev); g_persisting_l2_max = prop.persistingL2CacheMaxSize; if (g_persisting_l2_max > 0) { cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, g_persisting_l2_max); } void* raw_host = nullptr; size_t x_total_bytes = (size_t)NUM_RING_SLOTS * MAX_STREAMED_STEPS * HIDDEN_SIZE * sizeof(__nv_bfloat16); size_t bytes = x_total_bytes + NUM_RING_SLOTS * 128; cudaError_t herr = cudaHostAlloc(&raw_host, bytes, cudaHostAllocMapped); TORCH_CHECK(herr == cudaSuccess, "cudaHostAlloc failed: ", cudaGetErrorString(herr)); g_h_x_pinned = reinterpret_cast<__nv_bfloat16*>(raw_host); g_h_ready_pinned = reinterpret_cast(reinterpret_cast(raw_host) + x_total_bytes); cudaHostGetDevicePointer((void**)&g_d_x_pinned, (void*)g_h_x_pinned, 0); cudaHostGetDevicePointer((void**)&g_d_ready_pinned, (void*)g_h_ready_pinned, 0); for (int i = 0; i < NUM_RING_SLOTS; ++i) { cudaEventCreateWithFlags(&g_slot_events[i], cudaEventDisableTiming); g_slot_used[i] = false; } g_cpu_gen = at::detail::createCPUGenerator(); g_kernel_configured = true; } } static void populate_common_params( KernelParams& params, cudaStream_t stream, const torch::Tensor& packed_weights, const std::vector& layer_offsets, const std::vector& k_caches, const std::vector& v_caches, const torch::Tensor& inv_freq, int max_seq ) { const __nv_bfloat16* w_base = reinterpret_cast(packed_weights.data_ptr()); size_t w_bytes = packed_weights.numel() * sizeof(__nv_bfloat16); size_t win_bytes = std::min(w_bytes, g_persisting_l2_max); if (win_bytes > 0 && (w_base != g_last_persisted_ptr || win_bytes != g_last_persisted_bytes)) { cudaStreamAttrValue attr; memset(&attr, 0, sizeof(attr)); attr.accessPolicyWindow.base_ptr = const_cast<__nv_bfloat16*>(w_base); attr.accessPolicyWindow.num_bytes = win_bytes; attr.accessPolicyWindow.hitRatio = 1.0f; attr.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting; attr.accessPolicyWindow.missProp = cudaAccessPropertyStreaming; cudaStreamSetAttribute(stream, cudaStreamAttributeAccessPolicyWindow, &attr); g_last_persisted_ptr = w_base; g_last_persisted_bytes = win_bytes; } int num_layers = (int)k_caches.size(); TORCH_CHECK(num_layers <= MAX_LAYERS, "num_layers exceeds MAX_LAYERS"); TORCH_CHECK((int)layer_offsets.size() == num_layers * 9, "expected 9 offsets per layer"); params.num_layers = num_layers; params.max_seq = max_seq; params.persist_end = w_base + (win_bytes / sizeof(__nv_bfloat16)); params.d_host_x_rand = nullptr; params.d_host_ready = nullptr; const float* inv_ptr = inv_freq.data_ptr(); for (int i = 0; i < HEAD_DIM / 2; ++i) { params.inv_freq[i] = inv_ptr[i]; } for (int l = 0; l < num_layers; ++l) { const int64_t* o = &layer_offsets[l * 9]; params.layers[l].input_ln = w_base + o[0]; params.layers[l].q_norm = w_base + o[1]; params.layers[l].k_norm = w_base + o[2]; params.layers[l].post_ln = w_base + o[3]; params.layers[l].qkv_weight = w_base + o[4]; params.layers[l].o_weight = w_base + o[5]; params.layers[l].gate_weight = w_base + o[6]; params.layers[l].up_weight = w_base + o[7]; params.layers[l].down_weight = w_base + o[8]; params.layers[l].k_cache = reinterpret_cast<__nv_bfloat16*>(k_caches[l].data_ptr()); params.layers[l].v_cache = reinterpret_cast<__nv_bfloat16*>(v_caches[l].data_ptr()); } } void run_multistep_decode( torch::Tensor packed_weights, std::vector layer_offsets, std::vector k_caches, std::vector v_caches, torch::Tensor inv_freq, torch::Tensor x_rand_all, torch::Tensor hidden_in, torch::Tensor hidden_out, torch::Tensor workspace_f32, int start_pos, int n_steps, int max_seq ) { ensure_kernel_configured(); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); KernelParams params; populate_common_params(params, stream, packed_weights, layer_offsets, k_caches, v_caches, inv_freq, max_seq); float* ws = workspace_f32.data_ptr(); float* g_residual = ws + 0; float* g_post_attn = ws + 1024; float* g_q = ws + 2048; float* g_k = ws + 4096; float* g_attn_out = ws + 5120; float* g_mlp_inter = ws + 7168; unsigned int* g_kv_done = reinterpret_cast(ws + 10240); __nv_bfloat16* g_x_stage = reinterpret_cast<__nv_bfloat16*>(ws + 10272); float2* g_part_ml = reinterpret_cast(ws + 11008); float* g_part_out = ws + 12160; const __nv_bfloat16* x_all_ptr = reinterpret_cast(x_rand_all.data_ptr()); const __nv_bfloat16* hidden_in_ptr = reinterpret_cast(hidden_in.data_ptr()); __nv_bfloat16* hidden_out_ptr = reinterpret_cast<__nv_bfloat16*>(hidden_out.data_ptr()); constexpr int CHUNK_STEPS = 4096; int rem_steps = n_steps; int cur_start = start_pos; int step_off = 0; while (rem_steps > 0) { int mode; int limit_steps; if (cur_start < 2560) { mode = 0; limit_steps = std::min(rem_steps, 2560 - cur_start); } else if (cur_start < 49152) { mode = 1; limit_steps = std::min(rem_steps, 49152 - cur_start); } else { mode = 2; limit_steps = rem_steps; } int cur_steps = std::min(CHUNK_STEPS, limit_steps); const __nv_bfloat16* cur_x_ptr = x_all_ptr + (int64_t)step_off * HIDDEN_SIZE; const __nv_bfloat16* cur_h_in = (step_off == 0) ? hidden_in_ptr : hidden_out_ptr; void* kernel_args[] = { (void*)¶ms, (void*)&cur_x_ptr, (void*)&cur_h_in, (void*)&hidden_out_ptr, (void*)&g_x_stage, (void*)&g_residual, (void*)&g_post_attn, (void*)&g_q, (void*)&g_k, (void*)&g_attn_out, (void*)&g_mlp_inter, (void*)&g_part_ml, (void*)&g_part_out, (void*)&g_kv_done, (void*)&cur_start, (void*)&cur_steps }; void* fn = (mode == 0) ? (void*)megaqwen_multistep_decode_kernel<0> : ((mode == 1) ? (void*)megaqwen_multistep_decode_kernel<1> : (void*)megaqwen_multistep_decode_kernel<2>); cudaError_t err = cudaLaunchCooperativeKernel( fn, dim3(MK_NUM_BLOCKS), dim3(MK_BLOCK_SIZE), kernel_args, 0, stream ); TORCH_CHECK(err == cudaSuccess, "cudaLaunchCooperativeKernel failed: ", cudaGetErrorString(err)); cur_start += cur_steps; step_off += cur_steps; rem_steps -= cur_steps; } } void run_decode_steps_seeded( torch::Tensor packed_weights, std::vector layer_offsets, std::vector k_caches, std::vector v_caches, torch::Tensor inv_freq, torch::Tensor hidden_in, torch::Tensor hidden_out, torch::Tensor workspace_f32, int start_pos, int n_steps, int max_seq, int64_t rng_seed ) { ensure_kernel_configured(); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); KernelParams params; populate_common_params(params, stream, packed_weights, layer_offsets, k_caches, v_caches, inv_freq, max_seq); float* ws = workspace_f32.data_ptr(); float* g_residual = ws + 0; float* g_post_attn = ws + 1024; float* g_q = ws + 2048; float* g_k = ws + 4096; float* g_attn_out = ws + 5120; float* g_mlp_inter = ws + 7168; unsigned int* g_kv_done = reinterpret_cast(ws + 10240); __nv_bfloat16* g_x_stage = reinterpret_cast<__nv_bfloat16*>(ws + 10272); float2* g_part_ml = reinterpret_cast(ws + 11008); float* g_part_out = ws + 12160; int slot = (int)(g_ring_idx++ & (NUM_RING_SLOTS - 1)); if (g_slot_used[slot]) { cudaEventSynchronize(g_slot_events[slot]); } __nv_bfloat16* slot_h_x = g_h_x_pinned + (size_t)slot * MAX_STREAMED_STEPS * HIDDEN_SIZE; const __nv_bfloat16* slot_d_x = g_d_x_pinned + (size_t)slot * MAX_STREAMED_STEPS * HIDDEN_SIZE; int* slot_h_ready = g_h_ready_pinned + slot * 32; const int* slot_d_ready = g_d_ready_pinned + slot * 32; g_cpu_gen.set_current_seed((uint64_t)rng_seed); __atomic_store_n(slot_h_ready, 0, __ATOMIC_RELAXED); { auto t0 = torch::from_blob( params.first_x_rand, {1, HIDDEN_SIZE}, torch::TensorOptions().dtype(torch::kBFloat16).device(torch::kCPU) ); at::randn_out(t0, {1, HIDDEN_SIZE}, g_cpu_gen); } __atomic_store_n(slot_h_ready, 1, __ATOMIC_RELEASE); params.d_host_x_rand = slot_d_x; params.d_host_ready = slot_d_ready; const __nv_bfloat16* null_x = nullptr; const __nv_bfloat16* hidden_in_ptr = reinterpret_cast(hidden_in.data_ptr()); __nv_bfloat16* hidden_out_ptr = reinterpret_cast<__nv_bfloat16*>(hidden_out.data_ptr()); void* kernel_args[] = { (void*)¶ms, (void*)&null_x, (void*)&hidden_in_ptr, (void*)&hidden_out_ptr, (void*)&g_x_stage, (void*)&g_residual, (void*)&g_post_attn, (void*)&g_q, (void*)&g_k, (void*)&g_attn_out, (void*)&g_mlp_inter, (void*)&g_part_ml, (void*)&g_part_out, (void*)&g_kv_done, (void*)&start_pos, (void*)&n_steps }; int end_pos = start_pos + n_steps - 1; void* fn = (end_pos < 2560) ? (void*)megaqwen_multistep_decode_kernel<0> : ((end_pos < 49152) ? (void*)megaqwen_multistep_decode_kernel<1> : (void*)megaqwen_multistep_decode_kernel<2>); cudaError_t err = cudaLaunchCooperativeKernel( fn, dim3(MK_NUM_BLOCKS), dim3(MK_BLOCK_SIZE), kernel_args, 0, stream ); TORCH_CHECK(err == cudaSuccess, "cudaLaunchCooperativeKernel failed: ", cudaGetErrorString(err)); cudaEventRecord(g_slot_events[slot], stream); g_slot_used[slot] = true; if (n_steps > 1) { auto trem = torch::from_blob( slot_h_x + HIDDEN_SIZE, {n_steps - 1, HIDDEN_SIZE}, torch::TensorOptions().dtype(torch::kBFloat16).device(torch::kCPU) ); at::randn_out(trem, {n_steps - 1, HIDDEN_SIZE}, g_cpu_gen); __sync_synchronize(); __atomic_store_n(slot_h_ready, n_steps, __ATOMIC_RELEASE); } } PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) { m.def("run_multistep_decode", &run_multistep_decode, "MegaQwen SM120 multistep decode v6"); m.def("run_decode_steps_seeded", &run_decode_steps_seeded, "MegaQwen SM120 streamed-RNG decode v6"); } """ _EXT = load_inline( name="megaqwen_sm120_ext_v6", cpp_sources="", cuda_sources=_CUDA_SRC, extra_cuda_cflags=["-O3", "-arch=sm_120", "--expt-relaxed-constexpr"], verbose=False, ) class Block(nn.Module): def __init__(self): super().__init__() H, I, D = HIDDEN, INTERMEDIATE, HEAD_DIM self.input_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16)) self.q_proj = nn.Parameter(torch.empty(NUM_Q * D, H, dtype=torch.bfloat16)) self.k_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16)) self.v_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16)) self.q_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16)) self.k_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16)) self.o_proj = nn.Parameter(torch.empty(H, NUM_Q * D, dtype=torch.bfloat16)) self.post_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16)) self.gate_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16)) self.up_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16)) self.down_proj = nn.Parameter(torch.empty(H, I, dtype=torch.bfloat16)) for p in self.parameters(): if p is self.input_ln or p is self.post_ln or p is self.q_norm or p is self.k_norm: continue nn.init.normal_(p, std=0.02) class Model(nn.Module): def __init__(self, num_layers: int = NUM_LAYERS, max_seq: int = 131072): super().__init__() self.num_layers = num_layers self.max_seq = max_seq self.blocks = nn.ModuleList([Block() for _ in range(num_layers)]) self._param_list = list(self.parameters()) self._packed_sig = None self._packed_weights = None self._layer_offsets: list[int] = [] self._inv_freq_cpu = 1.0 / ( 10000.0 ** (torch.arange(0, HEAD_DIM // 2, dtype=torch.float32) / (HEAD_DIM // 2)) ) self._workspace_f32 = None self._gpu_rand_buf = None def _ensure_packed(self, device: torch.device): if self._packed_weights is not None and self._packed_weights.device == device: sig = tuple((p.data_ptr(), p._version) for p in self._param_list) if self._packed_sig == sig: return # Priority order for 80 MiB persisting L2 window (ord2): # 1. All norms (20 KB) # 2. All QKV projections (32 MiB) # 3. All O projections (16 MiB) # 4. All Down projections (24 MiB) -> 72.02 MiB total (100% persisted) # 5. Gate & Up projections (~7.98 MiB persisted + ~40.02 MiB streamed with 4 v8 loads/warp) layer_elems = 2560 + 4096 * 1024 + 1024 * 2048 + 6144 * 1024 + 1024 * 3072 total_elems = self.num_layers * layer_elems if self._packed_weights is None or self._packed_weights.device != device or self._packed_weights.numel() != total_elems: self._packed_weights = torch.empty(total_elems, dtype=torch.bfloat16, device=device) buf = self._packed_weights pos = [{} for _ in range(self.num_layers)] cur = 0 for l, b in enumerate(self.blocks): pos[l]["input_ln"] = cur buf[cur : cur + 1024].copy_(b.input_ln) pos[l]["q_norm"] = cur + 1024 buf[cur + 1024 : cur + 1152].copy_(b.q_norm) pos[l]["k_norm"] = cur + 1152 buf[cur + 1152 : cur + 1280].copy_(b.k_norm) pos[l]["post_ln"] = cur + 1280 buf[cur + 1280 : cur + 2304].copy_(b.post_ln) buf[cur + 2304 : cur + 2560].zero_() cur += 2560 for l, b in enumerate(self.blocks): pos[l]["qkv"] = cur buf[cur : cur + 2048 * 1024].copy_(b.q_proj.reshape(-1)) cur += 2048 * 1024 buf[cur : cur + 1024 * 1024].copy_(b.k_proj.reshape(-1)) cur += 1024 * 1024 buf[cur : cur + 1024 * 1024].copy_(b.v_proj.reshape(-1)) cur += 1024 * 1024 for l, b in enumerate(self.blocks): pos[l]["o"] = cur buf[cur : cur + 1024 * 2048].copy_(b.o_proj.reshape(-1)) cur += 1024 * 2048 for l, b in enumerate(self.blocks): pos[l]["down"] = cur buf[cur : cur + 1024 * 3072].copy_(b.down_proj.reshape(-1)) cur += 1024 * 3072 for l, b in enumerate(self.blocks): pos[l]["gate"] = cur buf[cur : cur + 3072 * 1024].copy_(b.gate_proj.reshape(-1)) cur += 3072 * 1024 pos[l]["up"] = cur buf[cur : cur + 3072 * 1024].copy_(b.up_proj.reshape(-1)) cur += 3072 * 1024 offs: list[int] = [] for l in range(self.num_layers): p = pos[l] offs.extend([ p["input_ln"], p["q_norm"], p["k_norm"], p["post_ln"], p["qkv"], p["o"], p["gate"], p["up"], p["down"], ]) self._layer_offsets = offs if self._workspace_f32 is None or self._workspace_f32.device != device or self._workspace_f32.numel() < 90000: self._workspace_f32 = torch.zeros(90000, dtype=torch.float32, device=device) else: self._workspace_f32[10240:10272].zero_() self._packed_sig = tuple((p.data_ptr(), p._version) for p in self._param_list) def _get_gpu_rand_buf(self, n_steps: int, device: torch.device) -> torch.Tensor: cap = max(128, 1 << (max(1, n_steps) - 1).bit_length()) if self._gpu_rand_buf is None or self._gpu_rand_buf.device != device or self._gpu_rand_buf.shape[0] < cap: self._gpu_rand_buf = torch.empty((cap, HIDDEN), dtype=torch.bfloat16, device=device) return self._gpu_rand_buf[:n_steps] def forward( self, x: torch.Tensor, k_caches: list[torch.Tensor], v_caches: list[torch.Tensor], position: int, ): device = x.device self._ensure_packed(device) x_c = x if x.is_contiguous() else x.contiguous() h_out = torch.empty_like(x_c) x_in = x_c.view(1, HIDDEN) max_seq = k_caches[0].shape[1] _EXT.run_multistep_decode( self._packed_weights, self._layer_offsets, k_caches, v_caches, self._inv_freq_cpu, x_in, x_c, h_out, self._workspace_f32, int(position), 1, int(max_seq), ) return h_out, k_caches, v_caches _PINNED_RAND_CACHE: dict[int, torch.Tensor] = {} def _get_pinned_rand_buf(n_steps: int) -> torch.Tensor: cap = max(128, 1 << (max(1, n_steps) - 1).bit_length()) buf = _PINNED_RAND_CACHE.get(cap) if buf is None: buf = torch.empty((cap, HIDDEN), dtype=torch.bfloat16, pin_memory=True) _PINNED_RAND_CACHE[cap] = buf return buf[:n_steps] def empty_caches(num_layers: int, max_seq: int, device, dtype=torch.bfloat16): k_slab = torch.empty((num_layers, NUM_KV, max_seq, HEAD_DIM), device=device, dtype=dtype) v_slab = torch.empty((num_layers, NUM_KV, max_seq, HEAD_DIM), device=device, dtype=dtype) k = [k_slab[i] for i in range(num_layers)] v = [v_slab[i] for i in range(num_layers)] return k, v def _seeded_hidden(seed: int, device) -> torch.Tensor: g = torch.Generator(device="cpu") g.manual_seed(seed) return torch.randn(HIDDEN, generator=g, dtype=torch.bfloat16).to(device) @torch.no_grad() def prefill( model: Model, ctx_len: int, seed: int, device: torch.device | None = None, ): device = device or next(model.parameters()).device model = model.to(device).eval() assert ctx_len <= model.max_seq model._ensure_packed(device) h = _seeded_hidden(seed, device).contiguous() k_caches, v_caches = empty_caches(model.num_layers, model.max_seq, device) if ctx_len > 0: g = torch.Generator(device="cpu") g.manual_seed(seed + 1) x_cpu = _get_pinned_rand_buf(ctx_len) torch.randn((ctx_len, HIDDEN), generator=g, out=x_cpu) x_gpu = model._get_gpu_rand_buf(ctx_len, device) x_gpu.copy_(x_cpu, non_blocking=True) _EXT.run_multistep_decode( model._packed_weights, model._layer_offsets, k_caches, v_caches, model._inv_freq_cpu, x_gpu, h, h, model._workspace_f32, 0, int(ctx_len), int(model.max_seq), ) return h, k_caches, v_caches @torch.no_grad() def decode_steps( model: Model, hidden: torch.Tensor, k_caches: list[torch.Tensor], v_caches: list[torch.Tensor], start_pos: int, n_steps: int, seed: int, ): device = hidden.device model._ensure_packed(device) h_in = hidden if hidden.is_contiguous() else hidden.contiguous() if n_steps <= 0: return h_in.clone(), k_caches, v_caches h_out = torch.empty_like(h_in) max_seq = int(k_caches[0].shape[1]) s_pos = int(start_pos) steps = int(n_steps) end_pos = s_pos + steps - 1 if steps <= 256 and ( end_pos < 2560 or (s_pos >= 2560 and end_pos < 49152) or s_pos >= 49152 ): _EXT.run_decode_steps_seeded( model._packed_weights, model._layer_offsets, k_caches, v_caches, model._inv_freq_cpu, h_in, h_out, model._workspace_f32, s_pos, steps, max_seq, int(seed) + 2, ) else: g = torch.Generator(device="cpu") g.manual_seed(int(seed) + 2) x_cpu = _get_pinned_rand_buf(steps) torch.randn((steps, HIDDEN), generator=g, out=x_cpu) x_gpu = model._get_gpu_rand_buf(steps, device) x_gpu.copy_(x_cpu, non_blocking=True) _EXT.run_multistep_decode( model._packed_weights, model._layer_offsets, k_caches, v_caches, model._inv_freq_cpu, x_gpu, h_in, h_out, model._workspace_f32, s_pos, steps, max_seq, ) return h_out, k_caches, v_caches def run( ctx_len: int, n_decode: int, seed: int, model: Model | None = None, max_seq: int | None = None, ) -> dict: device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") max_seq = max(ctx_len + n_decode, 512) if max_seq is None else max_seq if model is None: model = Model(NUM_LAYERS, max_seq) else: if getattr(model, "max_seq", 0) < ctx_len + n_decode: raise ValueError( f"model.max_seq={getattr(model, 'max_seq', None)} too small for " f"ctx_len={ctx_len}+n_decode={n_decode}" ) model = model.to(device).eval() h, k_caches, v_caches = prefill(model, ctx_len, seed, device=device) h, k_caches, v_caches = decode_steps( model, h, k_caches, v_caches, start_pos=ctx_len, n_steps=n_decode, seed=seed ) return { "last_hidden": h.detach(), "ctx_len": ctx_len, "decode_steps": n_decode, } def get_init_inputs(): return [NUM_LAYERS, 131072] def get_inputs(): return []