"""DeepSeek NSA-inspired sparse attention in CUDA for RTX PRO 6000 (SM120). Implements exact bench semantics: - Causal block importance = mean(q ยท k / sqrt(D)) over causal keys in 64-token blocks - Top-8 blocks unioned with 64-token causal sliding window - Exact FP32 block importance via zero-smem 128-bit uint4 burst K_mean (3xBF16 unified buffer) + parallel warp Hillis-Steele diagonal importance - Zero-SMEM Register-Direct 4-Thread Split Top-8 (0 stack, 0 spills across all fast-path kernels) - Two specialized SM120 BF16 Tensor Core MMA execution paths (software-pipelined ldmatrix, fused-FMA softmax): * Path A: 2-Warp-Group Pipelined BN=16 Exact 3xBF16 Split-KV Direct Fused Attention (n_blocks <= 16), with zero-stall startup overlap, early K_next prefetch, and folded LPT grid balancing * Path B: 4-Kernel Lock-Free Inverted-Index Block-Centric Sparse Attention: 1. compute_k_mean_and_diag_kernel (zero-smem 128-bit uint4 burst K/Q + unified [BH, 3, nb, D] K_h) 2. score_topk_and_scatter_64/128 (fused Q+K_h1[0] startup, exact 3xBF16 Top-8, 6-CTA/SM D=64 / 3-CTA/SM D=128) 3. remote_block_attn_kernel (register-preloaded ql_b before QK MMA, overlapped Q_next prefetch, Causal-LPT bi-outer Phase 2, PTX cvt.rni.sat.s8+prmt MMA-native int8 o_partial) 4. local_window_and_merge_64/128 (LPT local window, early K_next prefetch, __ldcs register-direct MMA-native merge with 4-wide batching & mls_partial preload on D=128, __stcs streaming output write) """ from __future__ import annotations import os import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline BLOCK_SIZE = 64 TOP_N_BLOCKS = 8 SLIDING_WINDOW = 64 CUDA_SRC = r""" #include #include #include #include #include #include #include #define BLOCK_SIZE 64 #define TOP_N 8 struct __align__(4) i8x4 { int8_t x, y, z, w; }; // ----------------------------------------------------------------------------- // Low-level SM120 PTX Primitives // ----------------------------------------------------------------------------- __device__ __forceinline__ float fast_exp2f(float x) { float y; asm volatile("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x)); return y; } __device__ __forceinline__ uint32_t pack_bf16x2(float a, float b) { __nv_bfloat162 v = __floats2bfloat162_rn(a, b); return *reinterpret_cast(&v); } // 7-instruction hardware PTX float4 -> saturated signed int8x4 packed in uint32_t! __device__ __forceinline__ uint32_t pack_i8x4(float a, float b, float c, float d) { uint32_t ra, rb, rc, rd, ab, cd, res; asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(ra) : "f"(a)); asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(rb) : "f"(b)); asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(rc) : "f"(c)); asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(rd) : "f"(d)); asm("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(ab) : "r"(ra), "r"(rb)); asm("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(cd) : "r"(rc), "r"(rd)); asm("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(res) : "r"(ab), "r"(cd)); return res; } __device__ __forceinline__ void bar_sync_wg(int wg) { if (wg == 0) { asm volatile("bar.sync 1, 128;\n"); } else { asm volatile("bar.sync 2, 128;\n"); } } // Compact 8-byte (m: fp32, l: fp16 in [1, 64], raw_s: bf16 exact fp32 exponent range) __device__ __forceinline__ uint2 pack_mls(float m, float l, float raw_s) { uint32_t u_m = __float_as_uint(m); __half hl = __float2half_rn(l); __nv_bfloat16 bs = __float2bfloat16_rn(raw_s); uint16_t u_l = *reinterpret_cast(&hl); uint16_t u_s = *reinterpret_cast(&bs); return make_uint2(u_m, static_cast(u_l) | (static_cast(u_s) << 16)); } __device__ __forceinline__ void unpack_mls(uint2 pk, float& m, float& l, float& raw_s) { m = __uint_as_float(pk.x); uint16_t u_l = static_cast(pk.y & 0xFFFFu); uint16_t u_s = static_cast(pk.y >> 16); l = __half2float(*reinterpret_cast(&u_l)); raw_s = __bfloat162float(*reinterpret_cast(&u_s)); } // Order-preserving uint64 key: (v1 > v2) || (v1 == v2 && b1 > b2) __device__ __forceinline__ uint64_t make_score_key(float val, int bi) { val = val + 0.0f; uint32_t u = __float_as_uint(val); uint32_t ord = (u & 0x80000000u) ? ~u : (u ^ 0x80000000u); return (static_cast(ord) << 32) | static_cast(bi); } __device__ __forceinline__ void swap_u64(uint64_t& a, uint64_t& b) { uint64_t t = a; a = b; b = t; } __device__ __forceinline__ void sift_down_3(uint64_t& h3, uint64_t& h7) { if (h3 > h7) swap_u64(h3, h7); } __device__ __forceinline__ void sift_down_1(uint64_t& h1, uint64_t& h3, uint64_t& h4, uint64_t& h7) { if (h3 < h4) { if (h1 > h3) { swap_u64(h1, h3); sift_down_3(h3, h7); } } else { if (h1 > h4) { swap_u64(h1, h4); } } } __device__ __forceinline__ void sift_down_2(uint64_t& h2, uint64_t& h5, uint64_t& h6) { if (h5 < h6) { if (h2 > h5) swap_u64(h2, h5); } else { if (h2 > h6) swap_u64(h2, h6); } } __device__ __forceinline__ void heap8_sift_root( uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3, uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7 ) { if (h1 < h2) { if (h0 > h1) { swap_u64(h0, h1); sift_down_1(h1, h3, h4, h7); } } else { if (h0 > h2) { swap_u64(h0, h2); sift_down_2(h2, h5, h6); } } } __device__ __forceinline__ void heap8_build( uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3, uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7 ) { sift_down_3(h3, h7); sift_down_1(h1, h3, h4, h7); sift_down_2(h2, h5, h6); heap8_sift_root(h0, h1, h2, h3, h4, h5, h6, h7); } __device__ __forceinline__ void heap8_push( uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3, uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7, uint64_t x ) { if (x > h0) { h0 = x; heap8_sift_root(h0, h1, h2, h3, h4, h5, h6, h7); } } __device__ __forceinline__ uint64_t pop_max4_9( uint64_t& k0, uint64_t& k1, uint64_t& k2, uint64_t& k3, uint64_t& k4, uint64_t& k5, uint64_t& k6, uint64_t& k7, uint64_t& k8 ) { uint64_t m01 = (k0 > k1) ? k0 : k1; uint64_t m23 = (k2 > k3) ? k2 : k3; uint64_t m45 = (k4 > k5) ? k4 : k5; uint64_t m67 = (k6 > k7) ? k6 : k7; uint64_t m03 = (m01 > m23) ? m01 : m23; uint64_t m47 = (m45 > m67) ? m45 : m67; uint64_t lm = (m03 > m47) ? m03 : m47; if (k8 > lm) lm = k8; uint64_t o1 = __shfl_xor_sync(0xffffffff, lm, 1); if (o1 > lm) lm = o1; uint64_t o2 = __shfl_xor_sync(0xffffffff, lm, 2); if (o2 > lm) lm = o2; uint32_t lm_lo = static_cast(lm); if (static_cast(k0) == lm_lo) k0 = 0ULL; if (static_cast(k1) == lm_lo) k1 = 0ULL; if (static_cast(k2) == lm_lo) k2 = 0ULL; if (static_cast(k3) == lm_lo) k3 = 0ULL; if (static_cast(k4) == lm_lo) k4 = 0ULL; if (static_cast(k5) == lm_lo) k5 = 0ULL; if (static_cast(k6) == lm_lo) k6 = 0ULL; if (static_cast(k7) == lm_lo) k7 = 0ULL; if (static_cast(k8) == lm_lo) k8 = 0ULL; return lm; } __device__ __forceinline__ uint64_t pop_max4_5( uint64_t& k0, uint64_t& k1, uint64_t& k2, uint64_t& k3, uint64_t& k4 ) { uint64_t m01 = (k0 > k1) ? k0 : k1; uint64_t m23 = (k2 > k3) ? k2 : k3; uint64_t lm = (m01 > m23) ? m01 : m23; if (k4 > lm) lm = k4; uint64_t o1 = __shfl_xor_sync(0xffffffff, lm, 1); if (o1 > lm) lm = o1; uint64_t o2 = __shfl_xor_sync(0xffffffff, lm, 2); if (o2 > lm) lm = o2; uint32_t lm_lo = static_cast(lm); if (static_cast(k0) == lm_lo) k0 = 0ULL; if (static_cast(k1) == lm_lo) k1 = 0ULL; if (static_cast(k2) == lm_lo) k2 = 0ULL; if (static_cast(k3) == lm_lo) k3 = 0ULL; if (static_cast(k4) == lm_lo) k4 = 0ULL; return lm; } __device__ __forceinline__ void cp_async_16(void* smem_ptr, const void* glob_ptr) { uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" :: "r"(smem_addr), "l"(glob_ptr)); } __device__ __forceinline__ void cp_async_16_pred(void* smem_ptr, const void* glob_ptr, bool pred) { uint32_t smem_addr = static_cast(__cvta_generic_to_shared(smem_ptr)); int src_size = pred ? 16 : 0; asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;\n" :: "r"(smem_addr), "l"(glob_ptr), "r"(src_size)); } __device__ __forceinline__ void cp_async_commit() { asm volatile("cp.async.commit_group;\n"); } template __device__ __forceinline__ void cp_async_wait() { asm volatile("cp.async.wait_group %0;\n" :: "n"(N)); } __device__ __forceinline__ void ldmatrix_x4( uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr ) { uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile( "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr) ); } __device__ __forceinline__ void ldmatrix_x4_trans( uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr ) { uint32_t addr = static_cast(__cvta_generic_to_shared(smem_ptr)); asm volatile( "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr) ); } __device__ __forceinline__ void mma_m16n8k16_bf16( float& d0, float& d1, float& d2, float& d3, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1 ) { asm volatile( "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n" : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1) ); } template __device__ __forceinline__ int swz_chunk_idx(int row, int col_chunk) { constexpr int CHUNKS_PER_ROW = D / 8; return row * CHUNKS_PER_ROW + (col_chunk ^ (row & 7)); } template __device__ __forceinline__ void load_tile_swz_async( uint4* smem_tile, const __nv_bfloat16* glob_base, int row_start, int S, int tid, int num_threads ) { constexpr int CHUNKS_PER_ROW = D / 8; constexpr int TOTAL_CHUNKS = ROWS * CHUNKS_PER_ROW; const uint4* glob_u4 = reinterpret_cast(glob_base); #pragma unroll for (int idx = tid; idx < TOTAL_CHUNKS; idx += num_threads) { int r = idx / CHUNKS_PER_ROW; int c = idx % CHUNKS_PER_ROW; int gr = row_start + r; int dst = swz_chunk_idx(r, c); bool valid = (gr < S); const uint4* src_ptr = glob_u4 + (valid ? (int64_t(gr) * CHUNKS_PER_ROW + c) : 0); cp_async_16_pred(&smem_tile[dst], src_ptr, valid); } } // Software-pipelined QK 16x64 Tensor Core GEMM (double-buffered b, single a!) template __device__ __forceinline__ void warp_gemm_qk_16x64( float s[8][4], const uint4* Q_smem, const uint4* K_smem, int warp_id, int lane ) { #pragma unroll for (int ni = 0; ni < 8; ++ni) { s[ni][0] = 0.0f; s[ni][1] = 0.0f; s[ni][2] = 0.0f; s[ni][3] = 0.0f; } int q_row = warp_id * 16 + (lane & 15); int q_k_sub = (lane >> 4); int k_row_sub = ((lane >> 4) << 3) + (lane & 7); int k_k_sub = (lane >> 3) & 1; constexpr int K_STEPS = D / 16; uint32_t b[2][4]; ldmatrix_x4(b[0][0], b[0][1], b[0][2], b[0][3], &K_smem[swz_chunk_idx(k_row_sub, k_k_sub)]); #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { int k_chunk = ks * 2; uint32_t a0, a1, a2, a3; ldmatrix_x4(a0, a1, a2, a3, &Q_smem[swz_chunk_idx(q_row, k_chunk + q_k_sub)]); #pragma unroll for (int np = 0; np < 4; ++np) { int cur_b = np & 1; if (np + 1 < 4) { int k_row_n = (np + 1) * 16 + k_row_sub; ldmatrix_x4(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3], &K_smem[swz_chunk_idx(k_row_n, k_chunk + k_k_sub)]); } else if (ks + 1 < K_STEPS) { ldmatrix_x4(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3], &K_smem[swz_chunk_idx(k_row_sub, (ks + 1) * 2 + k_k_sub)]); } mma_m16n8k16_bf16(s[np * 2][0], s[np * 2][1], s[np * 2][2], s[np * 2][3], a0, a1, a2, a3, b[cur_b][0], b[cur_b][1]); mma_m16n8k16_bf16(s[np * 2 + 1][0], s[np * 2 + 1][1], s[np * 2 + 1][2], s[np * 2 + 1][3], a0, a1, a2, a3, b[cur_b][2], b[cur_b][3]); } } } // Software-pipelined PV 16xD Tensor Core GEMM (double-buffered b!) template __device__ __forceinline__ void warp_gemm_pv_16xD( float o[D / 8][4], const uint32_t p_lo[8], const uint32_t p_hi[8], const uint4* V_smem, int lane ) { int v_row_sub = (lane & 15); int v_d_sub = (lane >> 4); constexpr int D_PAIRS = D / 16; uint32_t b[2][4]; ldmatrix_x4_trans(b[0][0], b[0][1], b[0][2], b[0][3], &V_smem[swz_chunk_idx(v_row_sub, v_d_sub)]); #pragma unroll for (int ks = 0; ks < 4; ++ks) { uint32_t a0 = p_lo[ks * 2]; uint32_t a1 = p_hi[ks * 2]; uint32_t a2 = p_lo[ks * 2 + 1]; uint32_t a3 = p_hi[ks * 2 + 1]; int v_row = ks * 16 + v_row_sub; #pragma unroll for (int dp = 0; dp < D_PAIRS; ++dp) { int cur_b = dp & 1; if (dp + 1 < D_PAIRS) { ldmatrix_x4_trans(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3], &V_smem[swz_chunk_idx(v_row, (dp + 1) * 2 + v_d_sub)]); } else if (ks + 1 < 4) { ldmatrix_x4_trans(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3], &V_smem[swz_chunk_idx((ks + 1) * 16 + v_row_sub, v_d_sub)]); } mma_m16n8k16_bf16(o[dp * 2][0], o[dp * 2][1], o[dp * 2][2], o[dp * 2][3], a0, a1, a2, a3, b[cur_b][0], b[cur_b][1]); mma_m16n8k16_bf16(o[dp * 2 + 1][0], o[dp * 2 + 1][1], o[dp * 2 + 1][2], o[dp * 2 + 1][3], a0, a1, a2, a3, b[cur_b][2], b[cur_b][3]); } } } __device__ __forceinline__ void unpack_u4_bf16(uint4 u, float v[8]) { float2 f0 = __bfloat1622float2(*reinterpret_cast(&u.x)); float2 f1 = __bfloat1622float2(*reinterpret_cast(&u.y)); float2 f2 = __bfloat1622float2(*reinterpret_cast(&u.z)); float2 f3 = __bfloat1622float2(*reinterpret_cast(&u.w)); v[0] = f0.x; v[1] = f0.y; v[2] = f1.x; v[3] = f1.y; v[4] = f2.x; v[5] = f2.y; v[6] = f3.x; v[7] = f3.y; } // ----------------------------------------------------------------------------- // Kernel 1: Zero-smem_K Exact FP32 K_mean (unified [BH, 3, nb, D]) + Diag Importance // - D=64: 128-bit uint4 burst (K read ONCE into regs + 4-row parallel Hillis-Steele scan) // - D=128: 12-CTA/SM 64-bit uint2 pipelined streaming // ----------------------------------------------------------------------------- template __global__ void compute_k_mean_and_diag_kernel( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, __nv_bfloat16* __restrict__ K_h, float* __restrict__ imp_diag, uint32_t* __restrict__ d_counters, int S, int n_blocks, float scale ) { int bi = blockIdx.x; int bh = blockIdx.y; int tid = threadIdx.x; int warp_id = tid >> 5; int lane = tid & 31; if (bi == 0 && bh == 0 && tid < 3) { d_counters[tid] = 0u; } __shared__ float warp_sums[4 * D]; __shared__ float smem_imp[BLOCK_SIZE]; int row_start = bi * BLOCK_SIZE; int rows = min(BLOCK_SIZE, S - row_start); const __nv_bfloat16* K_blk = K + int64_t(bh) * S * D + int64_t(row_start) * D; const __nv_bfloat16* Q_blk = Q + int64_t(bh) * S * D + int64_t(row_start) * D; int r_beg = warp_id * 16; int r_end = min(rows, r_beg + 16); if constexpr (D == 64) { const uint4* gK_u4 = reinterpret_cast(K_blk); const uint4* gQ_u4 = reinterpret_cast(Q_blk); int sub_r = lane >> 3; // 0..3 (4 rows per warp step) int chunk = lane & 7; // 0..7 (8 bf16 per lane) bool do_q = (bi >= TOP_N); uint4 kr[4], qr[4]; #pragma unroll for (int it = 0; it < 4; ++it) { int r = r_beg + it * 4 + sub_r; kr[it] = (r < rows) ? __ldg(&gK_u4[r * 8 + chunk]) : make_uint4(0u, 0u, 0u, 0u); } if (do_q) { #pragma unroll for (int it = 0; it < 4; ++it) { int r = r_beg + it * 4 + sub_r; qr[it] = (r < rows) ? __ldg(&gQ_u4[r * 8 + chunk]) : make_uint4(0u, 0u, 0u, 0u); } } float ws[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; #pragma unroll for (int it = 0; it < 4; ++it) { float kv[8]; unpack_u4_bf16(kr[it], kv); #pragma unroll for (int j = 0; j < 8; ++j) ws[j] += kv[j]; } #pragma unroll for (int j = 0; j < 8; ++j) { ws[j] += __shfl_xor_sync(0xffffffff, ws[j], 8); ws[j] += __shfl_xor_sync(0xffffffff, ws[j], 16); } if (sub_r == 0) { #pragma unroll for (int j = 0; j < 8; ++j) warp_sums[warp_id * 64 + chunk * 8 + j] = ws[j]; } __syncthreads(); if (do_q) { float acc[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; #pragma unroll for (int u = 0; u < 4; ++u) { if (u < warp_id) { #pragma unroll for (int j = 0; j < 8; ++j) acc[j] += warp_sums[u * 64 + chunk * 8 + j]; } } #pragma unroll for (int it = 0; it < 4; ++it) { float kv[8], qv[8]; unpack_u4_bf16(kr[it], kv); unpack_u4_bf16(qr[it], qv); float pfx[8]; #pragma unroll for (int j = 0; j < 8; ++j) { float x = kv[j]; float s1 = __shfl_up_sync(0xffffffff, x, 8); if (sub_r >= 1) x += s1; float s2 = __shfl_up_sync(0xffffffff, x, 16); if (sub_r >= 2) x += s2; pfx[j] = acc[j] + x; acc[j] += __shfl_sync(0xffffffff, x, 24 + chunk); } float dot = 0.0f; #pragma unroll for (int j = 0; j < 8; ++j) dot = fmaf(pfx[j], qv[j], dot); dot += __shfl_xor_sync(0xffffffff, dot, 1); dot += __shfl_xor_sync(0xffffffff, dot, 2); dot += __shfl_xor_sync(0xffffffff, dot, 4); int r = r_beg + it * 4 + sub_r; if (chunk == 0 && r < rows) { smem_imp[r] = dot * (scale * __frcp_rn(float(r + 1))); } } __syncwarp(); if (lane < r_end - r_beg) { imp_diag[int64_t(bh) * S + row_start + r_beg + lane] = smem_imp[r_beg + lane]; } } } else { // D == 128: 2-burst 128-bit uint4 (4 uint4 K+Q bursts, 2-row parallel Hillis-Steele scan) const uint4* gK_u4 = reinterpret_cast(K_blk); const uint4* gQ_u4 = reinterpret_cast(Q_blk); int sub_r = lane >> 4; // 0..1 int chunk = lane & 15; // 0..15 (8 bf16 per lane) float ws[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; #pragma unroll for (int half = 0; half < 2; ++half) { uint4 kr[4]; #pragma unroll for (int it = 0; it < 4; ++it) { int r = r_beg + (half * 4 + it) * 2 + sub_r; kr[it] = (r < rows) ? __ldg(&gK_u4[r * 16 + chunk]) : make_uint4(0u, 0u, 0u, 0u); } #pragma unroll for (int it = 0; it < 4; ++it) { float kv[8]; unpack_u4_bf16(kr[it], kv); #pragma unroll for (int j = 0; j < 8; ++j) ws[j] += kv[j]; } } #pragma unroll for (int j = 0; j < 8; ++j) ws[j] += __shfl_xor_sync(0xffffffff, ws[j], 16); if (sub_r == 0) { #pragma unroll for (int j = 0; j < 8; ++j) warp_sums[warp_id * 128 + chunk * 8 + j] = ws[j]; } __syncthreads(); if (bi >= TOP_N) { float acc[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f}; #pragma unroll for (int u = 0; u < 4; ++u) { if (u < warp_id) { #pragma unroll for (int j = 0; j < 8; ++j) acc[j] += warp_sums[u * 128 + chunk * 8 + j]; } } #pragma unroll for (int half = 0; half < 2; ++half) { uint4 kr[4], qr[4]; #pragma unroll for (int it = 0; it < 4; ++it) { int r = r_beg + (half * 4 + it) * 2 + sub_r; kr[it] = (r < rows) ? __ldg(&gK_u4[r * 16 + chunk]) : make_uint4(0u, 0u, 0u, 0u); qr[it] = (r < rows) ? __ldg(&gQ_u4[r * 16 + chunk]) : make_uint4(0u, 0u, 0u, 0u); } #pragma unroll for (int it = 0; it < 4; ++it) { float kv[8], qv[8]; unpack_u4_bf16(kr[it], kv); unpack_u4_bf16(qr[it], qv); float dot = 0.0f; #pragma unroll for (int j = 0; j < 8; ++j) { float x = kv[j]; float s1 = __shfl_up_sync(0xffffffff, x, 16); if (sub_r == 1) x += s1; dot = fmaf(acc[j] + x, qv[j], dot); acc[j] += __shfl_sync(0xffffffff, x, 16 + chunk); } dot += __shfl_xor_sync(0xffffffff, dot, 1); dot += __shfl_xor_sync(0xffffffff, dot, 2); dot += __shfl_xor_sync(0xffffffff, dot, 4); dot += __shfl_xor_sync(0xffffffff, dot, 8); int r = r_beg + (half * 4 + it) * 2 + sub_r; if (chunk == 0 && r < rows) { smem_imp[r] = dot * (scale * __frcp_rn(float(r + 1))); } } } __syncwarp(); if (lane < r_end - r_beg) { imp_diag[int64_t(bh) * S + row_start + r_beg + lane] = smem_imp[r_beg + lane]; } } } if (bi < n_blocks - 1 && tid < D) { int64_t km_stride = int64_t(n_blocks) * D; int64_t km_base = (int64_t(bh) * 3 * n_blocks + bi) * D + tid; float total = warp_sums[0 * D + tid] + warp_sums[1 * D + tid] + warp_sums[2 * D + tid] + warp_sums[3 * D + tid]; float v = total * __frcp_rn(float(rows)); __nv_bfloat16 h1 = __float2bfloat16_rn(v); float r1 = v - __bfloat162float(h1); __nv_bfloat16 h2 = __float2bfloat16_rn(r1); float r2 = r1 - __bfloat162float(h2); __nv_bfloat16 h3 = __float2bfloat16_rn(r2); K_h[km_base] = h1; K_h[km_base + km_stride] = h2; K_h[km_base + 2 * km_stride] = h3; } } // ----------------------------------------------------------------------------- // Zero-SMEM Register-Direct Exact 3xBF16 Block Scoring -> Register 4-Thread Heaps // Single unified K_h pointer ([BH, 3, nb, D]) saves 4 pointer registers! // ----------------------------------------------------------------------------- template __device__ __forceinline__ void score_3bf16_to_reg_heaps( uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3, uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7, uint64_t& g0, uint64_t& g1, uint64_t& g2, uint64_t& g3, uint64_t& g4, uint64_t& g5, uint64_t& g6, uint64_t& g7, uint4* K_dbl_smem, const uint4* Q_smem, const __nv_bfloat16* K_h_head, int n_blocks, int bi_cur, float scale, int tid, int warp_id, int lane ) { constexpr int CHUNKS_PER_ROW = D / 8; constexpr int TOTAL_CHUNKS = 32 * CHUNKS_PER_ROW; int cur_buf = 0; int q_row = warp_id * 16 + (lane & 15); int k_row_sub = ((lane >> 4) << 3) + (lane & 7); int tid_g = lane & 3; constexpr int K_STEPS = D / 16; int kh_stride_u4 = n_blocks * CHUNKS_PER_ROW; const uint4* kh_u4 = reinterpret_cast(K_h_head); auto load_buf = [&](int buf_id, int plane_off_u4, int row_off) { uint4* dst_buf = K_dbl_smem + (buf_id ? TOTAL_CHUNKS : 0); const uint4* src_u4 = kh_u4 + plane_off_u4; #pragma unroll for (int idx = tid; idx < TOTAL_CHUNKS; idx += 128) { int r = idx / CHUNKS_PER_ROW; int c = idx % CHUNKS_PER_ROW; int gb = row_off + r; bool valid = (gb < bi_cur); const uint4* sp = src_u4 + (valid ? (gb * CHUNKS_PER_ROW + c) : 0); cp_async_16_pred(&dst_buf[swz_chunk_idx(r, c)], sp, valid); } cp_async_commit(); }; auto mma_buf = [&](float s[4][4], int buf_id) { const uint4* kbuf = K_dbl_smem + (buf_id ? TOTAL_CHUNKS : 0); int q_k_sub = (lane >> 4); int k_k_sub = (lane >> 3) & 1; #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { int k_chunk = ks * 2; uint32_t a0, a1, a2, a3, b0, b1, b2, b3, b4, b5, b6, b7; ldmatrix_x4(a0, a1, a2, a3, &Q_smem[swz_chunk_idx(q_row, k_chunk + q_k_sub)]); ldmatrix_x4(b0, b1, b2, b3, &kbuf[swz_chunk_idx(k_row_sub, k_chunk + k_k_sub)]); ldmatrix_x4(b4, b5, b6, b7, &kbuf[swz_chunk_idx(16 + k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(s[0][0], s[0][1], s[0][2], s[0][3], a0, a1, a2, a3, b0, b1); mma_m16n8k16_bf16(s[1][0], s[1][1], s[1][2], s[1][3], a0, a1, a2, a3, b2, b3); mma_m16n8k16_bf16(s[2][0], s[2][1], s[2][2], s[2][3], a0, a1, a2, a3, b4, b5); mma_m16n8k16_bf16(s[3][0], s[3][1], s[3][2], s[3][3], a0, a1, a2, a3, b6, b7); } }; load_buf(0, 0, 0); for (int bi_tile = 0; bi_tile < bi_cur; bi_tile += 32) { float s[4][4]; #pragma unroll for (int ni = 0; ni < 4; ++ni) { s[ni][0] = 0.0f; s[ni][1] = 0.0f; s[ni][2] = 0.0f; s[ni][3] = 0.0f; } load_buf(cur_buf ^ 1, kh_stride_u4, bi_tile); cp_async_wait<1>(); __syncthreads(); mma_buf(s, cur_buf); __syncthreads(); load_buf(cur_buf, 2 * kh_stride_u4, bi_tile); cp_async_wait<1>(); __syncthreads(); mma_buf(s, cur_buf ^ 1); cp_async_wait<0>(); __syncthreads(); bool has_next = (bi_tile + 32 < bi_cur); if (has_next) { load_buf(cur_buf ^ 1, 0, bi_tile + 32); } mma_buf(s, cur_buf); int b_base = bi_tile + tid_g * 2; auto sc_key = [&](float v, int bi) -> uint64_t { if constexpr (D == 64) { return (bi < bi_cur) ? make_score_key(v, bi) : 0ULL; } else { return (bi < bi_cur) ? make_score_key(v * scale, bi) : 0ULL; } }; if (bi_tile == 0) { h0 = sc_key(s[0][0], b_base + 0); h1 = sc_key(s[0][1], b_base + 1); h2 = sc_key(s[1][0], b_base + 8); h3 = sc_key(s[1][1], b_base + 9); h4 = sc_key(s[2][0], b_base + 16); h5 = sc_key(s[2][1], b_base + 17); h6 = sc_key(s[3][0], b_base + 24); h7 = sc_key(s[3][1], b_base + 25); g0 = sc_key(s[0][2], b_base + 0); g1 = sc_key(s[0][3], b_base + 1); g2 = sc_key(s[1][2], b_base + 8); g3 = sc_key(s[1][3], b_base + 9); g4 = sc_key(s[2][2], b_base + 16); g5 = sc_key(s[2][3], b_base + 17); g6 = sc_key(s[3][2], b_base + 24); g7 = sc_key(s[3][3], b_base + 25); if (has_next) { heap8_build(h0, h1, h2, h3, h4, h5, h6, h7); heap8_build(g0, g1, g2, g3, g4, g5, g6, g7); } } else { heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[0][0], b_base + 0)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[0][1], b_base + 1)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[1][0], b_base + 8)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[1][1], b_base + 9)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[2][0], b_base + 16)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[2][1], b_base + 17)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[3][0], b_base + 24)); heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[3][1], b_base + 25)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[0][2], b_base + 0)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[0][3], b_base + 1)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[1][2], b_base + 8)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[1][3], b_base + 9)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[2][2], b_base + 16)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[2][3], b_base + 17)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[3][2], b_base + 24)); heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[3][3], b_base + 25)); } if (has_next) { __syncthreads(); // Ensure all warps done with cur_buf before next iter loads K_h2 into it! cur_buf ^= 1; } } } // ----------------------------------------------------------------------------- // Helper: Fused-FMA Online Softmax Prep // ----------------------------------------------------------------------------- template __device__ __forceinline__ void online_softmax_prep( float o[D / 8][4], uint32_t p_lo[8], uint32_t p_hi[8], float& m0, float& l0, bool& has0, float& m1, float& l1, bool& has1, float s[8][4], bool keep0, bool keep1, int r0, int r1, bool is_diag_block, bool full_prev0, bool full_prev1, float scale_log2e, int lane ) { int tid_g = lane & 3; float mb0 = -1e30f, mb1 = -1e30f; #pragma unroll for (int ni = 0; ni < 8; ++ni) { if constexpr (DO_COL_MASK) { int c0 = ni * 8 + tid_g * 2; int c1 = c0 + 1; if (is_diag_block) { if (c0 > r0) s[ni][0] = -1e30f; if (c1 > r0) s[ni][1] = -1e30f; if (c0 > r1) s[ni][2] = -1e30f; if (c1 > r1) s[ni][3] = -1e30f; } else { if (!full_prev0) { if (c0 <= r0) s[ni][0] = -1e30f; if (c1 <= r0) s[ni][1] = -1e30f; } if (!full_prev1) { if (c0 <= r1) s[ni][2] = -1e30f; if (c1 <= r1) s[ni][3] = -1e30f; } } } mb0 = fmaxf(mb0, fmaxf(s[ni][0], s[ni][1])); mb1 = fmaxf(mb1, fmaxf(s[ni][2], s[ni][3])); } mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 1)); mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 2)); mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 1)); mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 2)); bool eff0 = keep0 && (mb0 > -1e29f); bool eff1 = keep1 && (mb1 > -1e29f); float mb0_s = mb0 * scale_log2e; float mb1_s = mb1 * scale_log2e; float m0_new = eff0 ? (has0 ? fmaxf(m0, mb0_s) : mb0_s) : m0; float m1_new = eff1 ? (has1 ? fmaxf(m1, mb1_s) : mb1_s) : m1; float rescale0 = (eff0 && has0) ? fast_exp2f(m0 - m0_new) : 1.0f; float rescale1 = (eff1 && has1) ? fast_exp2f(m1 - m1_new) : 1.0f; if (eff0 && has0 && rescale0 != 1.0f) { #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][0] *= rescale0; o[di][1] *= rescale0; } l0 *= rescale0; } if (eff1 && has1 && rescale1 != 1.0f) { #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][2] *= rescale1; o[di][3] *= rescale1; } l1 *= rescale1; } float lb0 = 0.0f, lb1 = 0.0f; float neg_m0 = -m0_new; float neg_m1 = -m1_new; #pragma unroll for (int ni = 0; ni < 8; ++ni) { if (eff0) { float p0, p1; if constexpr (DO_COL_MASK) { p0 = (s[ni][0] > -1e29f) ? fast_exp2f(fmaf(s[ni][0], scale_log2e, neg_m0)) : 0.0f; p1 = (s[ni][1] > -1e29f) ? fast_exp2f(fmaf(s[ni][1], scale_log2e, neg_m0)) : 0.0f; } else { p0 = fast_exp2f(fmaf(s[ni][0], scale_log2e, neg_m0)); p1 = fast_exp2f(fmaf(s[ni][1], scale_log2e, neg_m0)); } lb0 += p0 + p1; p_lo[ni] = pack_bf16x2(p0, p1); } else { p_lo[ni] = 0u; } if (eff1) { float p2, p3; if constexpr (DO_COL_MASK) { p2 = (s[ni][2] > -1e29f) ? fast_exp2f(fmaf(s[ni][2], scale_log2e, neg_m1)) : 0.0f; p3 = (s[ni][3] > -1e29f) ? fast_exp2f(fmaf(s[ni][3], scale_log2e, neg_m1)) : 0.0f; } else { p2 = fast_exp2f(fmaf(s[ni][2], scale_log2e, neg_m1)); p3 = fast_exp2f(fmaf(s[ni][3], scale_log2e, neg_m1)); } lb1 += p2 + p3; p_hi[ni] = pack_bf16x2(p2, p3); } else { p_hi[ni] = 0u; } } lb0 += __shfl_xor_sync(0xffffffff, lb0, 1); lb0 += __shfl_xor_sync(0xffffffff, lb0, 2); lb1 += __shfl_xor_sync(0xffffffff, lb1, 1); lb1 += __shfl_xor_sync(0xffffffff, lb1, 2); if (eff0) { l0 = has0 ? (l0 + lb0) : lb0; m0 = m0_new; has0 = true; } if (eff1) { l1 = has1 ? (l1 + lb1) : lb1; m1 = m1_new; has1 = true; } } template __device__ __forceinline__ void store_O_bf16_via_smem( __nv_bfloat16* O_head, uint4* smem_stage, float o[D / 8][4], float l0, float l1, int bi_cur, int S, int warp_id, int lane ) { float inv_l0 = DO_NORM ? ((l0 > 0.0f) ? __frcp_rn(l0) : 0.0f) : 1.0f; float inv_l1 = DO_NORM ? ((l1 > 0.0f) ? __frcp_rn(l1) : 0.0f) : 1.0f; __nv_bfloat162* s2 = reinterpret_cast<__nv_bfloat162*>(smem_stage); int gid = lane >> 2; int tid_g = lane & 3; int r0 = warp_id * 16 + gid; int r1 = r0 + 8; #pragma unroll for (int di = 0; di < D / 8; ++di) { int swz0 = (di ^ (r0 & 7)) * 4 + tid_g; int swz1 = (di ^ (r1 & 7)) * 4 + tid_g; if constexpr (DO_NORM) { s2[r0 * (D / 2) + swz0] = __floats2bfloat162_rn(o[di][0] * inv_l0, o[di][1] * inv_l0); s2[r1 * (D / 2) + swz1] = __floats2bfloat162_rn(o[di][2] * inv_l1, o[di][3] * inv_l1); } else { s2[r0 * (D / 2) + swz0] = __floats2bfloat162_rn(o[di][0], o[di][1]); s2[r1 * (D / 2) + swz1] = __floats2bfloat162_rn(o[di][2], o[di][3]); } } __syncwarp(); constexpr int CHUNKS_PER_ROW = D / 8; constexpr int WARP_CHUNKS = 16 * CHUNKS_PER_ROW; uint4* dst_u4 = reinterpret_cast(O_head); int base_row = bi_cur * BLOCK_SIZE + warp_id * 16; #pragma unroll for (int idx = lane; idx < WARP_CHUNKS; idx += 32) { int r = idx / CHUNKS_PER_ROW; int c = idx % CHUNKS_PER_ROW; int gr = base_row + r; if (gr < S) { int r_smem = warp_id * 16 + r; __stcs(&dst_u4[int64_t(gr) * CHUNKS_PER_ROW + c], smem_stage[swz_chunk_idx(r_smem, c)]); } } } // ============================================================================= // PATH A (D=64): 2-Warp-Group Pipelined BN=16 Exact 3xBF16 Split-KV (n_blocks <= 16) // Early K_next prefetch + folded LPT wave balancing + unmasked bi_cur < TOP_N // ============================================================================= __global__ __launch_bounds__(256, 2) void fused_nsa_direct_bn16_splitkv_64( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const __nv_bfloat16* __restrict__ K_h, const float* __restrict__ imp_diag, __nv_bfloat16* __restrict__ O, int BH, int S, int n_blocks, float scale ) { extern __shared__ char smem_raw[]; constexpr int D = 64; constexpr int TILE_U4 = BLOCK_SIZE * (D / 8); uint4* Q_smem = reinterpret_cast(smem_raw); uint4* K0_smem = Q_smem + TILE_U4; uint4* V0_smem = K0_smem + TILE_U4; uint4* K1_smem = V0_smem + TILE_U4; uint4* V1_smem = K1_smem + TILE_U4; int total_ctas = BH * n_blocks; int lin = blockIdx.x; int w1_end = min(376, total_ctas); int idx_lpt = (lin >= 188 && lin < w1_end) ? (188 + (w1_end - 1 - lin)) : lin; int bi_cur = (n_blocks - 1) - (idx_lpt / BH); int bh = idx_lpt % BH; int tid = threadIdx.x; int wg = tid >> 7; int ltid = tid & 127; int warp_id = ltid >> 5; int lane = ltid & 31; int gid = lane >> 2; int tid_g = lane & 3; int r0 = warp_id * 16 + gid; int r1 = r0 + 8; const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D; const __nv_bfloat16* K_head = K + int64_t(bh) * S * D; const __nv_bfloat16* V_head = V + int64_t(bh) * S * D; __nv_bfloat16* O_head = O + int64_t(bh) * S * D; load_tile_swz_async(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 256); uint32_t msk0 = 0u, msk1 = 0u; bool fp0 = false, fp1 = false; bool has_first = false; int bi_first = 0; uint32_t my_rem = 0u; if (bi_cur < TOP_N) { // Overlap K[0], K[1] in Group 0 with Q_smem, and V[0], V[1] in Group 1 -> ZERO startup stall! if (wg == 0) { load_tile_swz_async(K0_smem, K_head, 0, S, ltid, 128); } else if (bi_cur >= 1) { load_tile_swz_async(K1_smem, K_head, BLOCK_SIZE, S, ltid, 128); } cp_async_commit(); if (wg == 0) { load_tile_swz_async(V0_smem, V_head, 0, S, ltid, 128); } else if (bi_cur >= 1) { load_tile_swz_async(V1_smem, V_head, BLOCK_SIZE, S, ltid, 128); } cp_async_commit(); cp_async_wait<1>(); __syncthreads(); uint32_t all_m = (1u << (bi_cur + 1)) - 1u; msk0 = all_m; msk1 = all_m; fp0 = true; fp1 = true; if (wg == 0) { has_first = true; bi_first = 0; my_rem = all_m & 0x55555554u; // even blocks >= 2: 2, 4, 6 } else if (bi_cur >= 1) { has_first = true; bi_first = 1; my_rem = all_m & 0xAAAAAAA8u; // odd blocks >= 3: 3, 5, 7 } } else { int kh_stride_u4 = n_blocks * 8; const uint4* kh_u4 = reinterpret_cast(K_h + int64_t(bh) * 3 * n_blocks * D); // All 256 threads load K_h1/h2/h3 into K0_smem (Group 0 alongside Q_smem) #pragma unroll for (int idx = tid; idx < 3 * 128; idx += 256) { int plane = idx >> 7; int off = idx & 127; int r = off >> 3, c = off & 7; bool valid = (r < bi_cur); const uint4* sp = kh_u4 + plane * kh_stride_u4 + (valid ? off : 0); cp_async_16_pred(&K0_smem[swz_chunk_idx(plane * 16 + r, c)], sp, valid); } cp_async_commit(); // Simultaneously prefetch guaranteed sliding-window tiles V[bi_cur-1], K[bi_cur], V[bi_cur] in Group 1! if (wg == 0) { load_tile_swz_async(V0_smem, V_head, (bi_cur - 1) * BLOCK_SIZE, S, ltid, 128); } else { load_tile_swz_async(K1_smem, K_head, bi_cur * BLOCK_SIZE, S, ltid, 128); load_tile_swz_async(V1_smem, V_head, bi_cur * BLOCK_SIZE, S, ltid, 128); } cp_async_commit(); cp_async_wait<1>(); __syncthreads(); int q_row = warp_id * 16 + (lane & 15); int q_k_sub = (lane >> 4); int k_row_sub = ((lane >> 4) << 3) + (lane & 7); int k_k_sub = (lane >> 3) & 1; float sc00=0,sc01=0,sc02=0,sc03=0, sc10=0,sc11=0,sc12=0,sc13=0; #pragma unroll for (int ks = 0; ks < 4; ++ks) { int k_chunk = ks * 2; uint32_t a0,a1,a2,a3, b0,b1,b2,b3; ldmatrix_x4(a0,a1,a2,a3, &Q_smem[swz_chunk_idx(q_row, k_chunk + q_k_sub)]); ldmatrix_x4(b0,b1,b2,b3, &K0_smem[swz_chunk_idx(k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1); mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3); ldmatrix_x4(b0,b1,b2,b3, &K0_smem[swz_chunk_idx(16+k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1); mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3); ldmatrix_x4(b0,b1,b2,b3, &K0_smem[swz_chunk_idx(32+k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1); mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3); } // K0_smem is now free! Overlap K[bi_cur-1] load behind all 16 pop_max4_5 Top-8 iterations! __syncthreads(); load_tile_swz_async(K0_smem, K_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 256); cp_async_commit(); int b_base = tid_g * 2; uint64_t a0 = (b_base+0 < bi_cur) ? make_score_key(sc00, b_base+0) : 0ULL; uint64_t a1 = (b_base+1 < bi_cur) ? make_score_key(sc01, b_base+1) : 0ULL; uint64_t a2 = (b_base+8 < bi_cur) ? make_score_key(sc10, b_base+8) : 0ULL; uint64_t a3 = (b_base+9 < bi_cur) ? make_score_key(sc11, b_base+9) : 0ULL; uint64_t g0 = (b_base+0 < bi_cur) ? make_score_key(sc02, b_base+0) : 0ULL; uint64_t g1 = (b_base+1 < bi_cur) ? make_score_key(sc03, b_base+1) : 0ULL; uint64_t g2 = (b_base+8 < bi_cur) ? make_score_key(sc12, b_base+8) : 0ULL; uint64_t g3 = (b_base+9 < bi_cur) ? make_score_key(sc13, b_base+9) : 0ULL; const float* imp_head = imp_diag + int64_t(bh) * S; int t0 = bi_cur * BLOCK_SIZE + r0, t1 = bi_cur * BLOCK_SIZE + r1; bool v0 = (t0 < S), v1 = (t1 < S); uint64_t d0 = (tid_g == 0 && v0) ? make_score_key(imp_head[t0] * 8.0f, bi_cur) : 0ULL; uint64_t d1 = (tid_g == 0 && v1) ? make_score_key(imp_head[t1] * 8.0f, bi_cur) : 0ULL; msk0 = v0 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u; msk1 = v1 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u; #pragma unroll for (int u = 0; u < TOP_N; ++u) { uint64_t w0 = pop_max4_5(a0, a1, a2, a3, d0); if (v0 && w0 != 0ULL) { int b = int(w0 & 0xFFFFFFFFu); msk0 |= (1u << b); if (b == bi_cur - 1) fp0 = true; } uint64_t w1 = pop_max4_5(g0, g1, g2, g3, d1); if (v1 && w1 != 0ULL) { int b = int(w1 & 0xFFFFFFFFu); msk1 |= (1u << b); if (b == bi_cur - 1) fp1 = true; } } uint32_t my_u = msk0 | msk1; #pragma unroll for (int off = 16; off > 0; off >>= 1) my_u |= __shfl_xor_sync(0xffffffff, my_u, off); __shared__ uint32_t s_um[4]; if (wg == 0 && lane == 0) s_um[warp_id] = my_u; cp_async_wait<0>(); __syncthreads(); uint32_t rem_mask = s_um[0] | s_um[1] | s_um[2] | s_um[3]; // wg=0 starts on bi_cur-1 (already in K0/V0!), wg=1 starts on bi_cur (already in K1/V1!) has_first = true; bi_first = (wg == 0) ? (bi_cur - 1) : bi_cur; uint32_t extra = rem_mask & ~((1u << bi_cur) | (1u << (bi_cur - 1))); int parity = 0; while (extra != 0u) { int b = __ffs(extra) - 1; extra &= (extra - 1u); if ((parity & 1) == wg) my_rem |= (1u << b); parity++; } } uint4* K_my = (wg == 0) ? K0_smem : K1_smem; uint4* V_my = (wg == 0) ? V0_smem : V1_smem; float o[D / 8][4]; #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][0]=0; o[di][1]=0; o[di][2]=0; o[di][3]=0; } float m0 = -1e30f, l0 = 0.0f; bool has0 = false; float m1 = -1e30f, l1 = 0.0f; bool has1 = false; const float scale_log2e = scale * 1.4426950408889634f; if (has_first) { int bi = bi_first; while (true) { float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_my, warp_id, lane); bool has_next = (my_rem != 0u); int bi_next = 0; if (has_next) { bar_sync_wg(wg); bi_next = __ffs(my_rem) - 1; my_rem &= (my_rem - 1u); load_tile_swz_async(K_my, K_head, bi_next * BLOCK_SIZE, S, ltid, 128); cp_async_commit(); } uint32_t p_lo[8], p_hi[8]; bool keep0 = ((msk0 >> bi) & 1u) != 0u; bool keep1 = ((msk1 >> bi) & 1u) != 0u; bool need_mask = (bi == bi_cur) || (bi == bi_cur - 1 && bi_cur >= TOP_N); if (!need_mask) { online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, keep0, keep1, r0, r1, false, false, false, scale_log2e, lane ); } else { online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, keep0, keep1, r0, r1, (bi == bi_cur), fp0, fp1, scale_log2e, lane ); } if (has_next) { cp_async_wait<1>(); } else { cp_async_wait<0>(); } bar_sync_wg(wg); warp_gemm_pv_16xD(o, p_lo, p_hi, V_my, lane); if (!has_next) break; cp_async_wait<0>(); bar_sync_wg(wg); load_tile_swz_async(V_my, V_head, bi_next * BLOCK_SIZE, S, ltid, 128); cp_async_commit(); bi = bi_next; } } if (bi_cur == 0) { if (wg == 0) store_O_bf16_via_smem(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane); return; } __syncthreads(); float* O1_smem = reinterpret_cast(K0_smem); float2* ml1_smem = reinterpret_cast(K1_smem); if (wg == 1) { #pragma unroll for (int di = 0; di < D / 8; ++di) { int col0 = (di ^ (r0 & 7)) * 8 + tid_g * 2; int col1 = (di ^ (r1 & 7)) * 8 + tid_g * 2; O1_smem[r0 * D + col0] = o[di][0]; O1_smem[r0 * D + col0 + 1] = o[di][1]; O1_smem[r1 * D + col1] = o[di][2]; O1_smem[r1 * D + col1 + 1] = o[di][3]; } if (tid_g == 0) { ml1_smem[r0] = make_float2(m0, has0 ? l0 : 0.0f); ml1_smem[r1] = make_float2(m1, has1 ? l1 : 0.0f); } } __syncthreads(); if (wg == 0) { float2 ml0_1 = ml1_smem[r0]; float2 ml1_1 = ml1_smem[r1]; bool h0_1 = (ml0_1.y > 0.0f); bool h1_1 = (ml1_1.y > 0.0f); float m0_fin = has0 ? (h0_1 ? fmaxf(m0, ml0_1.x) : m0) : ml0_1.x; float m1_fin = has1 ? (h1_1 ? fmaxf(m1, ml1_1.x) : m1) : ml1_1.x; float s0_0 = has0 ? fast_exp2f(m0 - m0_fin) : 0.0f; float s0_1 = h0_1 ? fast_exp2f(ml0_1.x - m0_fin) : 0.0f; float s1_0 = has1 ? fast_exp2f(m1 - m1_fin) : 0.0f; float s1_1 = h1_1 ? fast_exp2f(ml1_1.x - m1_fin) : 0.0f; float l0_fin = l0 * s0_0 + ml0_1.y * s0_1; float l1_fin = l1 * s1_0 + ml1_1.y * s1_1; #pragma unroll for (int di = 0; di < D / 8; ++di) { int col0 = (di ^ (r0 & 7)) * 8 + tid_g * 2; int col1 = (di ^ (r1 & 7)) * 8 + tid_g * 2; o[di][0] = o[di][0] * s0_0 + O1_smem[r0 * D + col0] * s0_1; o[di][1] = o[di][1] * s0_0 + O1_smem[r0 * D + col0 + 1] * s0_1; o[di][2] = o[di][2] * s1_0 + O1_smem[r1 * D + col1] * s1_1; o[di][3] = o[di][3] * s1_0 + O1_smem[r1 * D + col1 + 1] * s1_1; } store_O_bf16_via_smem(O_head, Q_smem, o, l0_fin, l1_fin, bi_cur, S, warp_id, lane); } } // PATH A (D=128 edge-shape support) __global__ __launch_bounds__(128, 2) void fused_nsa_direct_bn16_128( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const __nv_bfloat16* __restrict__ K_h, const float* __restrict__ imp_diag, __nv_bfloat16* __restrict__ O, int S, int n_blocks, float scale ) { extern __shared__ char smem_raw[]; constexpr int D = 128; constexpr int TILE_U4 = BLOCK_SIZE * (D / 8); uint4* Q_smem = reinterpret_cast(smem_raw); uint4* K_smem = Q_smem + TILE_U4; uint4* V_smem = K_smem + TILE_U4; int bh = blockIdx.x; int bi_cur = (n_blocks - 1) - blockIdx.y; int tid = threadIdx.x; int warp_id = tid >> 5; int lane = tid & 31; int gid = lane >> 2; int tid_g = lane & 3; int r0 = warp_id * 16 + gid; int r1 = r0 + 8; const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D; const __nv_bfloat16* K_head = K + int64_t(bh) * S * D; const __nv_bfloat16* V_head = V + int64_t(bh) * S * D; __nv_bfloat16* O_head = O + int64_t(bh) * S * D; load_tile_swz_async(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 128); if (bi_cur < TOP_N) { load_tile_swz_async(K_smem, K_head, 0, S, tid, 128); cp_async_commit(); load_tile_swz_async(V_smem, V_head, 0, S, tid, 128); cp_async_commit(); cp_async_wait<1>(); __syncthreads(); float o[D / 8][4]; #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][0]=0; o[di][1]=0; o[di][2]=0; o[di][3]=0; } float m0 = -1e30f, l0 = 0.0f; bool has0 = false; float m1 = -1e30f, l1 = 0.0f; bool has1 = false; const float scale_log2e = scale * 1.4426950408889634f; for (int bi = 0; bi <= bi_cur; ++bi) { float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); uint32_t p_lo[8], p_hi[8]; if (bi < bi_cur) { online_softmax_prep(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, false, true, true, scale_log2e, lane); } else { online_softmax_prep(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, true, true, true, scale_log2e, lane); } cp_async_wait<0>(); __syncthreads(); if (bi + 1 <= bi_cur) { load_tile_swz_async(K_smem, K_head, (bi + 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); } warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); if (bi + 1 <= bi_cur) { cp_async_wait<0>(); __syncthreads(); load_tile_swz_async(V_smem, V_head, (bi + 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); } } __syncthreads(); store_O_bf16_via_smem(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane); return; } int64_t kh_stride = int64_t(n_blocks) * D; const __nv_bfloat16* kh_base = K_h + int64_t(bh) * 3 * kh_stride; const uint4* kh1_u4 = reinterpret_cast(kh_base); const uint4* kh2_u4 = reinterpret_cast(kh_base + kh_stride); const uint4* kh3_u4 = reinterpret_cast(kh_base + 2 * kh_stride); constexpr int HALF16_U4 = 16 * (D / 8); #pragma unroll for (int idx = tid; idx < HALF16_U4; idx += 128) { int r = idx / (D / 8), c = idx % (D / 8); bool valid = (r < bi_cur); int off = valid ? idx : 0; cp_async_16_pred(&K_smem[swz_chunk_idx(r, c)], &kh1_u4[off], valid); cp_async_16_pred(&K_smem[swz_chunk_idx(16 + r, c)], &kh2_u4[off], valid); cp_async_16_pred(&K_smem[swz_chunk_idx(32 + r, c)], &kh3_u4[off], valid); } cp_async_commit(); cp_async_wait<0>(); __syncthreads(); int q_row = warp_id * 16 + (lane & 15); int q_k_sub = (lane >> 4); int k_row_sub = ((lane >> 4) << 3) + (lane & 7); int k_k_sub = (lane >> 3) & 1; float sc00=0,sc01=0,sc02=0,sc03=0, sc10=0,sc11=0,sc12=0,sc13=0; #pragma unroll for (int ks = 0; ks < D / 16; ++ks) { int k_chunk = ks * 2; uint32_t a0,a1,a2,a3, b0,b1,b2,b3; ldmatrix_x4(a0,a1,a2,a3, &Q_smem[swz_chunk_idx(q_row, k_chunk + q_k_sub)]); ldmatrix_x4(b0,b1,b2,b3, &K_smem[swz_chunk_idx(k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1); mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3); ldmatrix_x4(b0,b1,b2,b3, &K_smem[swz_chunk_idx(16+k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1); mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3); ldmatrix_x4(b0,b1,b2,b3, &K_smem[swz_chunk_idx(32+k_row_sub, k_chunk + k_k_sub)]); mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1); mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3); } int b_base = tid_g * 2; uint64_t a0 = (b_base+0 < bi_cur) ? make_score_key(sc00*scale, b_base+0) : 0ULL; uint64_t a1 = (b_base+1 < bi_cur) ? make_score_key(sc01*scale, b_base+1) : 0ULL; uint64_t a2 = (b_base+8 < bi_cur) ? make_score_key(sc10*scale, b_base+8) : 0ULL; uint64_t a3 = (b_base+9 < bi_cur) ? make_score_key(sc11*scale, b_base+9) : 0ULL; uint64_t g0 = (b_base+0 < bi_cur) ? make_score_key(sc02*scale, b_base+0) : 0ULL; uint64_t g1 = (b_base+1 < bi_cur) ? make_score_key(sc03*scale, b_base+1) : 0ULL; uint64_t g2 = (b_base+8 < bi_cur) ? make_score_key(sc12*scale, b_base+8) : 0ULL; uint64_t g3 = (b_base+9 < bi_cur) ? make_score_key(sc13*scale, b_base+9) : 0ULL; const float* imp_head = imp_diag + int64_t(bh) * S; int t0 = bi_cur * BLOCK_SIZE + r0, t1 = bi_cur * BLOCK_SIZE + r1; bool v0 = (t0 < S), v1 = (t1 < S); uint64_t d0 = (tid_g == 0 && v0) ? make_score_key(imp_head[t0], bi_cur) : 0ULL; uint64_t d1 = (tid_g == 0 && v1) ? make_score_key(imp_head[t1], bi_cur) : 0ULL; uint32_t msk0 = v0 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u; uint32_t msk1 = v1 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u; bool fp0 = false, fp1 = false; #pragma unroll for (int u = 0; u < TOP_N; ++u) { uint64_t w0 = pop_max4_5(a0, a1, a2, a3, d0); if (v0 && w0 != 0ULL) { int b = int(w0 & 0xFFFFFFFFu); msk0 |= (1u << b); if (b == bi_cur - 1) fp0 = true; } uint64_t w1 = pop_max4_5(g0, g1, g2, g3, d1); if (v1 && w1 != 0ULL) { int b = int(w1 & 0xFFFFFFFFu); msk1 |= (1u << b); if (b == bi_cur - 1) fp1 = true; } } uint32_t my_u = msk0 | msk1; #pragma unroll for (int off = 16; off > 0; off >>= 1) my_u |= __shfl_xor_sync(0xffffffff, my_u, off); __shared__ uint32_t s_um[4]; if (lane == 0) s_um[warp_id] = my_u; __syncthreads(); uint32_t rem_mask = s_um[0] | s_um[1] | s_um[2] | s_um[3]; int bi = __ffs(rem_mask) - 1; rem_mask &= (rem_mask - 1u); load_tile_swz_async(K_smem, K_head, bi * BLOCK_SIZE, S, tid, 128); cp_async_commit(); load_tile_swz_async(V_smem, V_head, bi * BLOCK_SIZE, S, tid, 128); cp_async_commit(); cp_async_wait<1>(); __syncthreads(); float o[D / 8][4]; #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][0]=0; o[di][1]=0; o[di][2]=0; o[di][3]=0; } float m0 = -1e30f, l0 = 0.0f; bool has0 = false; float m1 = -1e30f, l1 = 0.0f; bool has1 = false; const float scale_log2e = scale * 1.4426950408889634f; while (true) { float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); uint32_t p_lo[8], p_hi[8]; bool keep0 = ((msk0 >> bi) & 1u) != 0u; bool keep1 = ((msk1 >> bi) & 1u) != 0u; if (bi < bi_cur - 1) { online_softmax_prep(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, keep0, keep1, r0, r1, false, false, false, scale_log2e, lane); } else { online_softmax_prep(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, keep0, keep1, r0, r1, (bi == bi_cur), fp0, fp1, scale_log2e, lane); } cp_async_wait<0>(); __syncthreads(); bool has_next = (rem_mask != 0u); int bi_next = 0; if (has_next) { bi_next = __ffs(rem_mask) - 1; rem_mask &= (rem_mask - 1u); load_tile_swz_async(K_smem, K_head, bi_next * BLOCK_SIZE, S, tid, 128); cp_async_commit(); } warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); if (!has_next) break; cp_async_wait<0>(); __syncthreads(); load_tile_swz_async(V_smem, V_head, bi_next * BLOCK_SIZE, S, tid, 128); cp_async_commit(); bi = bi_next; } __syncthreads(); store_O_bf16_via_smem(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane); } // ============================================================================= // PATH B: 4-Kernel Lock-Free Inverted-Index Block-Centric Sparse FlashAttention // ============================================================================= struct WorkChunk { uint16_t g; uint16_t bi; uint32_t start; uint32_t count; }; // B-Kernel 2 (FUSED!): Exact 3xBF16 Top-8 + Lock-Free Direct q_list Scatter + Chunk Emission! template __device__ __forceinline__ void score_topk_and_scatter_body( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K_h, const float* __restrict__ imp_diag, uint8_t* __restrict__ remote_count, uint8_t* __restrict__ full_prev, uint32_t* __restrict__ bucket_count, uint32_t* __restrict__ q_list, WorkChunk* __restrict__ chunk_table, uint32_t* __restrict__ d_num_chunks, int bh_start, int S, int n_blocks, int q_stride, int chunk_q, float scale ) { extern __shared__ char smem_raw[]; constexpr int Q_U4 = BLOCK_SIZE * (D / 8); uint4* Q_smem = reinterpret_cast(smem_raw); uint4* K_dbl_smem = Q_smem + Q_U4; int g = blockIdx.x; int bi_cur = (n_blocks - 1) - blockIdx.y; int bh = bh_start + g; int tid = threadIdx.x; int warp_id = tid >> 5; int lane = tid & 31; int gid = lane >> 2; int tid_g = lane & 3; int r0 = warp_id * 16 + gid; int r1 = r0 + 8; const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D; const __nv_bfloat16* kh_head = K_h + int64_t(bh) * 3 * n_blocks * D; // Issue Q_smem cp.async WITHOUT waiting: score_3bf16_to_reg_heaps commits Q_smem + K_h1[0] // in one batch and overlaps K_h2[0] behind their arrival! load_tile_swz_async(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 128); uint64_t h0, h1, h2, h3, h4, h5, h6, h7; uint64_t g0, g1, g2, g3, g4, g5, g6, g7; score_3bf16_to_reg_heaps( h0, h1, h2, h3, h4, h5, h6, h7, g0, g1, g2, g3, g4, g5, g6, g7, K_dbl_smem, Q_smem, kh_head, n_blocks, bi_cur, scale, tid, warp_id, lane ); const float* imp_head = imp_diag + int64_t(bh) * S; constexpr int MAX_SMEM_HIST = (D == 64) ? 3500 : 7500; bool use_smem_hist = (bi_cur <= MAX_SMEM_HIST); uint32_t* smem_rblp = reinterpret_cast(smem_raw); uint8_t* smem_krem = reinterpret_cast(smem_rblp + BLOCK_SIZE * TOP_N); uint32_t* smem_hist = reinterpret_cast(smem_krem + 64); __syncthreads(); if (use_smem_hist) { for (int b = tid; b < bi_cur; b += 128) smem_hist[b] = 0u; __syncthreads(); } int t0 = bi_cur * BLOCK_SIZE + r0; int t1 = bi_cur * BLOCK_SIZE + r1; bool v0 = (t0 < S); bool v1 = (t1 < S); float d_mult = (D == 64) ? 8.0f : 1.0f; uint64_t d0 = (tid_g == 0 && v0) ? make_score_key(imp_head[t0] * d_mult, bi_cur) : 0ULL; uint64_t d1 = (tid_g == 0 && v1) ? make_score_key(imp_head[t1] * d_mult, bi_cur) : 0ULL; uint32_t* bc_head = bucket_count + int64_t(g) * n_blocks; uint32_t* ql_head = q_list + int64_t(g) * n_blocks * q_stride; uint32_t cq = static_cast(chunk_q); auto emit_row_and_hist = [&]( uint64_t& k0, uint64_t& k1, uint64_t& k2, uint64_t& k3, uint64_t& k4, uint64_t& k5, uint64_t& k6, uint64_t& k7, uint64_t& kd, int r_loc, int t, bool valid_t ) { uint8_t fp = 0; uint8_t k_rem = 0; #pragma unroll for (int u = 0; u < TOP_N; ++u) { uint64_t w = pop_max4_9(k0, k1, k2, k3, k4, k5, k6, k7, kd); if (tid_g == 0 && valid_t && w != 0ULL) { int b = static_cast(w & 0xFFFFFFFFu); if (b == bi_cur - 1) { fp = 1; } else if (b < bi_cur - 1) { if (use_smem_hist) { uint32_t lp = atomicAdd(&smem_hist[b], 1u); smem_rblp[r_loc * TOP_N + k_rem] = (static_cast(b) << 8) | lp; } else { uint32_t pos = atomicAdd(&bc_head[b], 1u); ql_head[int64_t(b) * q_stride + pos] = (static_cast(t) << 3) | k_rem; uint32_t nc = pos + 1u; if ((nc % cq) == 0u) { uint32_t cid = atomicAdd(d_num_chunks, 1u); chunk_table[cid] = WorkChunk{ static_cast(g), static_cast(b), nc - cq, cq }; } } k_rem++; } } } if (tid_g == 0) { if (valid_t) { int64_t idx_gt = int64_t(g) * S + t; full_prev[idx_gt] = fp; remote_count[idx_gt] = k_rem; } if (use_smem_hist) { smem_krem[r_loc] = k_rem; } } }; emit_row_and_hist(h0, h1, h2, h3, h4, h5, h6, h7, d0, r0, t0, v0); emit_row_and_hist(g0, g1, g2, g3, g4, g5, g6, g7, d1, r1, t1, v1); if (use_smem_hist) { __syncthreads(); for (int b = tid; b < bi_cur; b += 128) { uint32_t c = smem_hist[b]; if (c > 0u) { uint32_t old_c = atomicAdd(&bc_head[b], c); smem_hist[b] = old_c; uint32_t new_c = old_c + c; uint32_t k_first = (old_c / cq) + 1u; uint32_t k_last = new_c / cq; for (uint32_t k = k_first; k <= k_last; ++k) { uint32_t cid = atomicAdd(d_num_chunks, 1u); chunk_table[cid] = WorkChunk{ static_cast(g), static_cast(b), (k - 1u) * cq, cq }; } } } __syncthreads(); int base_t = bi_cur * BLOCK_SIZE; #pragma unroll for (int idx = tid; idx < BLOCK_SIZE * TOP_N; idx += 128) { int r = idx >> 3; int s = idx & 7; if (s < smem_krem[r]) { uint32_t rblp = smem_rblp[idx]; uint32_t b = rblp >> 8; uint32_t lp = rblp & 0xFFu; uint32_t pos = smem_hist[b] + lp; uint32_t t = static_cast(base_t + r); ql_head[int64_t(b) * q_stride + pos] = (t << 3) | static_cast(s); } } } } __global__ __launch_bounds__(128, 6) void score_topk_and_scatter_64( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K_h, const float* __restrict__ imp_diag, uint8_t* __restrict__ remote_count, uint8_t* __restrict__ full_prev, uint32_t* __restrict__ bucket_count, uint32_t* __restrict__ q_list, WorkChunk* __restrict__ chunk_table, uint32_t* __restrict__ d_num_chunks, int bh_start, int S, int n_blocks, int q_stride, int chunk_q, float scale ) { score_topk_and_scatter_body<64>( Q, K_h, imp_diag, remote_count, full_prev, bucket_count, q_list, chunk_table, d_num_chunks, bh_start, S, n_blocks, q_stride, chunk_q, scale ); } __global__ __launch_bounds__(128, 3) void score_topk_and_scatter_128( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K_h, const float* __restrict__ imp_diag, uint8_t* __restrict__ remote_count, uint8_t* __restrict__ full_prev, uint32_t* __restrict__ bucket_count, uint32_t* __restrict__ q_list, WorkChunk* __restrict__ chunk_table, uint32_t* __restrict__ d_num_chunks, int bh_start, int S, int n_blocks, int q_stride, int chunk_q, float scale ) { score_topk_and_scatter_body<128>( Q, K_h, imp_diag, remote_count, full_prev, bucket_count, q_list, chunk_table, d_num_chunks, bh_start, S, n_blocks, q_stride, chunk_q, scale ); } // B-Kernel 3: MMA-Native Zero-SMEM-Epilogue + Register-Preloaded ql_b + Overlapped Q Prefetch! template __global__ __launch_bounds__(128, (D == 64) ? 4 : 2) void remote_block_attn_kernel( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const uint32_t* __restrict__ q_list, const WorkChunk* __restrict__ chunk_table, uint32_t* __restrict__ bucket_count, uint32_t* __restrict__ d_counters, int8_t* __restrict__ o_partial, uint2* __restrict__ mls_partial, int bh_start, int H_grp, int S, int n_blocks, int q_stride, int chunk_q, float scale ) { extern __shared__ char smem_raw[]; constexpr int TILE_U4 = BLOCK_SIZE * (D / 8); uint4* K_smem = reinterpret_cast(smem_raw); uint4* V_smem = K_smem + TILE_U4; uint4* Q_smem = V_smem + TILE_U4; WorkChunk* s_wc = reinterpret_cast(Q_smem + TILE_U4); int tid = threadIdx.x; int warp_id = tid >> 5; int lane = tid & 31; int gid = lane >> 2; int tid_g = lane & 3; int r0 = warp_id * 16 + gid; int r1 = r0 + 8; uint32_t num_chunks = d_counters[0]; uint32_t rem_nb = (n_blocks > 2) ? static_cast(n_blocks - 2) : 0u; uint32_t total_rem_buckets = static_cast(H_grp) * rem_nb; uint32_t cq = static_cast(chunk_q); const float scale_log2e = scale * 1.4426950408889634f; constexpr int CHUNKS_PER_ROW = D / 8; constexpr int WARP_Q_CHUNKS = 16 * CHUNKS_PER_ROW; constexpr int I8_U4_PER_ROW = D / 16; bool in_phase2 = false; while (true) { if (tid == 0) { WorkChunk wc_tmp{0, 0, 0, 0}; if (!in_phase2) { uint32_t cid = atomicAdd(&d_counters[1], 1u); if (cid < num_chunks) { wc_tmp = chunk_table[cid]; } else { in_phase2 = true; } } if (in_phase2) { while (true) { uint32_t seq_idx = atomicAdd(&d_counters[2], 1u); if (seq_idx >= total_rem_buckets) { wc_tmp.count = 0u; break; } uint32_t bi_t = seq_idx / static_cast(H_grp); uint32_t g_t = seq_idx % static_cast(H_grp); uint32_t b_idx = g_t * static_cast(n_blocks) + bi_t; uint32_t c = bucket_count[b_idx]; if (c > 0u) { bucket_count[b_idx] = 0u; uint32_t rem = c % cq; if (rem > 0u) { wc_tmp.g = static_cast(g_t); wc_tmp.bi = static_cast(bi_t); wc_tmp.start = c - rem; wc_tmp.count = rem; break; } } } } *s_wc = wc_tmp; } __syncthreads(); WorkChunk wc = *s_wc; if (wc.count == 0u) break; int g = wc.g; int bh = bh_start + g; int bi = wc.bi; const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D; const __nv_bfloat16* K_head = K + int64_t(bh) * S * D; const __nv_bfloat16* V_head = V + int64_t(bh) * S * D; const uint4* Q_u4 = reinterpret_cast(Q_head); const uint32_t* ql_b = q_list + (int64_t(g) * n_blocks + bi) * q_stride; uint4* o_dst_u4 = reinterpret_cast(o_partial + int64_t(g) * S * TOP_N * D); uint2* mls_part_head = mls_partial + int64_t(g) * S * TOP_N; auto load_warp_q_from_reg = [&](uint32_t pk_lane, int m_val_load) { #pragma unroll for (int idx = lane; idx < WARP_Q_CHUNKS; idx += 32) { int rl = idx / CHUNKS_PER_ROW; int c = idx % CHUNKS_PER_ROW; int r = warp_id * 16 + rl; uint32_t pk_r = __shfl_sync(0xffffffff, pk_lane, rl); bool valid = (r < m_val_load); int t = valid ? static_cast(pk_r >> 3) : 0; cp_async_16_pred(&Q_smem[swz_chunk_idx(r, c)], Q_u4 + int64_t(t) * CHUNKS_PER_ROW + c, valid); } }; int m_valid0 = min(BLOCK_SIZE, static_cast(wc.count)); uint32_t pk_cur = 0u; if (lane < 16) { int r = warp_id * 16 + lane; if (r < m_valid0) pk_cur = __ldcs(&ql_b[wc.start + r]); } load_tile_swz_async(K_smem, K_head, bi * BLOCK_SIZE, S, tid, 128); if (warp_id * 16 < m_valid0) { load_warp_q_from_reg(pk_cur, m_valid0); } cp_async_commit(); load_tile_swz_async(V_smem, V_head, bi * BLOCK_SIZE, S, tid, 128); cp_async_commit(); uint32_t pk0 = __shfl_sync(0xffffffff, pk_cur, gid); uint32_t pk1 = __shfl_sync(0xffffffff, pk_cur, gid + 8); cp_async_wait<1>(); __syncthreads(); for (uint32_t q_off = 0; q_off < wc.count; q_off += BLOCK_SIZE) { int m_valid = min(BLOCK_SIZE, static_cast(wc.count - q_off)); bool active_warp = (warp_id * 16 < m_valid); bool is_first = (q_off == 0u); if (!is_first) { if (!active_warp) break; cp_async_wait<0>(); __syncwarp(); } bool has_next_q = (q_off + BLOCK_SIZE < wc.count); int m_next = has_next_q ? min(BLOCK_SIZE, static_cast(wc.count - (q_off + BLOCK_SIZE))) : 0; bool active_next = has_next_q && (warp_id * 16 < m_next); uint32_t p_lo[8], p_hi[8]; float m0 = 0.0f, l0 = 1.0f, m1 = 0.0f, l1 = 1.0f; bool did_prefetch = false; uint32_t pk_next = 0u; if (active_warp) { // Issue non-blocking ql_b load for NEXT 64 queries BEFORE QK MMA -> hides L2 latency! if (active_next && lane < 16) { int r_n = warp_id * 16 + lane; if (r_n < m_next) pk_next = __ldcs(&ql_b[wc.start + q_off + BLOCK_SIZE + r_n]); } float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); __syncwarp(); if (active_next) { load_warp_q_from_reg(pk_next, m_next); cp_async_commit(); did_prefetch = true; } float mb0 = -1e30f, mb1 = -1e30f; #pragma unroll for (int ni = 0; ni < 8; ++ni) { mb0 = fmaxf(mb0, fmaxf(s[ni][0], s[ni][1])); mb1 = fmaxf(mb1, fmaxf(s[ni][2], s[ni][3])); } mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 1)); mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 2)); mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 1)); mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 2)); m0 = mb0 * scale_log2e; m1 = mb1 * scale_log2e; float neg_m0 = -m0; float neg_m1 = -m1; float lb0 = 0.0f, lb1 = 0.0f; #pragma unroll for (int ni = 0; ni < 8; ++ni) { float p0 = fast_exp2f(fmaf(s[ni][0], scale_log2e, neg_m0)); float p1 = fast_exp2f(fmaf(s[ni][1], scale_log2e, neg_m0)); float p2 = fast_exp2f(fmaf(s[ni][2], scale_log2e, neg_m1)); float p3 = fast_exp2f(fmaf(s[ni][3], scale_log2e, neg_m1)); lb0 += p0 + p1; lb1 += p2 + p3; p_lo[ni] = pack_bf16x2(p0, p1); p_hi[ni] = pack_bf16x2(p2, p3); } lb0 += __shfl_xor_sync(0xffffffff, lb0, 1); lb0 += __shfl_xor_sync(0xffffffff, lb0, 2); lb1 += __shfl_xor_sync(0xffffffff, lb1, 1); lb1 += __shfl_xor_sync(0xffffffff, lb1, 2); l0 = lb0; l1 = lb1; } if (is_first) { if (did_prefetch) { cp_async_wait<1>(); } else { cp_async_wait<0>(); } __syncthreads(); } if (active_warp) { float o[D / 8][4]; #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][0] = 0.0f; o[di][1] = 0.0f; o[di][2] = 0.0f; o[di][3] = 0.0f; } warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); float amax0 = 0.0f, amax1 = 0.0f; #pragma unroll for (int di = 0; di < D / 8; ++di) { amax0 = fmaxf(amax0, fmaxf(fabsf(o[di][0]), fabsf(o[di][1]))); amax1 = fmaxf(amax1, fmaxf(fabsf(o[di][2]), fabsf(o[di][3]))); } amax0 = fmaxf(amax0, __shfl_xor_sync(0xffffffff, amax0, 1)); amax0 = fmaxf(amax0, __shfl_xor_sync(0xffffffff, amax0, 2)); amax1 = fmaxf(amax1, __shfl_xor_sync(0xffffffff, amax1, 1)); amax1 = fmaxf(amax1, __shfl_xor_sync(0xffffffff, amax1, 2)); float inv_s0 = (amax0 > 0.0f) ? __fdividef(127.0f, amax0) : 0.0f; float inv_s1 = (amax1 > 0.0f) ? __fdividef(127.0f, amax1) : 0.0f; float raw_s0 = amax0 * (1.0f / 127.0f); float raw_s1 = amax1 * (1.0f / 127.0f); if (r0 < m_valid) { int64_t row0 = static_cast(pk0); if (tid_g == 0) mls_part_head[row0] = pack_mls(m0, l0, raw_s0); uint4 u0; u0.x = pack_i8x4(o[0][0] * inv_s0, o[0][1] * inv_s0, o[1][0] * inv_s0, o[1][1] * inv_s0); u0.y = pack_i8x4(o[2][0] * inv_s0, o[2][1] * inv_s0, o[3][0] * inv_s0, o[3][1] * inv_s0); u0.z = pack_i8x4(o[4][0] * inv_s0, o[4][1] * inv_s0, o[5][0] * inv_s0, o[5][1] * inv_s0); u0.w = pack_i8x4(o[6][0] * inv_s0, o[6][1] * inv_s0, o[7][0] * inv_s0, o[7][1] * inv_s0); o_dst_u4[row0 * I8_U4_PER_ROW + tid_g] = u0; if constexpr (D == 128) { uint4 u0h; u0h.x = pack_i8x4(o[8][0] * inv_s0, o[8][1] * inv_s0, o[9][0] * inv_s0, o[9][1] * inv_s0); u0h.y = pack_i8x4(o[10][0] * inv_s0, o[10][1] * inv_s0, o[11][0] * inv_s0, o[11][1] * inv_s0); u0h.z = pack_i8x4(o[12][0] * inv_s0, o[12][1] * inv_s0, o[13][0] * inv_s0, o[13][1] * inv_s0); u0h.w = pack_i8x4(o[14][0] * inv_s0, o[14][1] * inv_s0, o[15][0] * inv_s0, o[15][1] * inv_s0); o_dst_u4[row0 * I8_U4_PER_ROW + 4 + tid_g] = u0h; } } if (r1 < m_valid) { int64_t row1 = static_cast(pk1); if (tid_g == 0) mls_part_head[row1] = pack_mls(m1, l1, raw_s1); uint4 u1; u1.x = pack_i8x4(o[0][2] * inv_s1, o[0][3] * inv_s1, o[1][2] * inv_s1, o[1][3] * inv_s1); u1.y = pack_i8x4(o[2][2] * inv_s1, o[2][3] * inv_s1, o[3][2] * inv_s1, o[3][3] * inv_s1); u1.z = pack_i8x4(o[4][2] * inv_s1, o[4][3] * inv_s1, o[5][2] * inv_s1, o[5][3] * inv_s1); u1.w = pack_i8x4(o[6][2] * inv_s1, o[6][3] * inv_s1, o[7][2] * inv_s1, o[7][3] * inv_s1); o_dst_u4[row1 * I8_U4_PER_ROW + tid_g] = u1; if constexpr (D == 128) { uint4 u1h; u1h.x = pack_i8x4(o[8][2] * inv_s1, o[8][3] * inv_s1, o[9][2] * inv_s1, o[9][3] * inv_s1); u1h.y = pack_i8x4(o[10][2] * inv_s1, o[10][3] * inv_s1, o[11][2] * inv_s1, o[11][3] * inv_s1); u1h.z = pack_i8x4(o[12][2] * inv_s1, o[12][3] * inv_s1, o[13][2] * inv_s1, o[13][3] * inv_s1); u1h.w = pack_i8x4(o[14][2] * inv_s1, o[14][3] * inv_s1, o[15][2] * inv_s1, o[15][3] * inv_s1); o_dst_u4[row1 * I8_U4_PER_ROW + 4 + tid_g] = u1h; } } if (active_next) { pk0 = __shfl_sync(0xffffffff, pk_next, gid); pk1 = __shfl_sync(0xffffffff, pk_next, gid + 8); } } } __syncthreads(); } } // B-Kernel 4: ZERO-SPILL LPT Local Window + Register-Direct MMA-Native Merge! // On-the-fly alA/alB broadcast frees 6 registers -> 0 spills on BOTH D=64 (<=128 regs, 4 CTAs/SM) AND D=128 (2 CTAs/SM)! template __device__ __forceinline__ void local_window_and_merge_body( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const uint8_t* __restrict__ remote_count, const uint8_t* __restrict__ full_prev, const int8_t* __restrict__ o_partial, const uint2* __restrict__ mls_partial, uint32_t* __restrict__ d_counters, __nv_bfloat16* __restrict__ O, int bh_start, int S, int n_blocks, float scale ) { extern __shared__ char smem_raw[]; constexpr int TILE_U4 = BLOCK_SIZE * (D / 8); uint4* Q_smem = reinterpret_cast(smem_raw); uint4* K_smem = Q_smem + TILE_U4; uint4* V_smem = K_smem + TILE_U4; int g = blockIdx.x; int by = blockIdx.y; int bi_cur; if (n_blocks <= TOP_N) { bi_cur = (n_blocks - 1) - by; } else if (by < 4) { bi_cur = 7 - by; // 7, 6, 5, 4 (heaviest 8..5 blocks first!) } else if (by < n_blocks - 4) { bi_cur = 8 + (by - 4); // 8 .. n_blocks - 1 } else { bi_cur = (n_blocks - 1) - by; // 3, 2, 1, 0 (lightest tail last!) } int bh = bh_start + g; int tid = threadIdx.x; int warp_id = tid >> 5; int lane = tid & 31; int gid = lane >> 2; int tid_g = lane & 3; int base_lane = gid << 2; int r0 = warp_id * 16 + gid; int r1 = r0 + 8; if (blockIdx.x == 0 && blockIdx.y == 0 && tid < 3) { d_counters[tid] = 0u; } const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D; const __nv_bfloat16* K_head = K + int64_t(bh) * S * D; const __nv_bfloat16* V_head = V + int64_t(bh) * S * D; __nv_bfloat16* O_head = O + int64_t(bh) * S * D; load_tile_swz_async(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 128); cp_async_commit(); float o[D / 8][4]; #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][0] = 0.0f; o[di][1] = 0.0f; o[di][2] = 0.0f; o[di][3] = 0.0f; } float m0 = -1e30f, l0 = 0.0f; bool has0 = false; float m1 = -1e30f, l1 = 0.0f; bool has1 = false; const float scale_log2e = scale * 1.4426950408889634f; if (bi_cur < TOP_N) { load_tile_swz_async(K_smem, K_head, 0, S, tid, 128); cp_async_commit(); load_tile_swz_async(V_smem, V_head, 0, S, tid, 128); cp_async_commit(); cp_async_wait<1>(); __syncthreads(); for (int bi = 0; bi <= bi_cur; ++bi) { float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); bool has_next = (bi + 1 <= bi_cur); if (has_next) { __syncthreads(); load_tile_swz_async(K_smem, K_head, (bi + 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); } uint32_t p_lo[8], p_hi[8]; if (bi < bi_cur) { online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, false, true, true, scale_log2e, lane ); } else { online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, true, true, true, scale_log2e, lane ); } if (has_next) { cp_async_wait<1>(); } else { cp_async_wait<0>(); } __syncthreads(); warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); if (has_next) { cp_async_wait<0>(); __syncthreads(); load_tile_swz_async(V_smem, V_head, (bi + 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); } } __syncthreads(); store_O_bf16_via_smem(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane); return; } int t0 = bi_cur * BLOCK_SIZE + r0; int t1 = bi_cur * BLOCK_SIZE + r1; bool fp0 = (t0 < S) ? (full_prev[int64_t(g) * S + t0] != 0) : false; bool fp1 = (t1 < S) ? (full_prev[int64_t(g) * S + t1] != 0) : false; load_tile_swz_async(K_smem, K_head, bi_cur * BLOCK_SIZE, S, tid, 128); cp_async_commit(); load_tile_swz_async(V_smem, V_head, bi_cur * BLOCK_SIZE, S, tid, 128); cp_async_commit(); cp_async_wait<1>(); __syncthreads(); { float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); __syncthreads(); load_tile_swz_async(K_smem, K_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); uint32_t p_lo[8], p_hi[8]; online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, true, false, false, scale_log2e, lane ); cp_async_wait<1>(); __syncthreads(); warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); cp_async_wait<0>(); __syncthreads(); } if constexpr (D == 128) { const uint4* op_u4_head = reinterpret_cast(o_partial + int64_t(g) * S * TOP_N * D); const uint2* mls_part_head = mls_partial + int64_t(g) * S * TOP_N; const uint8_t* rc_head = remote_count + int64_t(g) * S; constexpr int I8_U4_PER_ROW = D / 16; bool valid_t0 = (t0 < S); bool valid_t1 = (t1 < S); int k_rem0 = valid_t0 ? rc_head[t0] : 0; int k_rem1 = valid_t1 ? rc_head[t1] : 0; int sA = tid_g; int sB = tid_g + 4; uint2 pkA0 = make_uint2(0u, 0u), pkB0 = make_uint2(0u, 0u); uint2 pkA1 = make_uint2(0u, 0u), pkB1 = make_uint2(0u, 0u); { load_tile_swz_async(V_smem, V_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); uint32_t p_lo[8], p_hi[8]; online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, false, fp0, fp1, scale_log2e, lane ); if (valid_t0 && sA < k_rem0) pkA0 = __ldcs(&mls_part_head[int64_t(t0) * TOP_N + sA]); if (valid_t0 && sB < k_rem0) pkB0 = __ldcs(&mls_part_head[int64_t(t0) * TOP_N + sB]); if (valid_t1 && sA < k_rem1) pkA1 = __ldcs(&mls_part_head[int64_t(t1) * TOP_N + sA]); if (valid_t1 && sB < k_rem1) pkB1 = __ldcs(&mls_part_head[int64_t(t1) * TOP_N + sB]); cp_async_wait<0>(); __syncthreads(); warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); } auto merge_reg_row_128 = [&](int t, bool valid_t, int k_rem, float m_loc, float l_loc, uint2 pkA, uint2 pkB, int col_off) { float mA = -1e30f, lA = 0.0f, rsA = 0.0f; float mB = -1e30f, lB = 0.0f, rsB = 0.0f; if (valid_t && sA < k_rem) unpack_mls(pkA, mA, lA, rsA); if (valid_t && sB < k_rem) unpack_mls(pkB, mB, lB, rsB); float m_fin = fmaxf(mA, mB); if (tid_g == 0) m_fin = fmaxf(m_fin, m_loc); m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 1)); m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 2)); float w_loc_scale = fast_exp2f(m_loc - m_fin); float w_loc = l_loc * w_loc_scale; float expA = (valid_t && sA < k_rem) ? fast_exp2f(mA - m_fin) : 0.0f; float expB = (valid_t && sB < k_rem) ? fast_exp2f(mB - m_fin) : 0.0f; float w_rem = lA * expA + lB * expB; w_rem += __shfl_xor_sync(0xffffffff, w_rem, 1); w_rem += __shfl_xor_sync(0xffffffff, w_rem, 2); float inv_l = __frcp_rn(w_loc + w_rem); float alpha_loc = w_loc_scale * inv_l; float alA = (expA > 1e-8f) ? (expA * inv_l * rsA) : 0.0f; float alB = (expB > 1e-8f) ? (expB * inv_l * rsB) : 0.0f; if (valid_t) { #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][col_off] *= alpha_loc; o[di][col_off + 1] *= alpha_loc; } } const uint4* op_row = op_u4_head + int64_t(t) * TOP_N * I8_U4_PER_ROW; auto accum_slot = [&](bool ok, float w, uint4 rlo, uint4 rhi) { if (!ok) return; i8x4 c0 = *reinterpret_cast(&rlo.x); i8x4 c1 = *reinterpret_cast(&rlo.y); i8x4 c2 = *reinterpret_cast(&rlo.z); i8x4 c3 = *reinterpret_cast(&rlo.w); o[0][col_off] = fmaf(w, float(c0.x), o[0][col_off]); o[0][col_off+1] = fmaf(w, float(c0.y), o[0][col_off+1]); o[1][col_off] = fmaf(w, float(c0.z), o[1][col_off]); o[1][col_off+1] = fmaf(w, float(c0.w), o[1][col_off+1]); o[2][col_off] = fmaf(w, float(c1.x), o[2][col_off]); o[2][col_off+1] = fmaf(w, float(c1.y), o[2][col_off+1]); o[3][col_off] = fmaf(w, float(c1.z), o[3][col_off]); o[3][col_off+1] = fmaf(w, float(c1.w), o[3][col_off+1]); o[4][col_off] = fmaf(w, float(c2.x), o[4][col_off]); o[4][col_off+1] = fmaf(w, float(c2.y), o[4][col_off+1]); o[5][col_off] = fmaf(w, float(c2.z), o[5][col_off]); o[5][col_off+1] = fmaf(w, float(c2.w), o[5][col_off+1]); o[6][col_off] = fmaf(w, float(c3.x), o[6][col_off]); o[6][col_off+1] = fmaf(w, float(c3.y), o[6][col_off+1]); o[7][col_off] = fmaf(w, float(c3.z), o[7][col_off]); o[7][col_off+1] = fmaf(w, float(c3.w), o[7][col_off+1]); i8x4 h0 = *reinterpret_cast(&rhi.x); i8x4 h1 = *reinterpret_cast(&rhi.y); i8x4 h2 = *reinterpret_cast(&rhi.z); i8x4 h3 = *reinterpret_cast(&rhi.w); o[8][col_off] = fmaf(w, float(h0.x), o[8][col_off]); o[8][col_off+1] = fmaf(w, float(h0.y), o[8][col_off+1]); o[9][col_off] = fmaf(w, float(h0.z), o[9][col_off]); o[9][col_off+1] = fmaf(w, float(h0.w), o[9][col_off+1]); o[10][col_off] = fmaf(w, float(h1.x), o[10][col_off]); o[10][col_off+1] = fmaf(w, float(h1.y), o[10][col_off+1]); o[11][col_off] = fmaf(w, float(h1.z), o[11][col_off]); o[11][col_off+1] = fmaf(w, float(h1.w), o[11][col_off+1]); o[12][col_off] = fmaf(w, float(h2.x), o[12][col_off]); o[12][col_off+1] = fmaf(w, float(h2.y), o[12][col_off+1]); o[13][col_off] = fmaf(w, float(h2.z), o[13][col_off]); o[13][col_off+1] = fmaf(w, float(h2.w), o[13][col_off+1]); o[14][col_off] = fmaf(w, float(h3.x), o[14][col_off]); o[14][col_off+1] = fmaf(w, float(h3.y), o[14][col_off+1]); o[15][col_off] = fmaf(w, float(h3.z), o[15][col_off]); o[15][col_off+1] = fmaf(w, float(h3.w), o[15][col_off+1]); }; #pragma unroll for (int base_s = 0; base_s < TOP_N; base_s += 4) { float src_al = (base_s == 0) ? alA : alB; float w0 = __shfl_sync(0xffffffff, src_al, base_lane + 0); float w1 = __shfl_sync(0xffffffff, src_al, base_lane + 1); float w2 = __shfl_sync(0xffffffff, src_al, base_lane + 2); float w3 = __shfl_sync(0xffffffff, src_al, base_lane + 3); if (valid_t) { bool ok0 = (base_s + 0 < k_rem) && (w0 != 0.0f); bool ok1 = (base_s + 1 < k_rem) && (w1 != 0.0f); bool ok2 = (base_s + 2 < k_rem) && (w2 != 0.0f); bool ok3 = (base_s + 3 < k_rem) && (w3 != 0.0f); uint4 rlo0, rlo1, rlo2, rlo3, rhi0, rhi1, rhi2, rhi3; if (ok0) { rlo0 = __ldcs(&op_row[(base_s + 0) * I8_U4_PER_ROW + tid_g]); rhi0 = __ldcs(&op_row[(base_s + 0) * I8_U4_PER_ROW + 4 + tid_g]); } if (ok1) { rlo1 = __ldcs(&op_row[(base_s + 1) * I8_U4_PER_ROW + tid_g]); rhi1 = __ldcs(&op_row[(base_s + 1) * I8_U4_PER_ROW + 4 + tid_g]); } if (ok2) { rlo2 = __ldcs(&op_row[(base_s + 2) * I8_U4_PER_ROW + tid_g]); rhi2 = __ldcs(&op_row[(base_s + 2) * I8_U4_PER_ROW + 4 + tid_g]); } if (ok3) { rlo3 = __ldcs(&op_row[(base_s + 3) * I8_U4_PER_ROW + tid_g]); rhi3 = __ldcs(&op_row[(base_s + 3) * I8_U4_PER_ROW + 4 + tid_g]); } accum_slot(ok0, w0, rlo0, rhi0); accum_slot(ok1, w1, rlo1, rhi1); accum_slot(ok2, w2, rlo2, rhi2); accum_slot(ok3, w3, rlo3, rhi3); } } }; merge_reg_row_128(t0, valid_t0, k_rem0, m0, l0, pkA0, pkB0, 0); merge_reg_row_128(t1, valid_t1, k_rem1, m1, l1, pkA1, pkB1, 2); } else { { load_tile_swz_async(V_smem, V_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); float s[8][4]; warp_gemm_qk_16x64(s, Q_smem, K_smem, warp_id, lane); uint32_t p_lo[8], p_hi[8]; online_softmax_prep( o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, false, fp0, fp1, scale_log2e, lane ); cp_async_wait<0>(); __syncthreads(); warp_gemm_pv_16xD(o, p_lo, p_hi, V_smem, lane); } const uint4* op_u4_head = reinterpret_cast(o_partial + int64_t(g) * S * TOP_N * D); const uint2* mls_part_head = mls_partial + int64_t(g) * S * TOP_N; const uint8_t* rc_head = remote_count + int64_t(g) * S; constexpr int I8_U4_PER_ROW = D / 16; auto merge_reg_row_64 = [&](int t, float m_loc, float l_loc, int col_off) { bool valid_t = (t < S); int k_rem = valid_t ? rc_head[t] : 0; int sA = tid_g; int sB = tid_g + 4; float mA = -1e30f, lA = 0.0f, rsA = 0.0f; float mB = -1e30f, lB = 0.0f, rsB = 0.0f; if (valid_t && sA < k_rem) unpack_mls(__ldcs(&mls_part_head[int64_t(t) * TOP_N + sA]), mA, lA, rsA); if (valid_t && sB < k_rem) unpack_mls(__ldcs(&mls_part_head[int64_t(t) * TOP_N + sB]), mB, lB, rsB); float m_fin = fmaxf(mA, mB); if (tid_g == 0) m_fin = fmaxf(m_fin, m_loc); m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 1)); m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 2)); float w_loc_scale = fast_exp2f(m_loc - m_fin); float w_loc = l_loc * w_loc_scale; float expA = (valid_t && sA < k_rem) ? fast_exp2f(mA - m_fin) : 0.0f; float expB = (valid_t && sB < k_rem) ? fast_exp2f(mB - m_fin) : 0.0f; float w_rem = lA * expA + lB * expB; w_rem += __shfl_xor_sync(0xffffffff, w_rem, 1); w_rem += __shfl_xor_sync(0xffffffff, w_rem, 2); float inv_l = __frcp_rn(w_loc + w_rem); float alpha_loc = w_loc_scale * inv_l; float alA = (expA > 1e-8f) ? (expA * inv_l * rsA) : 0.0f; float alB = (expB > 1e-8f) ? (expB * inv_l * rsB) : 0.0f; if (valid_t) { #pragma unroll for (int di = 0; di < D / 8; ++di) { o[di][col_off] *= alpha_loc; o[di][col_off + 1] *= alpha_loc; } } const uint4* op_row = op_u4_head + int64_t(t) * TOP_N * I8_U4_PER_ROW; #pragma unroll for (int base_s = 0; base_s < TOP_N; base_s += 2) { float src_al = (base_s < 4) ? alA : alB; float w0 = __shfl_sync(0xffffffff, src_al, base_lane + (base_s & 3)); float w1 = __shfl_sync(0xffffffff, src_al, base_lane + ((base_s + 1) & 3)); if (valid_t) { bool ok0 = (base_s < k_rem) && (w0 != 0.0f); bool ok1 = (base_s + 1 < k_rem) && (w1 != 0.0f); uint4 rlo0, rlo1; if (ok0) rlo0 = __ldcs(&op_row[(base_s) * I8_U4_PER_ROW + tid_g]); if (ok1) rlo1 = __ldcs(&op_row[(base_s + 1) * I8_U4_PER_ROW + tid_g]); auto accum_64 = [&](bool ok, float w, uint4 rlo) { if (!ok) return; i8x4 c0 = *reinterpret_cast(&rlo.x); i8x4 c1 = *reinterpret_cast(&rlo.y); i8x4 c2 = *reinterpret_cast(&rlo.z); i8x4 c3 = *reinterpret_cast(&rlo.w); o[0][col_off] = fmaf(w, float(c0.x), o[0][col_off]); o[0][col_off+1] = fmaf(w, float(c0.y), o[0][col_off+1]); o[1][col_off] = fmaf(w, float(c0.z), o[1][col_off]); o[1][col_off+1] = fmaf(w, float(c0.w), o[1][col_off+1]); o[2][col_off] = fmaf(w, float(c1.x), o[2][col_off]); o[2][col_off+1] = fmaf(w, float(c1.y), o[2][col_off+1]); o[3][col_off] = fmaf(w, float(c1.z), o[3][col_off]); o[3][col_off+1] = fmaf(w, float(c1.w), o[3][col_off+1]); o[4][col_off] = fmaf(w, float(c2.x), o[4][col_off]); o[4][col_off+1] = fmaf(w, float(c2.y), o[4][col_off+1]); o[5][col_off] = fmaf(w, float(c2.z), o[5][col_off]); o[5][col_off+1] = fmaf(w, float(c2.w), o[5][col_off+1]); o[6][col_off] = fmaf(w, float(c3.x), o[6][col_off]); o[6][col_off+1] = fmaf(w, float(c3.y), o[6][col_off+1]); o[7][col_off] = fmaf(w, float(c3.z), o[7][col_off]); o[7][col_off+1] = fmaf(w, float(c3.w), o[7][col_off+1]); }; accum_64(ok0, w0, rlo0); accum_64(ok1, w1, rlo1); } } }; merge_reg_row_64(t0, m0, l0, 0); merge_reg_row_64(t1, m1, l1, 2); } store_O_bf16_via_smem(O_head, Q_smem, o, 1.0f, 1.0f, bi_cur, S, warp_id, lane); } __global__ __launch_bounds__(128, 4) void local_window_and_merge_64( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const uint8_t* __restrict__ rc, const uint8_t* __restrict__ fp, const int8_t* __restrict__ op, const uint2* __restrict__ mlsp, uint32_t* __restrict__ ctr, __nv_bfloat16* __restrict__ O, int bh_start, int S, int n_blocks, float scale ) { local_window_and_merge_body<64>(Q, K, V, rc, fp, op, mlsp, ctr, O, bh_start, S, n_blocks, scale); } __global__ void local_window_and_merge_128( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const uint8_t* __restrict__ rc, const uint8_t* __restrict__ fp, const int8_t* __restrict__ op, const uint2* __restrict__ mlsp, uint32_t* __restrict__ ctr, __nv_bfloat16* __restrict__ O, int bh_start, int S, int n_blocks, float scale ) { local_window_and_merge_body<128>(Q, K, V, rc, fp, op, mlsp, ctr, O, bh_start, S, n_blocks, scale); } // ============================================================================= // General fallback CUDA kernel (arbitrary D != 64, 128) // ============================================================================= __global__ void nsa_fallback_kernel( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, __nv_bfloat16* __restrict__ O, int S, int D, float scale ) { int bh = blockIdx.y; int t = blockIdx.x; int tid = threadIdx.x; if (t >= S) return; int bi_cur = t / BLOCK_SIZE; const __nv_bfloat16* q_t = Q + (int64_t(bh) * S + t) * D; const __nv_bfloat16* K_h = K + int64_t(bh) * S * D; const __nv_bfloat16* V_h = V + int64_t(bh) * S * D; __nv_bfloat16* o_t = O + (int64_t(bh) * S + t) * D; __shared__ float s_imp[1024]; __shared__ int s_top[TOP_N]; for (int bi = tid; bi <= bi_cur; bi += blockDim.x) { int s0 = bi * BLOCK_SIZE; int s1 = min(s0 + BLOCK_SIZE, t + 1); float sum = 0.0f; for (int j = s0; j < s1; ++j) { const __nv_bfloat16* kj = K_h + int64_t(j) * D; float dot = 0.0f; for (int d = 0; d < D; ++d) dot += __bfloat162float(q_t[d]) * __bfloat162float(kj[d]); sum += dot * scale; } s_imp[bi] = sum / float(s1 - s0); } __syncthreads(); if (tid == 0) { uint64_t top_k[TOP_N]; for (int u = 0; u < TOP_N; ++u) top_k[u] = 0ULL; for (int bi = 0; bi <= bi_cur; ++bi) { uint64_t k = make_score_key(s_imp[bi], bi); int mp = 0; for (int u = 1; u < TOP_N; ++u) if (top_k[u] < top_k[mp]) mp = u; if (k > top_k[mp]) top_k[mp] = k; } for (int u = 0; u < TOP_N; ++u) s_top[u] = (top_k[u] != 0ULL) ? static_cast(top_k[u] & 0xFFFFFFFFu) : -1; } __syncthreads(); int w0 = max(0, t + 1 - BLOCK_SIZE); __shared__ float s_m, s_l; if (tid == 0) { float m = -1e30f; for (int bi = 0; bi <= bi_cur; ++bi) { bool sel = false; for (int u = 0; u < TOP_N; ++u) if (s_top[u] == bi) sel = true; int s0 = bi * BLOCK_SIZE; int s1 = min(s0 + BLOCK_SIZE, t + 1); if (!sel && s1 <= w0) continue; int j0 = sel ? s0 : max(s0, w0); for (int j = j0; j < s1; ++j) { const __nv_bfloat16* kj = K_h + int64_t(j) * D; float dot = 0.0f; for (int d = 0; d < D; ++d) dot += __bfloat162float(q_t[d]) * __bfloat162float(kj[d]); m = fmaxf(m, dot * scale); } } s_m = m; float l = 0.0f; for (int bi = 0; bi <= bi_cur; ++bi) { bool sel = false; for (int u = 0; u < TOP_N; ++u) if (s_top[u] == bi) sel = true; int s0 = bi * BLOCK_SIZE; int s1 = min(s0 + BLOCK_SIZE, t + 1); if (!sel && s1 <= w0) continue; int j0 = sel ? s0 : max(s0, w0); for (int j = j0; j < s1; ++j) { const __nv_bfloat16* kj = K_h + int64_t(j) * D; float dot = 0.0f; for (int d = 0; d < D; ++d) dot += __bfloat162float(q_t[d]) * __bfloat162float(kj[d]); l += expf(dot * scale - m); } } s_l = l; } __syncthreads(); float m = s_m, inv_l = 1.0f / s_l; for (int d = tid; d < D; d += blockDim.x) { float acc = 0.0f; for (int bi = 0; bi <= bi_cur; ++bi) { bool sel = false; for (int u = 0; u < TOP_N; ++u) if (s_top[u] == bi) sel = true; int s0 = bi * BLOCK_SIZE; int s1 = min(s0 + BLOCK_SIZE, t + 1); if (!sel && s1 <= w0) continue; int j0 = sel ? s0 : max(s0, w0); for (int j = j0; j < s1; ++j) { const __nv_bfloat16* kj = K_h + int64_t(j) * D; float dot = 0.0f; for (int k = 0; k < D; ++k) dot += __bfloat162float(q_t[k]) * __bfloat162float(kj[k]); acc += expf(dot * scale - m) * inv_l * __bfloat162float(V_h[int64_t(j) * D + d]); } } o_t[d] = __float2bfloat16_rn(acc); } } // ============================================================================= // Host Dispatch + Persistent Workspace // ============================================================================= struct NSAWorkspace { torch::Tensor k_h; torch::Tensor imp_diag; torch::Tensor remote_count, full_prev; torch::Tensor bucket_count; torch::Tensor chunk_table, q_list; torch::Tensor o_partial, mls_partial; torch::Tensor d_counters; int64_t cap_km = 0; int64_t cap_bhs = 0; int64_t cap_grp_s = 0; int64_t cap_buckets = 0; int64_t cap_qlist = 0; int64_t cap_chunks = 0; int64_t cap_opart = 0; bool smem_configured = false; }; static NSAWorkspace g_ws; static void configure_smem_once() { if (g_ws.smem_configured) return; auto set_smem = [](const void* func, int bytes) { cudaFuncSetAttribute(func, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes); }; set_smem((const void*)fused_nsa_direct_bn16_splitkv_64, 49152); set_smem((const void*)fused_nsa_direct_bn16_128, 57344); set_smem((const void*)score_topk_and_scatter_64, 24576); set_smem((const void*)score_topk_and_scatter_128, 40960); set_smem((const void*)remote_block_attn_kernel<64>, 32768); set_smem((const void*)remote_block_attn_kernel<128>, 57344); set_smem((const void*)local_window_and_merge_64, 32768); set_smem((const void*)local_window_and_merge_128, 57344); g_ws.smem_configured = true; } template static void dispatch_nsa_impl( const torch::Tensor& q, const torch::Tensor& k, const torch::Tensor& v, torch::Tensor& out, int direct_max_blocks, int h_grp_budget_mb, int chunk_q_req ) { configure_smem_once(); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); int B = q.size(0); int H = q.size(1); int S = q.size(2); int BH = B * H; int n_blocks = (S + BLOCK_SIZE - 1) / BLOCK_SIZE; float scale = 1.0f / sqrtf(float(D)); auto opts_bf16 = q.options(); auto opts_f32 = q.options().dtype(torch::kFloat32); int64_t need_km = int64_t(BH) * 3 * n_blocks * D; if (g_ws.cap_km < need_km) { g_ws.k_h = torch::empty({need_km}, opts_bf16); g_ws.cap_km = need_km; } int64_t need_bhs = int64_t(BH) * S; if (g_ws.cap_bhs < need_bhs) { g_ws.imp_diag = torch::empty({need_bhs}, opts_f32); g_ws.cap_bhs = need_bhs; } if (!g_ws.d_counters.defined()) { g_ws.d_counters = torch::zeros({4}, q.options().dtype(torch::kInt32)); } const __nv_bfloat16* q_ptr = reinterpret_cast(q.data_ptr()); const __nv_bfloat16* k_ptr = reinterpret_cast(k.data_ptr()); const __nv_bfloat16* v_ptr = reinterpret_cast(v.data_ptr()); __nv_bfloat16* o_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()); __nv_bfloat16* kh_ptr = reinterpret_cast<__nv_bfloat16*>(g_ws.k_h.data_ptr()); float* imp_ptr = g_ws.imp_diag.data_ptr(); uint32_t* d_ctr_ptr = reinterpret_cast(g_ws.d_counters.data_ptr()); compute_k_mean_and_diag_kernel<<>>( q_ptr, k_ptr, kh_ptr, imp_ptr, d_ctr_ptr, S, n_blocks, scale ); if (n_blocks <= direct_max_blocks && n_blocks <= 16) { if constexpr (D == 64) { size_t smem_a = 5 * BLOCK_SIZE * 64 * sizeof(__nv_bfloat16); fused_nsa_direct_bn16_splitkv_64<<>>( q_ptr, k_ptr, v_ptr, kh_ptr, imp_ptr, o_ptr, BH, S, n_blocks, scale ); } else { size_t smem_a = 3 * BLOCK_SIZE * 128 * sizeof(__nv_bfloat16); fused_nsa_direct_bn16_128<<>>( q_ptr, k_ptr, v_ptr, kh_ptr, imp_ptr, o_ptr, S, n_blocks, scale ); } return; } int q_stride = ((S + 31) & ~31) + 32; int64_t bytes_per_head = int64_t(S) * TOP_N * D * sizeof(int8_t); int64_t budget_bytes = int64_t(h_grp_budget_mb) * 1024LL * 1024LL; int H_grp = static_cast(budget_bytes / std::max(1, bytes_per_head)); int64_t qlist_bytes_per_head = int64_t(n_blocks) * q_stride * sizeof(uint32_t); int H_grp_ql = static_cast((256LL * 1024LL * 1024LL) / std::max(1, qlist_bytes_per_head)); H_grp = std::max(1, std::min({BH, H_grp, std::max(1, H_grp_ql)})); int persistent_ctas = (D == 64) ? (188 * 4) : (188 * 2); int chunk_q = (chunk_q_req > 0) ? chunk_q_req : ((D == 128) ? 768 : ((n_blocks <= 80) ? 384 : 256)); int64_t need_grp_s = int64_t(H_grp) * S; int64_t need_buckets = int64_t(H_grp) * n_blocks; int64_t need_qlist = need_buckets * q_stride; int64_t need_chunks = (need_grp_s * TOP_N) / std::max(1, chunk_q) + need_buckets + 16; int64_t need_opart = need_grp_s * TOP_N * D; if (g_ws.cap_grp_s < need_grp_s) { g_ws.remote_count = torch::empty({need_grp_s}, q.options().dtype(torch::kUInt8)); g_ws.full_prev = torch::empty({need_grp_s}, q.options().dtype(torch::kUInt8)); g_ws.mls_partial = torch::empty({need_grp_s * TOP_N * 2}, q.options().dtype(torch::kInt32)); g_ws.cap_grp_s = need_grp_s; } if (g_ws.cap_buckets < need_buckets) { g_ws.bucket_count = torch::zeros({need_buckets}, q.options().dtype(torch::kInt32)); g_ws.cap_buckets = need_buckets; } if (g_ws.cap_qlist < need_qlist) { g_ws.q_list = torch::empty({need_qlist}, q.options().dtype(torch::kInt32)); g_ws.cap_qlist = need_qlist; } if (g_ws.cap_chunks < need_chunks) { g_ws.chunk_table = torch::empty({need_chunks * int64_t(sizeof(WorkChunk))}, q.options().dtype(torch::kUInt8)); g_ws.cap_chunks = need_chunks; } if (g_ws.cap_opart < need_opart) { g_ws.o_partial = torch::empty({need_opart}, q.options().dtype(torch::kInt8)); g_ws.cap_opart = need_opart; } uint8_t* rc_ptr = g_ws.remote_count.data_ptr(); uint8_t* fp_ptr = g_ws.full_prev.data_ptr(); uint32_t* bc_ptr = reinterpret_cast(g_ws.bucket_count.data_ptr()); WorkChunk* ct_ptr = reinterpret_cast(g_ws.chunk_table.data_ptr()); uint32_t* ql_ptr = reinterpret_cast(g_ws.q_list.data_ptr()); int8_t* op_ptr = g_ws.o_partial.data_ptr(); uint2* mlsp_ptr = reinterpret_cast(g_ws.mls_partial.data_ptr()); size_t smem_score = 2 * BLOCK_SIZE * D * sizeof(__nv_bfloat16); size_t smem_rem = 3 * BLOCK_SIZE * D * sizeof(__nv_bfloat16) + sizeof(WorkChunk) + 16; size_t smem_loc = 3 * BLOCK_SIZE * D * sizeof(__nv_bfloat16); for (int bh_start = 0; bh_start < BH; bh_start += H_grp) { int cur_grp = std::min(H_grp, BH - bh_start); if (n_blocks > TOP_N) { if constexpr (D == 64) { score_topk_and_scatter_64<<>>( q_ptr, kh_ptr, imp_ptr, rc_ptr, fp_ptr, bc_ptr, ql_ptr, ct_ptr, d_ctr_ptr, bh_start, S, n_blocks, q_stride, chunk_q, scale ); } else { score_topk_and_scatter_128<<>>( q_ptr, kh_ptr, imp_ptr, rc_ptr, fp_ptr, bc_ptr, ql_ptr, ct_ptr, d_ctr_ptr, bh_start, S, n_blocks, q_stride, chunk_q, scale ); } remote_block_attn_kernel<<>>( q_ptr, k_ptr, v_ptr, ql_ptr, ct_ptr, bc_ptr, d_ctr_ptr, op_ptr, mlsp_ptr, bh_start, cur_grp, S, n_blocks, q_stride, chunk_q, scale ); } if constexpr (D == 64) { local_window_and_merge_64<<>>( q_ptr, k_ptr, v_ptr, rc_ptr, fp_ptr, op_ptr, mlsp_ptr, d_ctr_ptr, o_ptr, bh_start, S, n_blocks, scale ); } else { local_window_and_merge_128<<>>( q_ptr, k_ptr, v_ptr, rc_ptr, fp_ptr, op_ptr, mlsp_ptr, d_ctr_ptr, o_ptr, bh_start, S, n_blocks, scale ); } } } torch::Tensor nsa_forward( torch::Tensor q, torch::Tensor k, torch::Tensor v, int direct_max_blocks, int h_grp_budget_mb, int chunk_q ) { TORCH_CHECK(q.is_cuda() && k.is_cuda() && v.is_cuda(), "q, k, v must be CUDA tensors"); const bool fast_ok = q.is_contiguous() && k.is_contiguous() && v.is_contiguous() && q.scalar_type() == torch::kBFloat16 && k.scalar_type() == torch::kBFloat16 && v.scalar_type() == torch::kBFloat16; torch::Tensor qc = fast_ok ? q : (q.scalar_type() == torch::kBFloat16 ? q.contiguous() : q.to(torch::kBFloat16).contiguous()); torch::Tensor kc = fast_ok ? k : (k.scalar_type() == torch::kBFloat16 ? k.contiguous() : k.to(torch::kBFloat16).contiguous()); torch::Tensor vc = fast_ok ? v : (v.scalar_type() == torch::kBFloat16 ? v.contiguous() : v.to(torch::kBFloat16).contiguous()); auto out = torch::empty_like(qc); int D = qc.size(3); if (D == 64) { dispatch_nsa_impl<64>(qc, kc, vc, out, direct_max_blocks, h_grp_budget_mb, chunk_q); } else if (D == 128) { dispatch_nsa_impl<128>(qc, kc, vc, out, direct_max_blocks, h_grp_budget_mb, chunk_q); } else { int B = qc.size(0), H = qc.size(1), S = qc.size(2); float scale = 1.0f / sqrtf(float(D)); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); nsa_fallback_kernel<<>>( reinterpret_cast(qc.data_ptr()), reinterpret_cast(kc.data_ptr()), reinterpret_cast(vc.data_ptr()), reinterpret_cast<__nv_bfloat16*>(out.data_ptr()), S, D, scale ); } return out; } """ CPP_SRC = r""" torch::Tensor nsa_forward( torch::Tensor q, torch::Tensor k, torch::Tensor v, int direct_max_blocks, int h_grp_budget_mb, int chunk_q ); """ _MOD = None def _get_mod(): global _MOD if _MOD is None: _MOD = load_inline( name="deepseek_nsa_sm120_final", cpp_sources=CPP_SRC, cuda_sources=CUDA_SRC, functions=["nsa_forward"], extra_cuda_cflags=["-O3", "-arch=sm_120a", "--expt-relaxed-constexpr"], verbose=False, ) return _MOD class Model(nn.Module): def __init__(self, B: int, H: int, S: int, D: int): super().__init__() self.B, self.H, self.S, self.D = B, H, S, D self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16)) self._mod = _get_mod() self._direct_max_blocks = int(os.environ.get("NSA_DIRECT_MAX_BLOCKS", "16")) self._h_grp_budget_mb = int(os.environ.get("NSA_H_GRP_BUDGET_MB", "72")) self._chunk_q = int(os.environ.get("NSA_CHUNK_Q", "0")) def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: return self._mod.nsa_forward( q, k, v, self._direct_max_blocks, self._h_grp_budget_mb, self._chunk_q )