from __future__ import annotations import os import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "12.0a") _CUDA_SOURCE = r""" #include #include #include #include #include #include #include #include #include namespace { constexpr int BLOCK_SIZE = 64; constexpr int TOP_N_BLOCKS = 8; constexpr int MAX_CHUNK_Q = 256; constexpr int BCOUNT_STRIDE = 32; // 32 ints = 128B cache-line isolation per bucket __device__ __forceinline__ uint32_t smem_u32(const void* ptr) { return static_cast(__cvta_generic_to_shared(ptr)); } __device__ __forceinline__ void cp_async_16B(uint32_t dst_smem, const void* src_gmem) { asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" :: "r"(dst_smem), "l"(src_gmem)); } __device__ __forceinline__ void cp_async_commit() { asm volatile("cp.async.commit_group;\n"); } template __device__ __forceinline__ void cp_async_wait() { asm volatile("cp.async.wait_group %0;\n" :: "n"(N)); } __device__ __forceinline__ bool pair_gt(float va, int ia, float vb, int ib) { return (va > vb) || (va == vb && ia > ib); } __device__ __forceinline__ void cas_pair(float& va, int& ia, float& vb, int& ib, bool desc) { bool swap = desc ? pair_gt(vb, ib, va, ia) : pair_gt(va, ia, vb, ib); if (swap) { float tv = va; va = vb; vb = tv; int ti = ia; ia = ib; ib = ti; } } __device__ __forceinline__ void bitonic_merge8_desc(float v[8], int idx[8]) { cas_pair(v[0], idx[0], v[4], idx[4], true); cas_pair(v[1], idx[1], v[5], idx[5], true); cas_pair(v[2], idx[2], v[6], idx[6], true); cas_pair(v[3], idx[3], v[7], idx[7], true); cas_pair(v[0], idx[0], v[2], idx[2], true); cas_pair(v[1], idx[1], v[3], idx[3], true); cas_pair(v[4], idx[4], v[6], idx[6], true); cas_pair(v[5], idx[5], v[7], idx[7], true); cas_pair(v[0], idx[0], v[1], idx[1], true); cas_pair(v[2], idx[2], v[3], idx[3], true); cas_pair(v[4], idx[4], v[5], idx[5], true); cas_pair(v[6], idx[6], v[7], idx[7], true); } __device__ __forceinline__ void bitonic_sort8_desc(float v[8], int idx[8]) { cas_pair(v[0], idx[0], v[1], idx[1], true); cas_pair(v[2], idx[2], v[3], idx[3], false); cas_pair(v[4], idx[4], v[5], idx[5], true); cas_pair(v[6], idx[6], v[7], idx[7], false); cas_pair(v[0], idx[0], v[2], idx[2], true); cas_pair(v[1], idx[1], v[3], idx[3], true); cas_pair(v[0], idx[0], v[1], idx[1], true); cas_pair(v[2], idx[2], v[3], idx[3], true); cas_pair(v[4], idx[4], v[6], idx[6], false); cas_pair(v[5], idx[5], v[7], idx[7], false); cas_pair(v[4], idx[4], v[5], idx[5], false); cas_pair(v[6], idx[6], v[7], idx[7], false); bitonic_merge8_desc(v, idx); } __device__ __forceinline__ void merge_into_top8(float top_v[8], int top_i[8], const float cand_v[8], const int cand_i[8]) { #pragma unroll for (int i = 0; i < 8; ++i) { float bv = cand_v[7 - i]; int bi = cand_i[7 - i]; if (pair_gt(bv, bi, top_v[i], top_i[i])) { top_v[i] = bv; top_i[i] = bi; } } bitonic_merge8_desc(top_v, top_i); } __device__ __forceinline__ void insert_one_top8(float top_v[8], int top_i[8], float cv, int ci) { #pragma unroll for (int i = 0; i < 8; ++i) { if (pair_gt(cv, ci, top_v[i], top_i[i])) { float tv = top_v[i]; top_v[i] = cv; cv = tv; int ti = top_i[i]; top_i[i] = ci; ci = ti; } } } // Merge 4 sorted-desc 8-arrays across a 4-lane quad (t_in_g = 0..3) into Global Top-16: // Upon return, t_in_g == 0 holds Top-16[0..7] and t_in_g == 1 holds Top-16[8..15] in (lv, li). __device__ __forceinline__ void quad_merge_top16(float lv[8], int li[8], int t_in_g) { bool is_even = ((t_in_g & 1) == 0); float pv[8]; int pi[8]; #pragma unroll for (int i = 0; i < 8; ++i) { pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 1, 4); pi[i] = __shfl_xor_sync(0xffffffff, li[i], 1, 4); } #pragma unroll for (int i = 0; i < 8; ++i) { float bv = pv[7 - i]; int bi = pi[7 - i]; bool b_gt_a = pair_gt(bv, bi, lv[i], li[i]); if (is_even ? b_gt_a : !b_gt_a) { lv[i] = bv; li[i] = bi; } } bitonic_merge8_desc(lv, li); #pragma unroll for (int i = 0; i < 8; ++i) { pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 3, 4); pi[i] = __shfl_xor_sync(0xffffffff, li[i], 3, 4); } #pragma unroll for (int i = 0; i < 8; ++i) { float bv = pv[7 - i]; int bi = pi[7 - i]; if (pair_gt(bv, bi, lv[i], li[i])) { lv[i] = bv; li[i] = bi; } } #pragma unroll for (int i = 0; i < 8; ++i) { pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 1, 4); pi[i] = __shfl_xor_sync(0xffffffff, li[i], 1, 4); } #pragma unroll for (int i = 0; i < 8; ++i) { bool p_gt_l = pair_gt(pv[i], pi[i], lv[i], li[i]); if (is_even ? p_gt_l : !p_gt_l) { lv[i] = pv[i]; li[i] = pi[i]; } } bitonic_merge8_desc(lv, li); } __device__ __forceinline__ void unpack_bf16x8(uint4 u, float out[8]) { const __nv_bfloat162* p = reinterpret_cast(&u); #pragma unroll for (int i = 0; i < 4; ++i) { float2 f = __bfloat1622float2(p[i]); out[i * 2 + 0] = f.x; out[i * 2 + 1] = f.y; } } __device__ __forceinline__ uint4 pack_bf16x8(const float in[8]) { uint4 u; __nv_bfloat162* p = reinterpret_cast<__nv_bfloat162*>(&u); #pragma unroll for (int i = 0; i < 4; ++i) { p[i] = __float22bfloat162_rn(make_float2(in[i * 2 + 0], in[i * 2 + 1])); } return u; } // --------------------------------------------------------------------------- // Kernel 1: High-MLP 256-thread per-block FP32 (+ BF16/err on D=128) key means // --------------------------------------------------------------------------- template __global__ __launch_bounds__(256) void k_mean_and_init_kernel( const __nv_bfloat16* __restrict__ K, float* __restrict__ K_mean_fp32, __nv_bfloat16* __restrict__ K_mean_bf16, float* __restrict__ K_mean_err, int* __restrict__ bucket_counts, uint32_t* __restrict__ g_counters, int S, int n_blocks ) { constexpr int CHUNKS = D / 8; constexpr int ROW_GROUPS = 256 / CHUNKS; constexpr int ITERS_FULL = BLOCK_SIZE / ROW_GROUPS; int bi = blockIdx.x; int bh = blockIdx.y; int tid = threadIdx.x; int warp = tid >> 5; int lane = tid & 31; if (bi == 0 && bh == 0 && tid == 0) { g_counters[0] = 0u; g_counters[1] = 0u; } if (tid == 0) { bucket_counts[(bh * n_blocks + bi) * BCOUNT_STRIDE] = 0; } int s0 = bi * BLOCK_SIZE; int valid_rows = min(BLOCK_SIZE, S - s0); int col_c = tid & (CHUNKS - 1); int row_g = tid / CHUNKS; float acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; const __nv_bfloat16* k_base = K + (static_cast(bh) * S + s0) * D; if (valid_rows == BLOCK_SIZE) { #pragma unroll for (int it = 0; it < ITERS_FULL; ++it) { int r = row_g + it * ROW_GROUPS; uint4 u = reinterpret_cast(k_base + r * D)[col_c]; float vals[8]; unpack_bf16x8(u, vals); #pragma unroll for (int j = 0; j < 8; ++j) { acc[j] += vals[j]; } } } else { for (int r = row_g; r < valid_rows; r += ROW_GROUPS) { uint4 u = reinterpret_cast(k_base + r * D)[col_c]; float vals[8]; unpack_bf16x8(u, vals); #pragma unroll for (int j = 0; j < 8; ++j) { acc[j] += vals[j]; } } } #pragma unroll for (int j = 0; j < 8; ++j) { acc[j] += __shfl_xor_sync(0xffffffff, acc[j], 16); if constexpr (CHUNKS == 8) { acc[j] += __shfl_xor_sync(0xffffffff, acc[j], 8); } } __shared__ __align__(16) float smem_red[8][D + 4]; int d0 = col_c * 8; if (lane < CHUNKS) { reinterpret_cast(&smem_red[warp][d0])[0] = make_float4(acc[0], acc[1], acc[2], acc[3]); reinterpret_cast(&smem_red[warp][d0])[1] = make_float4(acc[4], acc[5], acc[6], acc[7]); } __syncthreads(); if (warp == 0 && lane < CHUNKS) { float sum[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; #pragma unroll for (int rg = 0; rg < 8; ++rg) { float4 v0 = reinterpret_cast(&smem_red[rg][d0])[0]; float4 v1 = reinterpret_cast(&smem_red[rg][d0])[1]; sum[0] += v0.x; sum[1] += v0.y; sum[2] += v0.z; sum[3] += v0.w; sum[4] += v1.x; sum[5] += v1.y; sum[6] += v1.z; sum[7] += v1.w; } float inv_len = 1.0f / static_cast(valid_rows); float avg[8]; #pragma unroll for (int j = 0; j < 8; ++j) { avg[j] = sum[j] * inv_len; } size_t out_off = static_cast(bh * n_blocks + bi) * D + d0; float* out_f = K_mean_fp32 + out_off; reinterpret_cast(out_f)[0] = make_float4(avg[0], avg[1], avg[2], avg[3]); reinterpret_cast(out_f)[1] = make_float4(avg[4], avg[5], avg[6], avg[7]); if constexpr (D == 128) { uint4 packed_bf = pack_bf16x8(avg); reinterpret_cast(K_mean_bf16 + out_off)[0] = packed_bf; float avg_bf[8]; unpack_bf16x8(packed_bf, avg_bf); float err_sq = 0.f, avg_sq = 0.f; #pragma unroll for (int j = 0; j < 8; ++j) { float diff = avg[j] - avg_bf[j]; err_sq = fmaf(diff, diff, err_sq); avg_sq = fmaf(avg[j], avg[j], avg_sq); } constexpr uint32_t m_grp = 0x0000ffffu; err_sq += __shfl_xor_sync(m_grp, err_sq, 1); err_sq += __shfl_xor_sync(m_grp, err_sq, 2); err_sq += __shfl_xor_sync(m_grp, err_sq, 4); err_sq += __shfl_xor_sync(m_grp, err_sq, 8); avg_sq += __shfl_xor_sync(m_grp, avg_sq, 1); avg_sq += __shfl_xor_sync(m_grp, avg_sq, 2); avg_sq += __shfl_xor_sync(m_grp, avg_sq, 4); avg_sq += __shfl_xor_sync(m_grp, avg_sq, 8); if (tid == 0) { K_mean_err[bh * n_blocks + bi] = sqrtf(err_sq) * 1.01f + sqrtf(avg_sq) * 1e-5f; } } } } // --------------------------------------------------------------------------- // Kernel 2: Zero-Spill Local Attention + Adaptive SMEM / BF16 Tensor-Core // Top-16 Prefilter with Exact Cauchy-Schwarz Fast-Skip // --------------------------------------------------------------------------- template union LocalSharedBuf { uint4 kv[64 * (D / 8)]; float kmean16[D][20]; float kmean32[D][(D == 64) ? 36 : 20]; int bcount_base[512]; }; template __global__ __launch_bounds__(128, MIN_BLOCKS_PER_SM) void local_attn_score_select_kernel( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, const float* __restrict__ K_mean_fp32, const __nv_bfloat16* __restrict__ K_mean_bf16, const float* __restrict__ K_mean_err, __nv_bfloat16* __restrict__ O, float2* __restrict__ ml_all, uint8_t* __restrict__ remote_cnt, int* __restrict__ bucket_counts, uint32_t* __restrict__ bucket_items, uint2* __restrict__ g_work_queue, uint32_t* __restrict__ g_counters, int S, int n_blocks, int S_stride, float scale ) { constexpr int CHUNKS = D / 8; constexpr int K_STEPS = D / 16; constexpr int D_TILES = D / 8; constexpr int MAX_SMEM_BUCKETS = 256; constexpr int CHUNK_Q = (D == 64) ? 128 : 256; extern __shared__ __align__(128) char smem_local_raw[]; uint4* sQ = reinterpret_cast(smem_local_raw); LocalSharedBuf* sBuf = reinterpret_cast*>(sQ + 64 * CHUNKS); int* s_bcount_sep = (D == 64) ? reinterpret_cast(sBuf + 1) : nullptr; int* s_bbase_sep = (D == 64) ? (s_bcount_sep + MAX_SMEM_BUCKETS) : nullptr; int qb = (n_blocks - 1) - static_cast(blockIdx.x); int bh = blockIdx.y; int tid = threadIdx.x; int warp = tid >> 5; int lane = tid & 31; int groupID = lane >> 2; int t_in_g = lane & 3; int s0 = qb * BLOCK_SIZE; int M = min(BLOCK_SIZE, S - s0); int r0 = warp * 16 + groupID; int r1 = r0 + 8; const __nv_bfloat16* q_gmem = Q + static_cast(bh) * S * D; const __nv_bfloat16* k_gmem = K + static_cast(bh) * S * D; const __nv_bfloat16* v_gmem = V + static_cast(bh) * S * D; if constexpr (D == 64) { if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) { for (int b = tid; b < qb - 1; b += 128) { s_bcount_sep[b] = 0; } } } for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < M) { cp_async_16B(smem_u32(&sQ[r * CHUNKS + cs]), &q_gmem[(s0 + r) * D + c * 8]); if (qb == 0) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]); } } else { sQ[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); if (qb == 0) { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } } cp_async_commit(); cp_async_wait<0>(); __syncthreads(); // ----------------------------------------------------------------------- // PHASE 1: Score [0 .. qb-1] -> Only 16 regs (my_tv, my_ti) survive! // ----------------------------------------------------------------------- float my_tv[8]; int my_ti[8]; #pragma unroll for (int i = 0; i < 8; ++i) { my_tv[i] = -1e30f; my_ti[i] = -1; } if (qb >= 1) { if (qb <= 16) { // Fast Path A1 (1 <= qb <= 16): Single 16-col SMEM FP32 tile const float* km_fp32 = K_mean_fp32 + static_cast(bh * n_blocks) * D; constexpr int F4_PER_ROW = D / 4; for (int idx = tid; idx < 16 * F4_PER_ROW; idx += 128) { int br = idx / F4_PER_ROW; int dc4 = idx % F4_PER_ROW; int d = dc4 * 4; float4 fv = (br < qb) ? reinterpret_cast(km_fp32)[br * F4_PER_ROW + dc4] : make_float4(0.f, 0.f, 0.f, 0.f); sBuf->kmean16[d + 0][br] = fv.x; sBuf->kmean16[d + 1][br] = fv.y; sBuf->kmean16[d + 2][br] = fv.z; sBuf->kmean16[d + 3][br] = fv.w; } __syncthreads(); float cv0[4] = {0.f, 0.f, 0.f, 0.f}; float cv1[4] = {0.f, 0.f, 0.f, 0.f}; #pragma unroll 2 for (int dc = 0; dc < CHUNKS; ++dc) { uint4 uq0 = sQ[r0 * CHUNKS + (dc ^ (r0 & 7))]; uint4 uq1 = sQ[r1 * CHUNKS + (dc ^ (r1 & 7))]; float qv0[8], qv1[8]; unpack_bf16x8(uq0, qv0); unpack_bf16x8(uq1, qv1); #pragma unroll for (int j = 0; j < 8; ++j) { int d = dc * 8 + j; float q0v = qv0[j]; float q1v = qv1[j]; float4 km = *reinterpret_cast(&sBuf->kmean16[d][t_in_g * 4]); cv0[0] = fmaf(q0v, km.x, cv0[0]); cv1[0] = fmaf(q1v, km.x, cv1[0]); cv0[1] = fmaf(q0v, km.y, cv0[1]); cv1[1] = fmaf(q1v, km.y, cv1[1]); cv0[2] = fmaf(q0v, km.z, cv0[2]); cv1[2] = fmaf(q1v, km.z, cv1[2]); cv0[3] = fmaf(q0v, km.w, cv0[3]); cv1[3] = fmaf(q1v, km.w, cv1[3]); } } __syncthreads(); for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < M) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]); } else { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); int ci[4]; #pragma unroll for (int k = 0; k < 4; ++k) { int br = t_in_g * 4 + k; if (br < qb) { cv0[k] *= scale; cv1[k] *= scale; ci[k] = br; } else { cv0[k] = -1e30f; cv1[k] = -1e30f; ci[k] = -1; } } bool is_odd = ((t_in_g & 1) != 0); float send_v[4]; #pragma unroll for (int k = 0; k < 4; ++k) send_v[k] = is_odd ? cv0[k] : cv1[k]; #pragma unroll for (int k = 0; k < 4; ++k) { float recv_v = __shfl_xor_sync(0xffffffff, send_v[k], 1, 4); int recv_i = __shfl_xor_sync(0xffffffff, ci[k], 1, 4); my_tv[k] = is_odd ? recv_v : cv0[k]; my_ti[k] = is_odd ? recv_i : ci[k]; my_tv[4 + k] = is_odd ? cv1[k] : recv_v; my_ti[4 + k] = is_odd ? ci[k] : recv_i; } bitonic_sort8_desc(my_tv, my_ti); float other_v[8]; int other_i[8]; #pragma unroll for (int i = 0; i < 8; ++i) { other_v[i] = __shfl_xor_sync(0xffffffff, my_tv[i], 2, 4); other_i[i] = __shfl_xor_sync(0xffffffff, my_ti[i], 2, 4); } merge_into_top8(my_tv, my_ti, other_v, other_i); } else if constexpr (D == 64) { // Fast Path A2 (D == 64, qb > 16): Tiled 32-col SMEM FP32 exact scoring // Each lane t_in_g in {0..3} owns disjoint blocks (b_tile + t_in_g*8 + 0..7), // so zero warp shuffles occur inside the b_tile loop! float lv0[8], lv1[8]; int li0[8], li1[8]; #pragma unroll for (int i = 0; i < 8; ++i) { lv0[i] = -1e30f; li0[i] = -1; lv1[i] = -1e30f; li1[i] = -1; } const float* km_fp32 = K_mean_fp32 + static_cast(bh * n_blocks) * D; constexpr int F4_PER_ROW = D / 4; #pragma unroll 1 for (int b_tile = 0; b_tile < qb; b_tile += 32) { int b_cnt = min(32, qb - b_tile); for (int idx = tid; idx < 32 * F4_PER_ROW; idx += 128) { int br = idx / F4_PER_ROW; int dc4 = idx % F4_PER_ROW; int d = dc4 * 4; float4 fv = (br < b_cnt) ? reinterpret_cast(km_fp32)[(b_tile + br) * F4_PER_ROW + dc4] : make_float4(0.f, 0.f, 0.f, 0.f); sBuf->kmean32[d + 0][br] = fv.x; sBuf->kmean32[d + 1][br] = fv.y; sBuf->kmean32[d + 2][br] = fv.z; sBuf->kmean32[d + 3][br] = fv.w; } __syncthreads(); float cv0[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; float cv1[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f}; #pragma unroll 2 for (int dc = 0; dc < CHUNKS; ++dc) { uint4 uq0 = sQ[r0 * CHUNKS + (dc ^ (r0 & 7))]; uint4 uq1 = sQ[r1 * CHUNKS + (dc ^ (r1 & 7))]; float qv0[8], qv1[8]; unpack_bf16x8(uq0, qv0); unpack_bf16x8(uq1, qv1); #pragma unroll for (int j = 0; j < 8; ++j) { int d = dc * 8 + j; float q0v = qv0[j]; float q1v = qv1[j]; float4 km0 = *reinterpret_cast(&sBuf->kmean32[d][t_in_g * 8]); float4 km1 = *reinterpret_cast(&sBuf->kmean32[d][t_in_g * 8 + 4]); cv0[0] = fmaf(q0v, km0.x, cv0[0]); cv1[0] = fmaf(q1v, km0.x, cv1[0]); cv0[1] = fmaf(q0v, km0.y, cv0[1]); cv1[1] = fmaf(q1v, km0.y, cv1[1]); cv0[2] = fmaf(q0v, km0.z, cv0[2]); cv1[2] = fmaf(q1v, km0.z, cv1[2]); cv0[3] = fmaf(q0v, km0.w, cv0[3]); cv1[3] = fmaf(q1v, km0.w, cv1[3]); cv0[4] = fmaf(q0v, km1.x, cv0[4]); cv1[4] = fmaf(q1v, km1.x, cv1[4]); cv0[5] = fmaf(q0v, km1.y, cv0[5]); cv1[5] = fmaf(q1v, km1.y, cv1[5]); cv0[6] = fmaf(q0v, km1.z, cv0[6]); cv1[6] = fmaf(q1v, km1.z, cv1[6]); cv0[7] = fmaf(q0v, km1.w, cv0[7]); cv1[7] = fmaf(q1v, km1.w, cv1[7]); } } int ci0[8], ci1[8]; #pragma unroll for (int k = 0; k < 8; ++k) { int br = t_in_g * 8 + k; if (br < b_cnt) { int gbi = b_tile + br; cv0[k] *= scale; cv1[k] *= scale; ci0[k] = gbi; ci1[k] = gbi; } else { cv0[k] = -1e30f; ci0[k] = -1; cv1[k] = -1e30f; ci1[k] = -1; } } bitonic_sort8_desc(cv0, ci0); bitonic_sort8_desc(cv1, ci1); if (b_tile == 0) { #pragma unroll for (int k = 0; k < 8; ++k) { lv0[k] = cv0[k]; li0[k] = ci0[k]; lv1[k] = cv1[k]; li1[k] = ci1[k]; } } else { merge_into_top8(lv0, li0, cv0, ci0); merge_into_top8(lv1, li1, cv1, ci1); } __syncthreads(); } for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < M) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]); } else { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); // Simultaneous 2-step quad merge: lanes 0,1 merge r0 while lanes 2,3 merge r1! bool high_half = (t_in_g >= 2); float send_v[8]; int send_i[8]; #pragma unroll for (int i = 0; i < 8; ++i) { send_v[i] = high_half ? lv0[i] : lv1[i]; send_i[i] = high_half ? li0[i] : li1[i]; my_tv[i] = high_half ? lv1[i] : lv0[i]; my_ti[i] = high_half ? li1[i] : li0[i]; } float recv_v[8]; int recv_i[8]; #pragma unroll for (int i = 0; i < 8; ++i) { recv_v[i] = __shfl_xor_sync(0xffffffff, send_v[i], 2, 4); recv_i[i] = __shfl_xor_sync(0xffffffff, send_i[i], 2, 4); } merge_into_top8(my_tv, my_ti, recv_v, recv_i); #pragma unroll for (int i = 0; i < 8; ++i) { recv_v[i] = __shfl_xor_sync(0xffffffff, my_tv[i], 1, 4); recv_i[i] = __shfl_xor_sync(0xffffffff, my_ti[i], 1, 4); } merge_into_top8(my_tv, my_ti, recv_v, recv_i); #pragma unroll for (int i = 0; i < 8; ++i) { float r1_v = __shfl_xor_sync(0xffffffff, my_tv[i], 2, 4); int r1_i = __shfl_xor_sync(0xffffffff, my_ti[i], 2, 4); if (t_in_g == 1) { my_tv[i] = r1_v; my_ti[i] = r1_i; } } } else { // Fast Path B (D == 128, qb > 16): BF16 Tensor-Core Top-16 Prefilter // + Per-Candidate Exact Cauchy-Schwarz Fast-Skip Rescore float lv0[8], lv1[8]; int li0[8], li1[8]; #pragma unroll for (int i = 0; i < 8; ++i) { lv0[i] = -1e30f; li0[i] = -1; lv1[i] = -1e30f; li1[i] = -1; } const __nv_bfloat16* km_bf16 = K_mean_bf16 + static_cast(bh * n_blocks) * D; #pragma unroll 1 for (int b_tile = 0; b_tile < qb; b_tile += 64) { int b_cnt = min(64, qb - b_tile); for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < b_cnt) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &km_bf16[(b_tile + r) * D + c * 8]); } else { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); cp_async_wait<0>(); __syncthreads(); float km_acc[8][4]; #pragma unroll for (int nt = 0; nt < 8; ++nt) { km_acc[nt][0] = km_acc[nt][1] = km_acc[nt][2] = km_acc[nt][3] = 0.f; } #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { uint32_t q0, q1, q2, q3; { int rq = warp * 16 + (lane & 15); int cq = ks * 2 + (lane >> 4); uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]); asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3) : "r"(q_addr)); } #pragma unroll for (int np = 0; np < 4; ++np) { int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7); int c = ks * 2 + ((lane >> 3) & 1); int cs = c ^ (n_row & 7); uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]); uint32_t b00, b01, b10, b11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(km_acc[np * 2][0]), "=f"(km_acc[np * 2][1]), "=f"(km_acc[np * 2][2]), "=f"(km_acc[np * 2][3]) : "r"(q0), "r"(q1), "r"(q2), "r"(q3), "r"(b00), "r"(b01), "f"(km_acc[np * 2][0]), "f"(km_acc[np * 2][1]), "f"(km_acc[np * 2][2]), "f"(km_acc[np * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(km_acc[np * 2 + 1][0]), "=f"(km_acc[np * 2 + 1][1]), "=f"(km_acc[np * 2 + 1][2]), "=f"(km_acc[np * 2 + 1][3]) : "r"(q0), "r"(q1), "r"(q2), "r"(q3), "r"(b10), "r"(b11), "f"(km_acc[np * 2 + 1][0]), "f"(km_acc[np * 2 + 1][1]), "f"(km_acc[np * 2 + 1][2]), "f"(km_acc[np * 2 + 1][3])); } } #pragma unroll for (int half = 0; half < 2; ++half) { float hv[8]; int hi[8]; #pragma unroll for (int nt = 0; nt < 4; ++nt) { int col0 = (half * 4 + nt) * 8 + t_in_g * 2; int col1 = col0 + 1; hv[nt * 2 + 0] = (col0 < b_cnt) ? km_acc[half * 4 + nt][0] : -1e30f; hi[nt * 2 + 0] = (col0 < b_cnt) ? (b_tile + col0) : -1; hv[nt * 2 + 1] = (col1 < b_cnt) ? km_acc[half * 4 + nt][1] : -1e30f; hi[nt * 2 + 1] = (col1 < b_cnt) ? (b_tile + col1) : -1; } bitonic_sort8_desc(hv, hi); merge_into_top8(lv0, li0, hv, hi); #pragma unroll for (int nt = 0; nt < 4; ++nt) { int col0 = (half * 4 + nt) * 8 + t_in_g * 2; int col1 = col0 + 1; hv[nt * 2 + 0] = (col0 < b_cnt) ? km_acc[half * 4 + nt][2] : -1e30f; hi[nt * 2 + 0] = (col0 < b_cnt) ? (b_tile + col0) : -1; hv[nt * 2 + 1] = (col1 < b_cnt) ? km_acc[half * 4 + nt][3] : -1e30f; hi[nt * 2 + 1] = (col1 < b_cnt) ? (b_tile + col1) : -1; } bitonic_sort8_desc(hv, hi); merge_into_top8(lv1, li1, hv, hi); } __syncthreads(); } // Immediately prefetch K_qb -> sBuf->kv while running quad_merge_top16 + rescore! for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < M) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]); } else { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); quad_merge_top16(lv0, li0, t_in_g); quad_merge_top16(lv1, li1, t_in_g); constexpr int F16_STEPS = D / 16; uint2 q0_u2[F16_STEPS]; uint2 q1_u2[F16_STEPS]; float q0_sq = 0.f, q1_sq = 0.f; #pragma unroll for (int s = 0; s < F16_STEPS; ++s) { int c = s * 2 + (t_in_g >> 1); int sub = t_in_g & 1; uint2 u0 = reinterpret_cast(&sQ[r0 * CHUNKS + (c ^ (r0 & 7))])[sub]; uint2 u1 = reinterpret_cast(&sQ[r1 * CHUNKS + (c ^ (r1 & 7))])[sub]; q0_u2[s] = u0; q1_u2[s] = u1; float2 f0a = __bfloat1622float2(*reinterpret_cast(&u0.x)); float2 f0b = __bfloat1622float2(*reinterpret_cast(&u0.y)); q0_sq = fmaf(f0a.x, f0a.x, fmaf(f0a.y, f0a.y, fmaf(f0b.x, f0b.x, fmaf(f0b.y, f0b.y, q0_sq)))); float2 f1a = __bfloat1622float2(*reinterpret_cast(&u1.x)); float2 f1b = __bfloat1622float2(*reinterpret_cast(&u1.y)); q1_sq = fmaf(f1a.x, f1a.x, fmaf(f1a.y, f1a.y, fmaf(f1b.x, f1b.x, fmaf(f1b.y, f1b.y, q1_sq)))); } q0_sq += __shfl_xor_sync(0xffffffff, q0_sq, 1); q0_sq += __shfl_xor_sync(0xffffffff, q0_sq, 2); q1_sq += __shfl_xor_sync(0xffffffff, q1_sq, 1); q1_sq += __shfl_xor_sync(0xffffffff, q1_sq, 2); float max_k_err = 0.f; const float* k_err_base = K_mean_err + bh * n_blocks; for (int b = lane; b < qb; b += 32) { max_k_err = fmaxf(max_k_err, __ldg(&k_err_base[b])); } #pragma unroll for (int mask = 16; mask >= 1; mask >>= 1) { max_k_err = fmaxf(max_k_err, __shfl_xor_sync(0xffffffff, max_k_err, mask)); } float two_delta0 = 2.0f * sqrtf(q0_sq) * max_k_err; float two_delta1 = 2.0f * sqrtf(q1_sq) * max_k_err; float v7_0 = __shfl_sync(0xffffffff, lv0[7], 0, 4); float v7_1 = __shfl_sync(0xffffffff, lv1[7], 0, 4); uint32_t quad_mask = 0xfu << (groupID * 4); const float* km_fp32 = K_mean_fp32 + static_cast(bh * n_blocks) * D; #pragma unroll 1 for (int step = 0; step < 16; ++step) { int src_lane = (step < 8) ? 0 : 1; int b0 = __shfl_sync(0xffffffff, li0[0], src_lane, 4); float hat0 = __shfl_sync(0xffffffff, lv0[0], src_lane, 4); int b1 = __shfl_sync(0xffffffff, li1[0], src_lane, 4); float hat1 = __shfl_sync(0xffffffff, lv1[0], src_lane, 4); if ((t_in_g == 0 && step < 8) || (t_in_g == 1 && step >= 8)) { #pragma unroll for (int j = 0; j < 7; ++j) { li0[j] = li0[j + 1]; lv0[j] = lv0[j + 1]; li1[j] = li1[j + 1]; lv1[j] = lv1[j + 1]; } } bool sure_out0 = (b0 < 0) || (step >= 8 && (v7_0 - hat0 > two_delta0)); bool sure_in0 = (!sure_out0) && (step < 7 && (hat0 - v7_0 > two_delta0)); bool need0 = (!sure_out0) && (!sure_in0); bool sure_out1 = (b1 < 0) || (step >= 8 && (v7_1 - hat1 > two_delta1)); bool sure_in1 = (!sure_out1) && (step < 7 && (hat1 - v7_1 > two_delta1)); bool need1 = (!sure_out1) && (!sure_in1); float score0 = hat0 * scale; if (need0) { float dot0 = 0.f; const float4* km0 = reinterpret_cast(km_fp32 + static_cast(b0) * D); #pragma unroll for (int s = 0; s < F16_STEPS; ++s) { float4 kv = km0[s * 4 + t_in_g]; float2 q_lo = __bfloat1622float2(*reinterpret_cast(&q0_u2[s].x)); float2 q_hi = __bfloat1622float2(*reinterpret_cast(&q0_u2[s].y)); dot0 = fmaf(q_lo.x, kv.x, dot0); dot0 = fmaf(q_lo.y, kv.y, dot0); dot0 = fmaf(q_hi.x, kv.z, dot0); dot0 = fmaf(q_hi.y, kv.w, dot0); } dot0 += __shfl_xor_sync(quad_mask, dot0, 1); dot0 += __shfl_xor_sync(quad_mask, dot0, 2); score0 = dot0 * scale; } float score1 = hat1 * scale; if (need1) { float dot1 = 0.f; const float4* km1 = reinterpret_cast(km_fp32 + static_cast(b1) * D); #pragma unroll for (int s = 0; s < F16_STEPS; ++s) { float4 kv = km1[s * 4 + t_in_g]; float2 q_lo = __bfloat1622float2(*reinterpret_cast(&q1_u2[s].x)); float2 q_hi = __bfloat1622float2(*reinterpret_cast(&q1_u2[s].y)); dot1 = fmaf(q_lo.x, kv.x, dot1); dot1 = fmaf(q_lo.y, kv.y, dot1); dot1 = fmaf(q_hi.x, kv.z, dot1); dot1 = fmaf(q_hi.y, kv.w, dot1); } dot1 += __shfl_xor_sync(quad_mask, dot1, 1); dot1 += __shfl_xor_sync(quad_mask, dot1, 2); score1 = dot1 * scale; } bool my_keep = (t_in_g == 0) ? (!sure_out0) : (!sure_out1); if (t_in_g < 2 && my_keep) { int my_b = (t_in_g == 0) ? b0 : b1; float my_sc = (t_in_g == 0) ? score0 : score1; insert_one_top8(my_tv, my_ti, my_sc, my_b); } } } cp_async_wait<0>(); __syncthreads(); } // ----------------------------------------------------------------------- // PHASE 2: Diagonal Attention [qb] + imp_diag Top-8 Insert + Bucket Emit // (Load q0..q3 per ks from sQ so q_frag[K_STEPS][4] is NOT held live!) // ----------------------------------------------------------------------- float s_acc[8][4]; #pragma unroll for (int nt = 0; nt < 8; ++nt) { s_acc[nt][0] = s_acc[nt][1] = s_acc[nt][2] = s_acc[nt][3] = 0.f; } #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { uint32_t q0, q1, q2, q3; { int rq = warp * 16 + (lane & 15); int cq = ks * 2 + (lane >> 4); uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]); asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3) : "r"(q_addr)); } #pragma unroll for (int np = 0; np < 4; ++np) { int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7); int c = ks * 2 + ((lane >> 3) & 1); int cs = c ^ (n_row & 7); uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]); uint32_t b00, b01, b10, b11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(s_acc[np * 2][0]), "=f"(s_acc[np * 2][1]), "=f"(s_acc[np * 2][2]), "=f"(s_acc[np * 2][3]) : "r"(q0), "r"(q1), "r"(q2), "r"(q3), "r"(b00), "r"(b01), "f"(s_acc[np * 2][0]), "f"(s_acc[np * 2][1]), "f"(s_acc[np * 2][2]), "f"(s_acc[np * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(s_acc[np * 2 + 1][0]), "=f"(s_acc[np * 2 + 1][1]), "=f"(s_acc[np * 2 + 1][2]), "=f"(s_acc[np * 2 + 1][3]) : "r"(q0), "r"(q1), "r"(q2), "r"(q3), "r"(b10), "r"(b11), "f"(s_acc[np * 2 + 1][0]), "f"(s_acc[np * 2 + 1][1]), "f"(s_acc[np * 2 + 1][2]), "f"(s_acc[np * 2 + 1][3])); } } __syncthreads(); if constexpr (D == 64) { for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < M) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(s0 + r) * D + c * 8]); } else { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); } else { if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) { for (int b = tid; b < qb - 1; b += 128) { sBuf->bcount_base[b] = 0; } } } float diag_sum0 = 0.f, diag_sum1 = 0.f; float m0 = -1e30f, m1 = -1e30f; #pragma unroll for (int nt = 0; nt < 8; ++nt) { int col0 = nt * 8 + t_in_g * 2; int col1 = col0 + 1; float v00 = s_acc[nt][0] * scale; float v01 = s_acc[nt][1] * scale; float v10 = s_acc[nt][2] * scale; float v11 = s_acc[nt][3] * scale; if (col0 <= r0) { diag_sum0 += v00; m0 = fmaxf(m0, v00); s_acc[nt][0] = v00; } else { s_acc[nt][0] = -1e30f; } if (col1 <= r0) { diag_sum0 += v01; m0 = fmaxf(m0, v01); s_acc[nt][1] = v01; } else { s_acc[nt][1] = -1e30f; } if (col0 <= r1) { diag_sum1 += v10; m1 = fmaxf(m1, v10); s_acc[nt][2] = v10; } else { s_acc[nt][2] = -1e30f; } if (col1 <= r1) { diag_sum1 += v11; m1 = fmaxf(m1, v11); s_acc[nt][3] = v11; } else { s_acc[nt][3] = -1e30f; } } diag_sum0 += __shfl_xor_sync(0xffffffff, diag_sum0, 1); diag_sum0 += __shfl_xor_sync(0xffffffff, diag_sum0, 2); diag_sum1 += __shfl_xor_sync(0xffffffff, diag_sum1, 1); diag_sum1 += __shfl_xor_sync(0xffffffff, diag_sum1, 2); float imp_diag0 = diag_sum0 / static_cast(r0 + 1); float imp_diag1 = diag_sum1 / static_cast(r1 + 1); m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 1)); m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 2)); m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 1)); m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 2)); float l0 = 0.f, l1 = 0.f; #pragma unroll for (int nt = 0; nt < 8; ++nt) { s_acc[nt][0] = __expf(s_acc[nt][0] - m0); s_acc[nt][1] = __expf(s_acc[nt][1] - m0); s_acc[nt][2] = __expf(s_acc[nt][2] - m1); s_acc[nt][3] = __expf(s_acc[nt][3] - m1); l0 += s_acc[nt][0] + s_acc[nt][1]; l1 += s_acc[nt][2] + s_acc[nt][3]; } l0 += __shfl_xor_sync(0xffffffff, l0, 1); l0 += __shfl_xor_sync(0xffffffff, l0, 2); l1 += __shfl_xor_sync(0xffffffff, l1, 1); l1 += __shfl_xor_sync(0xffffffff, l1, 2); uint32_t p_frag[4][4]; #pragma unroll for (int ks = 0; ks < 4; ++ks) { __nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][0], s_acc[ks * 2][1])); __nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][2], s_acc[ks * 2][3])); __nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][0], s_acc[ks * 2 + 1][1])); __nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][2], s_acc[ks * 2 + 1][3])); p_frag[ks][0] = *reinterpret_cast(&v0); p_frag[ks][1] = *reinterpret_cast(&v1); p_frag[ks][2] = *reinterpret_cast(&v2); p_frag[ks][3] = *reinterpret_cast(&v3); } bool prev_in_top0 = false, prev_in_top1 = false; int cnt0 = 0, cnt1 = 0; if (qb >= 1) { if (t_in_g < 2) { insert_one_top8(my_tv, my_ti, (t_in_g == 0) ? imp_diag0 : imp_diag1, qb); } bool my_prev = false; int my_cnt = 0; #pragma unroll for (int i = 0; i < 8; ++i) { if (my_ti[i] == qb - 1) my_prev = true; if (my_ti[i] >= 0 && my_ti[i] < qb - 1) my_cnt++; } prev_in_top0 = __shfl_sync(0xffffffff, my_prev, 0, 4); prev_in_top1 = __shfl_sync(0xffffffff, my_prev, 1, 4); cnt0 = __shfl_sync(0xffffffff, my_cnt, 0, 4); cnt1 = __shfl_sync(0xffffffff, my_cnt, 1, 4); if (qb >= 2) { int my_r = (t_in_g == 0) ? r0 : r1; int my_lpos[8] = {0, 0, 0, 0, 0, 0, 0, 0}; if (qb - 1 <= MAX_SMEM_BUCKETS) { int* s_bc = (D == 64) ? s_bcount_sep : sBuf->bcount_base; int* s_bb = (D == 64) ? s_bbase_sep : (sBuf->bcount_base + MAX_SMEM_BUCKETS); if constexpr (D != 64) { __syncthreads(); } if (t_in_g < 2 && my_r < M) { #pragma unroll for (int i = 0; i < 8; ++i) { int bi = my_ti[i]; if (bi >= 0 && bi < qb - 1) { my_lpos[i] = atomicAdd(&s_bc[bi], 1); } } } __syncthreads(); for (int b = tid; b < qb - 1; b += 128) { int c_blk = s_bc[b]; if (c_blk > 0) { int bucket_id = bh * n_blocks + b; int gbase = atomicAdd(&bucket_counts[bucket_id * BCOUNT_STRIDE], c_blk); s_bb[b] = gbase; int first_chunk = (gbase + CHUNK_Q - 1) & ~(CHUNK_Q - 1); for (int p = first_chunk; p < gbase + c_blk; p += CHUNK_Q) { uint32_t wid = atomicAdd(&g_counters[0], 1u); g_work_queue[wid] = make_uint2(static_cast(bucket_id), static_cast(p)); } } } __syncthreads(); if (t_in_g < 2 && my_r < M) { int slot = 0; uint32_t t_global = static_cast(s0 + my_r); #pragma unroll for (int i = 0; i < 8; ++i) { int bi = my_ti[i]; if (bi >= 0 && bi < qb - 1) { int bucket_id = bh * n_blocks + bi; int pos = s_bb[bi] + my_lpos[i]; bucket_items[static_cast(bucket_id) * S_stride + pos] = (t_global << 4) | static_cast(slot); slot++; } } } } else { constexpr int chunk_mask = CHUNK_Q - 1; if (t_in_g < 2 && my_r < M) { int slot = 0; uint32_t t_global = static_cast(s0 + my_r); #pragma unroll for (int i = 0; i < 8; ++i) { int bi = my_ti[i]; if (bi >= 0 && bi < qb - 1) { int bucket_id = bh * n_blocks + bi; int pos = atomicAdd(&bucket_counts[bucket_id * BCOUNT_STRIDE], 1); bucket_items[static_cast(bucket_id) * S_stride + pos] = (t_global << 4) | static_cast(slot); if ((pos & chunk_mask) == 0) { uint32_t wid = atomicAdd(&g_counters[0], 1u); g_work_queue[wid] = make_uint2(static_cast(bucket_id), static_cast(pos)); } slot++; } } } } } } if constexpr (D == 64) { cp_async_wait<0>(); __syncthreads(); } else { if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) { __syncthreads(); } for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); if (r < M) { cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(s0 + r) * D + c * 8]); } else { sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); cp_async_wait<0>(); __syncthreads(); } // ----------------------------------------------------------------------- // PHASE 3: P_curr @ V_qb + [qb - 1] Attention + Bank-Conflict-Free Store // ----------------------------------------------------------------------- float o_acc[D_TILES][4]; #pragma unroll for (int dt = 0; dt < D_TILES; ++dt) { o_acc[dt][0] = o_acc[dt][1] = o_acc[dt][2] = o_acc[dt][3] = 0.f; } #pragma unroll for (int ks = 0; ks < 4; ++ks) { #pragma unroll for (int dp = 0; dp < D / 16; ++dp) { int r = ks * 16 + (lane & 15); int c = dp * 2 + (lane >> 4); int cs = c ^ (r & 7); uint32_t addr = smem_u32(&sBuf->kv[r * CHUNKS + cs]); uint32_t v00, v01, v10, v11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3]) : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]), "r"(v00), "r"(v01), "f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3]) : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]), "r"(v10), "r"(v11), "f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3])); } } if (qb >= 1) { __syncthreads(); int sp0 = (qb - 1) * BLOCK_SIZE; for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(sp0 + r) * D + c * 8]); } cp_async_commit(); cp_async_wait<0>(); __syncthreads(); float sp_acc[8][4]; #pragma unroll for (int nt = 0; nt < 8; ++nt) { sp_acc[nt][0] = sp_acc[nt][1] = sp_acc[nt][2] = sp_acc[nt][3] = 0.f; } #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { uint32_t q0, q1, q2, q3; { int rq = warp * 16 + (lane & 15); int cq = ks * 2 + (lane >> 4); uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]); asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3) : "r"(q_addr)); } #pragma unroll for (int np = 0; np < 4; ++np) { int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7); int c = ks * 2 + ((lane >> 3) & 1); int cs = c ^ (n_row & 7); uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]); uint32_t b00, b01, b10, b11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(sp_acc[np * 2][0]), "=f"(sp_acc[np * 2][1]), "=f"(sp_acc[np * 2][2]), "=f"(sp_acc[np * 2][3]) : "r"(q0), "r"(q1), "r"(q2), "r"(q3), "r"(b00), "r"(b01), "f"(sp_acc[np * 2][0]), "f"(sp_acc[np * 2][1]), "f"(sp_acc[np * 2][2]), "f"(sp_acc[np * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(sp_acc[np * 2 + 1][0]), "=f"(sp_acc[np * 2 + 1][1]), "=f"(sp_acc[np * 2 + 1][2]), "=f"(sp_acc[np * 2 + 1][3]) : "r"(q0), "r"(q1), "r"(q2), "r"(q3), "r"(b10), "r"(b11), "f"(sp_acc[np * 2 + 1][0]), "f"(sp_acc[np * 2 + 1][1]), "f"(sp_acc[np * 2 + 1][2]), "f"(sp_acc[np * 2 + 1][3])); } } __syncthreads(); for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(sp0 + r) * D + c * 8]); } cp_async_commit(); float mp0 = m0, mp1 = m1; #pragma unroll for (int nt = 0; nt < 8; ++nt) { int col0 = nt * 8 + t_in_g * 2; int col1 = col0 + 1; float v00 = (prev_in_top0 || col0 > r0) ? (sp_acc[nt][0] * scale) : -1e30f; float v01 = (prev_in_top0 || col1 > r0) ? (sp_acc[nt][1] * scale) : -1e30f; float v10 = (prev_in_top1 || col0 > r1) ? (sp_acc[nt][2] * scale) : -1e30f; float v11 = (prev_in_top1 || col1 > r1) ? (sp_acc[nt][3] * scale) : -1e30f; sp_acc[nt][0] = v00; sp_acc[nt][1] = v01; sp_acc[nt][2] = v10; sp_acc[nt][3] = v11; mp0 = fmaxf(mp0, fmaxf(v00, v01)); mp1 = fmaxf(mp1, fmaxf(v10, v11)); } mp0 = fmaxf(mp0, __shfl_xor_sync(0xffffffff, mp0, 1)); mp0 = fmaxf(mp0, __shfl_xor_sync(0xffffffff, mp0, 2)); mp1 = fmaxf(mp1, __shfl_xor_sync(0xffffffff, mp1, 1)); mp1 = fmaxf(mp1, __shfl_xor_sync(0xffffffff, mp1, 2)); float alpha0 = __expf(m0 - mp0); float alpha1 = __expf(m1 - mp1); m0 = mp0; m1 = mp1; #pragma unroll for (int dt = 0; dt < D_TILES; ++dt) { o_acc[dt][0] *= alpha0; o_acc[dt][1] *= alpha0; o_acc[dt][2] *= alpha1; o_acc[dt][3] *= alpha1; } float lp0 = 0.f, lp1 = 0.f; #pragma unroll for (int nt = 0; nt < 8; ++nt) { sp_acc[nt][0] = __expf(sp_acc[nt][0] - m0); sp_acc[nt][1] = __expf(sp_acc[nt][1] - m0); sp_acc[nt][2] = __expf(sp_acc[nt][2] - m1); sp_acc[nt][3] = __expf(sp_acc[nt][3] - m1); lp0 += sp_acc[nt][0] + sp_acc[nt][1]; lp1 += sp_acc[nt][2] + sp_acc[nt][3]; } lp0 += __shfl_xor_sync(0xffffffff, lp0, 1); lp0 += __shfl_xor_sync(0xffffffff, lp0, 2); lp1 += __shfl_xor_sync(0xffffffff, lp1, 1); lp1 += __shfl_xor_sync(0xffffffff, lp1, 2); l0 = l0 * alpha0 + lp0; l1 = l1 * alpha1 + lp1; #pragma unroll for (int ks = 0; ks < 4; ++ks) { __nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2][0], sp_acc[ks * 2][1])); __nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2][2], sp_acc[ks * 2][3])); __nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2 + 1][0], sp_acc[ks * 2 + 1][1])); __nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2 + 1][2], sp_acc[ks * 2 + 1][3])); p_frag[ks][0] = *reinterpret_cast(&v0); p_frag[ks][1] = *reinterpret_cast(&v1); p_frag[ks][2] = *reinterpret_cast(&v2); p_frag[ks][3] = *reinterpret_cast(&v3); } cp_async_wait<0>(); __syncthreads(); #pragma unroll for (int ks = 0; ks < 4; ++ks) { #pragma unroll for (int dp = 0; dp < D / 16; ++dp) { int r = ks * 16 + (lane & 15); int c = dp * 2 + (lane >> 4); int cs = c ^ (r & 7); uint32_t addr = smem_u32(&sBuf->kv[r * CHUNKS + cs]); uint32_t v00, v01, v10, v11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3]) : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]), "r"(v00), "r"(v01), "f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3]) : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]), "r"(v10), "r"(v11), "f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3])); } } } float inv_l0 = 1.0f / l0; float inv_l1 = 1.0f / l1; __syncthreads(); #pragma unroll for (int dt = 0; dt < D_TILES; ++dt) { int dts0 = dt ^ (r0 & 7); int dts1 = dt ^ (r1 & 7); reinterpret_cast<__nv_bfloat162*>(&sQ[r0 * CHUNKS + dts0])[t_in_g] = __float22bfloat162_rn(make_float2(o_acc[dt][0] * inv_l0, o_acc[dt][1] * inv_l0)); reinterpret_cast<__nv_bfloat162*>(&sQ[r1 * CHUNKS + dts1])[t_in_g] = __float22bfloat162_rn(make_float2(o_acc[dt][2] * inv_l1, o_acc[dt][3] * inv_l1)); } if (t_in_g == 0) { if (r0 < M) { size_t gt0 = static_cast(bh) * S + s0 + r0; remote_cnt[gt0] = static_cast(cnt0); if (cnt0 > 0) { ml_all[gt0 * 16 + 0] = make_float2(m0, l0); } } if (r1 < M) { size_t gt1 = static_cast(bh) * S + s0 + r1; remote_cnt[gt1] = static_cast(cnt1); if (cnt1 > 0) { ml_all[gt1 * 16 + 0] = make_float2(m1, l1); } } } __syncthreads(); __nv_bfloat16* o_blk = O + (static_cast(bh) * S + s0) * D; for (int i = tid; i < M * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; reinterpret_cast(o_blk)[i] = sQ[r * CHUNKS + (c ^ (r & 7))]; } } // --------------------------------------------------------------------------- // Kernel 3: Warp-Asynchronous High-Occupancy Persistent Remote Attention // --------------------------------------------------------------------------- template __global__ __launch_bounds__(128, (D == 64 ? 4 : 2)) void remote_kv_attn_kernel( const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V, __nv_bfloat16* __restrict__ partial_O, float2* __restrict__ ml_all, const int* __restrict__ bucket_counts, const uint32_t* __restrict__ bucket_items, const uint2* __restrict__ g_work_queue, uint32_t* __restrict__ g_counters, int S, int n_blocks, int S_stride, float scale ) { constexpr int CHUNKS = D / 8; constexpr int K_STEPS = D / 16; constexpr int D_TILES = D / 8; constexpr int CHUNK_Q = (D == 64) ? 128 : 256; extern __shared__ __align__(128) char smem_raw[]; uint4* sK = reinterpret_cast(smem_raw); uint4* sV = sK + 64 * CHUNKS; uint4* sQ = sV + 64 * CHUNKS; int tid = threadIdx.x; int warp = tid >> 5; int lane = tid & 31; int groupID = lane >> 2; int t_in_g = lane & 3; uint32_t total_w = __ldg(&g_counters[0]); if (blockIdx.x >= total_w) return; bool first_iter = true; while (true) { uint32_t* s_meta = reinterpret_cast(sQ); if (first_iter) { first_iter = false; if (tid == 0) { uint2 w = __ldg(&g_work_queue[blockIdx.x]); s_meta[1] = w.x; s_meta[2] = w.y; s_meta[3] = static_cast(__ldg(&bucket_counts[w.x * BCOUNT_STRIDE])); } __syncthreads(); } else { if (tid == 0) { uint32_t wid = gridDim.x + atomicAdd(&g_counters[1], 1u); if (wid < total_w) { s_meta[0] = wid; uint2 w = __ldg(&g_work_queue[wid]); s_meta[1] = w.x; s_meta[2] = w.y; s_meta[3] = static_cast(__ldg(&bucket_counts[w.x * BCOUNT_STRIDE])); } else { s_meta[0] = 0xffffffffu; } } __syncthreads(); if (s_meta[0] == 0xffffffffu) break; } int bucket_id = static_cast(s_meta[1]); int chunk_start = static_cast(s_meta[2]); int s_bucket_total = static_cast(s_meta[3]); int chunk_len = min(CHUNK_Q, s_bucket_total - chunk_start); int bh = bucket_id / n_blocks; int bi = bucket_id % n_blocks; int k_s0 = bi * BLOCK_SIZE; const __nv_bfloat16* q_head = Q + static_cast(bh) * S * D; const __nv_bfloat16* k_blk = K + (static_cast(bh) * S + k_s0) * D; const __nv_bfloat16* v_blk = V + (static_cast(bh) * S + k_s0) * D; const uint32_t* b_items = bucket_items + static_cast(bucket_id) * S_stride + chunk_start; int r_load = lane & 15; int q_base0 = warp * 16; uint32_t my_pk16 = (q_base0 + r_load < chunk_len) ? __ldg(&b_items[q_base0 + r_load]) : 0u; for (int i = tid; i < 64 * CHUNKS; i += 128) { int r = i / CHUNKS; int c = i % CHUNKS; int cs = c ^ (r & 7); cp_async_16B(smem_u32(&sK[r * CHUNKS + cs]), &k_blk[r * D + c * 8]); cp_async_16B(smem_u32(&sV[r * CHUNKS + cs]), &v_blk[r * D + c * 8]); } cp_async_commit(); cp_async_wait<0>(); __syncthreads(); uint4* sQ_w = sQ + warp * (16 * CHUNKS); for (int q_base = q_base0; q_base < chunk_len; q_base += 64) { int w_m = min(16, chunk_len - q_base); if (q_base > q_base0) { my_pk16 = (r_load < w_m) ? __ldg(&b_items[q_base + r_load]) : 0u; } #pragma unroll for (int it = 0; it < (16 * CHUNKS) / 32; ++it) { int idx = it * 32 + lane; int r = idx / CHUNKS; int c = idx % CHUNKS; int cs = c ^ (r & 7); uint32_t pk_r = __shfl_sync(0xffffffff, my_pk16, r); if (r < w_m) { uint32_t t_idx = pk_r >> 4; cp_async_16B(smem_u32(&sQ_w[r * CHUNKS + cs]), &q_head[t_idx * D + c * 8]); } else { sQ_w[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0); } } cp_async_commit(); cp_async_wait<0>(); __syncwarp(); uint32_t q_frag[K_STEPS][4]; #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { int r = lane & 15; int c = ks * 2 + (lane >> 4); int cs = c ^ (r & 7); uint32_t addr = smem_u32(&sQ_w[r * CHUNKS + cs]); asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(q_frag[ks][0]), "=r"(q_frag[ks][1]), "=r"(q_frag[ks][2]), "=r"(q_frag[ks][3]) : "r"(addr)); } float s_acc[8][4]; #pragma unroll for (int nt = 0; nt < 8; ++nt) { s_acc[nt][0] = s_acc[nt][1] = s_acc[nt][2] = s_acc[nt][3] = 0.f; } #pragma unroll for (int ks = 0; ks < K_STEPS; ++ks) { #pragma unroll for (int np = 0; np < 4; ++np) { int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7); int c = ks * 2 + ((lane >> 3) & 1); int cs = c ^ (n_row & 7); uint32_t addr = smem_u32(&sK[n_row * CHUNKS + cs]); uint32_t b00, b01, b10, b11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(s_acc[np * 2][0]), "=f"(s_acc[np * 2][1]), "=f"(s_acc[np * 2][2]), "=f"(s_acc[np * 2][3]) : "r"(q_frag[ks][0]), "r"(q_frag[ks][1]), "r"(q_frag[ks][2]), "r"(q_frag[ks][3]), "r"(b00), "r"(b01), "f"(s_acc[np * 2][0]), "f"(s_acc[np * 2][1]), "f"(s_acc[np * 2][2]), "f"(s_acc[np * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(s_acc[np * 2 + 1][0]), "=f"(s_acc[np * 2 + 1][1]), "=f"(s_acc[np * 2 + 1][2]), "=f"(s_acc[np * 2 + 1][3]) : "r"(q_frag[ks][0]), "r"(q_frag[ks][1]), "r"(q_frag[ks][2]), "r"(q_frag[ks][3]), "r"(b10), "r"(b11), "f"(s_acc[np * 2 + 1][0]), "f"(s_acc[np * 2 + 1][1]), "f"(s_acc[np * 2 + 1][2]), "f"(s_acc[np * 2 + 1][3])); } } float m0 = -1e30f, m1 = -1e30f; #pragma unroll for (int nt = 0; nt < 8; ++nt) { s_acc[nt][0] *= scale; s_acc[nt][1] *= scale; s_acc[nt][2] *= scale; s_acc[nt][3] *= scale; m0 = fmaxf(m0, fmaxf(s_acc[nt][0], s_acc[nt][1])); m1 = fmaxf(m1, fmaxf(s_acc[nt][2], s_acc[nt][3])); } m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 1)); m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 2)); m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 1)); m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 2)); float l0 = 0.f, l1 = 0.f; #pragma unroll for (int nt = 0; nt < 8; ++nt) { s_acc[nt][0] = __expf(s_acc[nt][0] - m0); s_acc[nt][1] = __expf(s_acc[nt][1] - m0); s_acc[nt][2] = __expf(s_acc[nt][2] - m1); s_acc[nt][3] = __expf(s_acc[nt][3] - m1); l0 += s_acc[nt][0] + s_acc[nt][1]; l1 += s_acc[nt][2] + s_acc[nt][3]; } l0 += __shfl_xor_sync(0xffffffff, l0, 1); l0 += __shfl_xor_sync(0xffffffff, l0, 2); l1 += __shfl_xor_sync(0xffffffff, l1, 1); l1 += __shfl_xor_sync(0xffffffff, l1, 2); uint32_t p_frag[4][4]; #pragma unroll for (int ks = 0; ks < 4; ++ks) { __nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][0], s_acc[ks * 2][1])); __nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][2], s_acc[ks * 2][3])); __nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][0], s_acc[ks * 2 + 1][1])); __nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][2], s_acc[ks * 2 + 1][3])); p_frag[ks][0] = *reinterpret_cast(&v0); p_frag[ks][1] = *reinterpret_cast(&v1); p_frag[ks][2] = *reinterpret_cast(&v2); p_frag[ks][3] = *reinterpret_cast(&v3); } float o_acc[D_TILES][4]; #pragma unroll for (int dt = 0; dt < D_TILES; ++dt) { o_acc[dt][0] = o_acc[dt][1] = o_acc[dt][2] = o_acc[dt][3] = 0.f; } #pragma unroll for (int ks = 0; ks < 4; ++ks) { #pragma unroll for (int dp = 0; dp < D / 16; ++dp) { int r = ks * 16 + (lane & 15); int c = dp * 2 + (lane >> 4); int cs = c ^ (r & 7); uint32_t addr = smem_u32(&sV[r * CHUNKS + cs]); uint32_t v00, v01, v10, v11; asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n" : "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11) : "r"(addr)); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3]) : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]), "r"(v00), "r"(v01), "f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3])); asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n" : "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3]) : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]), "r"(v10), "r"(v11), "f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3])); } } __syncwarp(); float inv_l0 = 1.0f / l0; float inv_l1 = 1.0f / l1; int r0_w = groupID; int r1_w = groupID + 8; #pragma unroll for (int dt = 0; dt < D_TILES; ++dt) { int dts0 = dt ^ r0_w; int dts1 = dt ^ (r1_w & 7); reinterpret_cast<__nv_bfloat162*>(&sQ_w[r0_w * CHUNKS + dts0])[t_in_g] = __float22bfloat162_rn(make_float2(o_acc[dt][0] * inv_l0, o_acc[dt][1] * inv_l0)); reinterpret_cast<__nv_bfloat162*>(&sQ_w[r1_w * CHUNKS + dts1])[t_in_g] = __float22bfloat162_rn(make_float2(o_acc[dt][2] * inv_l1, o_acc[dt][3] * inv_l1)); } uint32_t pk0 = __shfl_sync(0xffffffff, my_pk16, r0_w); uint32_t pk1 = __shfl_sync(0xffffffff, my_pk16, r1_w); if (t_in_g == 0) { if (r0_w < w_m) { size_t gt = static_cast(bh) * S + (pk0 >> 4); uint32_t slot = pk0 & 15u; ml_all[gt * 16 + 1 + slot] = make_float2(m0, l0); } if (r1_w < w_m) { size_t gt = static_cast(bh) * S + (pk1 >> 4); uint32_t slot = pk1 & 15u; ml_all[gt * 16 + 1 + slot] = make_float2(m1, l1); } } __syncwarp(); constexpr int ROWS_PER_STEP = 32 / CHUNKS; int sub_r = lane / CHUNKS; int sub_c = lane & (CHUNKS - 1); #pragma unroll for (int step = 0; step < 16; step += ROWS_PER_STEP) { int rl = step + sub_r; uint32_t pk = __shfl_sync(0xffffffff, my_pk16, rl); if (rl < w_m) { size_t gt = static_cast(bh) * S + (pk >> 4); uint32_t slot = pk & 15u; uint4 val = sQ_w[rl * CHUNKS + (sub_c ^ (rl & 7))]; reinterpret_cast(partial_O + (gt * 8 + slot) * D)[sub_c] = val; } } __syncwarp(); } __syncthreads(); } } // --------------------------------------------------------------------------- // Kernel 4: Unified High-Occupancy Combine Epilogue for t >= 128 // --------------------------------------------------------------------------- template __global__ __launch_bounds__(256) void combine_epilogue_kernel( const __nv_bfloat16* __restrict__ partial_O, const float2* __restrict__ ml_all, const uint8_t* __restrict__ remote_cnt, __nv_bfloat16* __restrict__ O, int S ) { constexpr int WIDTH = D / 8; constexpr int Q_PER_CTA = 256 / WIDTH; int tid = threadIdx.x; int q_local = tid / WIDTH; int sub_lane = tid & (WIDTH - 1); int t = 128 + blockIdx.x * Q_PER_CTA + q_local; if (t >= S) return; int bh = blockIdx.y; size_t gt = static_cast(bh) * S + t; int cnt = static_cast(__ldg(&remote_cnt[gt])); if (cnt == 0) return; int total_parts = cnt + 1; const float2* ml_ptr = ml_all + gt * 16; float m_arr[9]; float w_arr[9]; float m_max = -1e30f; #pragma unroll for (int k = 0; k < 9; ++k) { if (k < total_parts) { float2 mv = __ldg(&ml_ptr[k]); m_arr[k] = mv.x; w_arr[k] = mv.y; m_max = fmaxf(m_max, mv.x); } else { m_arr[k] = -1e30f; w_arr[k] = 0.f; } } float w_sum = 0.f; #pragma unroll for (int k = 0; k < 9; ++k) { float w = (k < total_parts) ? (w_arr[k] * __expf(m_arr[k] - m_max)) : 0.f; w_arr[k] = w; w_sum += w; } float inv_w_sum = 1.0f / w_sum; float acc[8]; uint4 u_loc = reinterpret_cast(O + gt * D)[sub_lane]; unpack_bf16x8(u_loc, acc); float w0 = w_arr[0] * inv_w_sum; #pragma unroll for (int j = 0; j < 8; ++j) { acc[j] *= w0; } const uint4* po_u4 = reinterpret_cast(partial_O + (gt * 8) * D) + sub_lane; #pragma unroll for (int slot = 0; slot < 8; ++slot) { if (slot < cnt) { float wk = w_arr[slot + 1] * inv_w_sum; uint4 u = __ldg(&po_u4[slot * WIDTH]); float vals[8]; unpack_bf16x8(u, vals); #pragma unroll for (int j = 0; j < 8; ++j) { acc[j] = fmaf(wk, vals[j], acc[j]); } } } reinterpret_cast(O + gt * D)[sub_lane] = pack_bf16x8(acc); } // --------------------------------------------------------------------------- // Cached Workspace Manager // --------------------------------------------------------------------------- struct Workspace { void* ptr = nullptr; size_t cap = 0; void* ensure(size_t needed) { if (needed > cap) { if (ptr) cudaFree(ptr); size_t alloc_sz = ((needed + (1 << 20) - 1) & ~((size_t)(1 << 20) - 1)); TORCH_CHECK(cudaMalloc(&ptr, alloc_sz) == cudaSuccess, "NSA workspace cudaMalloc failed"); cap = alloc_sz; } return ptr; } }; static Workspace g_ws[8]; inline size_t align256(size_t x) { return (x + 255) & ~((size_t)255); } template void launch_nsa_subbatch( const __nv_bfloat16* q_ptr, const __nv_bfloat16* k_ptr, const __nv_bfloat16* v_ptr, torch::Tensor& out, c10::IntArrayRef out_sizes, c10::Device out_dev, size_t head_off, int BH, int S, int dev, cudaStream_t stream ) { constexpr int CHUNK_Q = (D == 64) ? 128 : MAX_CHUNK_Q; int n_blocks = (S + BLOCK_SIZE - 1) / BLOCK_SIZE; int S_stride = (S + 31) & ~31; int chunks_per_s = (S + CHUNK_Q - 1) / CHUNK_Q; size_t sz_kmean_f = align256(static_cast(BH) * n_blocks * D * sizeof(float)); size_t sz_kmean_bf = (D == 128) ? align256(static_cast(BH) * n_blocks * D * sizeof(__nv_bfloat16)) : 0; size_t sz_kmean_er = (D == 128) ? align256(static_cast(BH) * n_blocks * sizeof(float)) : 0; size_t sz_bcounts = align256(static_cast(BH) * n_blocks * BCOUNT_STRIDE * sizeof(int)); size_t sz_cntrs = align256(2 * sizeof(uint32_t)); size_t sz_bitems = align256(static_cast(BH) * n_blocks * S_stride * sizeof(uint32_t)); size_t sz_wqueue = align256(static_cast(BH) * n_blocks * chunks_per_s * sizeof(uint2)); size_t sz_po = align256(static_cast(BH) * S * 8 * D * sizeof(__nv_bfloat16)); size_t sz_ml = align256(static_cast(BH) * S * 16 * sizeof(float2)); size_t sz_rcnt = align256(static_cast(BH) * S * sizeof(uint8_t)); size_t total_bytes = sz_kmean_f + sz_kmean_bf + sz_kmean_er + sz_bcounts + sz_cntrs + sz_bitems + sz_wqueue + sz_po + sz_ml + sz_rcnt; char* base = static_cast(g_ws[dev].ensure(total_bytes)); float* k_mean_f = reinterpret_cast(base); base += sz_kmean_f; __nv_bfloat16* k_mean_b = (D == 128) ? reinterpret_cast<__nv_bfloat16*>(base) : nullptr; base += sz_kmean_bf; float* k_mean_err = (D == 128) ? reinterpret_cast(base) : nullptr; base += sz_kmean_er; int* bucket_counts = reinterpret_cast(base); base += sz_bcounts; uint32_t* g_counters = reinterpret_cast(base); base += sz_cntrs; uint32_t* bucket_items = reinterpret_cast(base); base += sz_bitems; uint2* g_work_queue = reinterpret_cast(base); base += sz_wqueue; __nv_bfloat16* part_O = reinterpret_cast<__nv_bfloat16*>(base); base += sz_po; float2* ml_all = reinterpret_cast(base); base += sz_ml; uint8_t* remote_cnt = reinterpret_cast(base); float scale = static_cast(1.0 / std::sqrt(static_cast(D))); dim3 grid_blk(n_blocks, BH); k_mean_and_init_kernel<<>>( k_ptr, k_mean_f, k_mean_b, k_mean_err, bucket_counts, g_counters, S, n_blocks ); if (!out.defined()) { out = torch::Tensor(at::detail::empty_cuda( out_sizes, at::ScalarType::BFloat16, out_dev, std::nullopt )); } __nv_bfloat16* o_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()) + head_off; constexpr int CHUNKS = D / 8; constexpr int MIN_BLK_LOCAL = (D == 64) ? 4 : 3; constexpr size_t SMEM_LOCAL = 64 * CHUNKS * sizeof(uint4) + sizeof(LocalSharedBuf) + ((D == 64) ? (2 * 256 * sizeof(int)) : 0); constexpr size_t SMEM_REM = 3 * 64 * CHUNKS * sizeof(uint4); static bool attr_set[8] = {false}; if (!attr_set[dev]) { cudaFuncSetAttribute( local_attn_score_select_kernel, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxShared ); cudaFuncSetAttribute( local_attn_score_select_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast(SMEM_LOCAL) ); cudaFuncSetAttribute( remote_kv_attn_kernel, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxShared ); cudaFuncSetAttribute( remote_kv_attn_kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast(SMEM_REM) ); attr_set[dev] = true; } local_attn_score_select_kernel<<>>( q_ptr, k_ptr, v_ptr, k_mean_f, k_mean_b, k_mean_err, o_ptr, ml_all, remote_cnt, bucket_counts, bucket_items, g_work_queue, g_counters, S, n_blocks, S_stride, scale ); if (S > 128) { constexpr int PERSISTENT_CTAS = (D == 64) ? (188 * 4) : (188 * 2); remote_kv_attn_kernel<<>>( q_ptr, k_ptr, v_ptr, part_O, ml_all, bucket_counts, bucket_items, g_work_queue, g_counters, S, n_blocks, S_stride, scale ); constexpr int Q_PER_CTA = 256 / (D / 8); int active_s = S - 128; dim3 grid_epi((active_s + Q_PER_CTA - 1) / Q_PER_CTA, BH); combine_epilogue_kernel<<>>( part_O, ml_all, remote_cnt, o_ptr, S ); } } template void launch_nsa_l2_tiled( const __nv_bfloat16* q_ptr, const __nv_bfloat16* k_ptr, const __nv_bfloat16* v_ptr, torch::Tensor& out, c10::IntArrayRef out_sizes, c10::Device out_dev, int BH, int S, int dev, cudaStream_t stream ) { size_t bytes_per_head = static_cast(S) * D * 12 * sizeof(__nv_bfloat16); int max_bh_l2 = static_cast((112ULL * 1024 * 1024) / std::max(1ULL, bytes_per_head)); int bh_step = (max_bh_l2 >= BH) ? BH : std::max(1, (BH + 1) / 2); size_t head_stride = static_cast(S) * D; for (int bh0 = 0; bh0 < BH; bh0 += bh_step) { int cur_bh = std::min(bh_step, BH - bh0); size_t off = static_cast(bh0) * head_stride; launch_nsa_subbatch( q_ptr + off, k_ptr + off, v_ptr + off, out, out_sizes, out_dev, off, cur_bh, S, dev, stream ); } } } // namespace torch::Tensor nsa_forward_cuda( const torch::Tensor& q_in, const torch::Tensor& k_in, const torch::Tensor& v_in ) { TORCH_CHECK(q_in.is_cuda() && k_in.is_cuda() && v_in.is_cuda(), "q, k, v must be CUDA tensors"); TORCH_CHECK(q_in.scalar_type() == torch::kBFloat16, "q must be bfloat16"); torch::Tensor q_tmp, k_tmp, v_tmp; const torch::Tensor& q = q_in.is_contiguous() ? q_in : (q_tmp = q_in.contiguous()); const torch::Tensor& k = k_in.is_contiguous() ? k_in : (k_tmp = k_in.contiguous()); const torch::Tensor& v = v_in.is_contiguous() ? v_in : (v_tmp = v_in.contiguous()); int B = static_cast(q.size(0)); int H = static_cast(q.size(1)); int S = static_cast(q.size(2)); int D = static_cast(q.size(3)); int BH = B * H; int dev = q.get_device(); cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev); const __nv_bfloat16* q_ptr = reinterpret_cast(q.data_ptr()); const __nv_bfloat16* k_ptr = reinterpret_cast(k.data_ptr()); const __nv_bfloat16* v_ptr = reinterpret_cast(v.data_ptr()); torch::Tensor out; if (D == 64) { launch_nsa_l2_tiled<64>(q_ptr, k_ptr, v_ptr, out, q.sizes(), q.device(), BH, S, dev, stream); } else if (D == 128) { launch_nsa_l2_tiled<128>(q_ptr, k_ptr, v_ptr, out, q.sizes(), q.device(), BH, S, dev, stream); } else { TORCH_CHECK(false, "Unsupported head dim D: ", D); } return out; } """ _CPP_SOURCE = "torch::Tensor nsa_forward_cuda(const torch::Tensor& q, const torch::Tensor& k, const torch::Tensor& v);" _ext = load_inline( name="deepseek_nsa_sm120_v10", cpp_sources=_CPP_SOURCE, cuda_sources=_CUDA_SOURCE, functions=["nsa_forward_cuda"], extra_cuda_cflags=["-O3"], ) class Model(nn.Module): def __init__(self, B: int, H: int, S: int, D: int): super().__init__() self.B, self.H, self.S, self.D = B, H, S, D self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16)) def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: return _ext.nsa_forward_cuda(q, k, v)