KernelBench cuda · RTX PRO 6000
DeepSeek NSA Gemini 4 Carbon (High)
manually audited: clean
Gemini 4 Carbon wrote NSA block selection and sparse attention as four raw CUDA launches with inline-PTX bf16 mma.sync. A key-mean kernel stores fp32 block means of K; a per-query-block kernel scores every earlier block as q·mean(k) (exact fp32 FMA for D=64; for D=128 a bf16 tensor-core prefilter to 16 candidates, then fp32 rescoring wherever a Cauchy-Schwarz error bound cannot decide membership), takes the causal mean of the diagonal block from its tensor-core scores, keeps a top-8 with ties to the larger block index, attends the diagonal block and the previous block (whole if selected, else only the sliding-window tail), and scatters each remote selection into a per-key-block bucket. A persistent key-stationary kernel loads each selected K/V block once and attends all of its bucketed queries, and an epilogue merges the bf16 partials by log-sum-exp.
Per-shape latencymilliseconds are the measured result; the dense-equivalent roofline is not a useful ceiling
geomean(0.055 ms · 0.101 ms · 0.111 ms · 0.208 ms · 0.048 ms · 0.075 ms) = 0.088 ms
Kernel source (redacted)
from __future__ import annotations
import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "12.0a")
_CUDA_SOURCE = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <ATen/cuda/EmptyTensor.h>
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
#include <cmath>
#include <algorithm>
namespace {
constexpr int BLOCK_SIZE = 64;
constexpr int TOP_N_BLOCKS = 8;
constexpr int MAX_CHUNK_Q = 256;
constexpr int BCOUNT_STRIDE = 32; // 32 ints = 128B cache-line isolation per bucket
__device__ __forceinline__ uint32_t smem_u32(const void* ptr) {
return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}
__device__ __forceinline__ void cp_async_16B(uint32_t dst_smem, const void* src_gmem) {
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" :: "r"(dst_smem), "l"(src_gmem));
}
__device__ __forceinline__ void cp_async_commit() {
asm volatile("cp.async.commit_group;\n");
}
template <int N>
__device__ __forceinline__ void cp_async_wait() {
asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}
__device__ __forceinline__ bool pair_gt(float va, int ia, float vb, int ib) {
return (va > vb) || (va == vb && ia > ib);
}
__device__ __forceinline__ void cas_pair(float& va, int& ia, float& vb, int& ib, bool desc) {
bool swap = desc ? pair_gt(vb, ib, va, ia) : pair_gt(va, ia, vb, ib);
if (swap) {
float tv = va; va = vb; vb = tv;
int ti = ia; ia = ib; ib = ti;
}
}
__device__ __forceinline__ void bitonic_merge8_desc(float v[8], int idx[8]) {
cas_pair(v[0], idx[0], v[4], idx[4], true);
cas_pair(v[1], idx[1], v[5], idx[5], true);
cas_pair(v[2], idx[2], v[6], idx[6], true);
cas_pair(v[3], idx[3], v[7], idx[7], true);
cas_pair(v[0], idx[0], v[2], idx[2], true);
cas_pair(v[1], idx[1], v[3], idx[3], true);
cas_pair(v[4], idx[4], v[6], idx[6], true);
cas_pair(v[5], idx[5], v[7], idx[7], true);
cas_pair(v[0], idx[0], v[1], idx[1], true);
cas_pair(v[2], idx[2], v[3], idx[3], true);
cas_pair(v[4], idx[4], v[5], idx[5], true);
cas_pair(v[6], idx[6], v[7], idx[7], true);
}
__device__ __forceinline__ void bitonic_sort8_desc(float v[8], int idx[8]) {
cas_pair(v[0], idx[0], v[1], idx[1], true);
cas_pair(v[2], idx[2], v[3], idx[3], false);
cas_pair(v[4], idx[4], v[5], idx[5], true);
cas_pair(v[6], idx[6], v[7], idx[7], false);
cas_pair(v[0], idx[0], v[2], idx[2], true);
cas_pair(v[1], idx[1], v[3], idx[3], true);
cas_pair(v[0], idx[0], v[1], idx[1], true);
cas_pair(v[2], idx[2], v[3], idx[3], true);
cas_pair(v[4], idx[4], v[6], idx[6], false);
cas_pair(v[5], idx[5], v[7], idx[7], false);
cas_pair(v[4], idx[4], v[5], idx[5], false);
cas_pair(v[6], idx[6], v[7], idx[7], false);
bitonic_merge8_desc(v, idx);
}
__device__ __forceinline__ void merge_into_top8(float top_v[8], int top_i[8], const float cand_v[8], const int cand_i[8]) {
#pragma unroll
for (int i = 0; i < 8; ++i) {
float bv = cand_v[7 - i];
int bi = cand_i[7 - i];
if (pair_gt(bv, bi, top_v[i], top_i[i])) {
top_v[i] = bv;
top_i[i] = bi;
}
}
bitonic_merge8_desc(top_v, top_i);
}
__device__ __forceinline__ void insert_one_top8(float top_v[8], int top_i[8], float cv, int ci) {
#pragma unroll
for (int i = 0; i < 8; ++i) {
if (pair_gt(cv, ci, top_v[i], top_i[i])) {
float tv = top_v[i]; top_v[i] = cv; cv = tv;
int ti = top_i[i]; top_i[i] = ci; ci = ti;
}
}
}
// Merge 4 sorted-desc 8-arrays across a 4-lane quad (t_in_g = 0..3) into Global Top-16:
// Upon return, t_in_g == 0 holds Top-16[0..7] and t_in_g == 1 holds Top-16[8..15] in (lv, li).
__device__ __forceinline__ void quad_merge_top16(float lv[8], int li[8], int t_in_g) {
bool is_even = ((t_in_g & 1) == 0);
float pv[8];
int pi[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 1, 4);
pi[i] = __shfl_xor_sync(0xffffffff, li[i], 1, 4);
}
#pragma unroll
for (int i = 0; i < 8; ++i) {
float bv = pv[7 - i];
int bi = pi[7 - i];
bool b_gt_a = pair_gt(bv, bi, lv[i], li[i]);
if (is_even ? b_gt_a : !b_gt_a) {
lv[i] = bv;
li[i] = bi;
}
}
bitonic_merge8_desc(lv, li);
#pragma unroll
for (int i = 0; i < 8; ++i) {
pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 3, 4);
pi[i] = __shfl_xor_sync(0xffffffff, li[i], 3, 4);
}
#pragma unroll
for (int i = 0; i < 8; ++i) {
float bv = pv[7 - i];
int bi = pi[7 - i];
if (pair_gt(bv, bi, lv[i], li[i])) {
lv[i] = bv;
li[i] = bi;
}
}
#pragma unroll
for (int i = 0; i < 8; ++i) {
pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 1, 4);
pi[i] = __shfl_xor_sync(0xffffffff, li[i], 1, 4);
}
#pragma unroll
for (int i = 0; i < 8; ++i) {
bool p_gt_l = pair_gt(pv[i], pi[i], lv[i], li[i]);
if (is_even ? p_gt_l : !p_gt_l) {
lv[i] = pv[i];
li[i] = pi[i];
}
}
bitonic_merge8_desc(lv, li);
}
__device__ __forceinline__ void unpack_bf16x8(uint4 u, float out[8]) {
const __nv_bfloat162* p = reinterpret_cast<const __nv_bfloat162*>(&u);
#pragma unroll
for (int i = 0; i < 4; ++i) {
float2 f = __bfloat1622float2(p[i]);
out[i * 2 + 0] = f.x;
out[i * 2 + 1] = f.y;
}
}
__device__ __forceinline__ uint4 pack_bf16x8(const float in[8]) {
uint4 u;
__nv_bfloat162* p = reinterpret_cast<__nv_bfloat162*>(&u);
#pragma unroll
for (int i = 0; i < 4; ++i) {
p[i] = __float22bfloat162_rn(make_float2(in[i * 2 + 0], in[i * 2 + 1]));
}
return u;
}
// ---------------------------------------------------------------------------
// Kernel 1: High-MLP 256-thread per-block FP32 (+ BF16/err on D=128) key means
// ---------------------------------------------------------------------------
template <int D>
__global__ __launch_bounds__(256) void k_mean_and_init_kernel(
const __nv_bfloat16* __restrict__ K,
float* __restrict__ K_mean_fp32,
__nv_bfloat16* __restrict__ K_mean_bf16,
float* __restrict__ K_mean_err,
int* __restrict__ bucket_counts,
uint32_t* __restrict__ g_counters,
int S,
int n_blocks
) {
constexpr int CHUNKS = D / 8;
constexpr int ROW_GROUPS = 256 / CHUNKS;
constexpr int ITERS_FULL = BLOCK_SIZE / ROW_GROUPS;
int bi = blockIdx.x;
int bh = blockIdx.y;
int tid = threadIdx.x;
int warp = tid >> 5;
int lane = tid & 31;
if (bi == 0 && bh == 0 && tid == 0) {
g_counters[0] = 0u;
g_counters[1] = 0u;
}
if (tid == 0) {
bucket_counts[(bh * n_blocks + bi) * BCOUNT_STRIDE] = 0;
}
int s0 = bi * BLOCK_SIZE;
int valid_rows = min(BLOCK_SIZE, S - s0);
int col_c = tid & (CHUNKS - 1);
int row_g = tid / CHUNKS;
float acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
const __nv_bfloat16* k_base = K + (static_cast<size_t>(bh) * S + s0) * D;
if (valid_rows == BLOCK_SIZE) {
#pragma unroll
for (int it = 0; it < ITERS_FULL; ++it) {
int r = row_g + it * ROW_GROUPS;
uint4 u = reinterpret_cast<const uint4*>(k_base + r * D)[col_c];
float vals[8];
unpack_bf16x8(u, vals);
#pragma unroll
for (int j = 0; j < 8; ++j) {
acc[j] += vals[j];
}
}
} else {
for (int r = row_g; r < valid_rows; r += ROW_GROUPS) {
uint4 u = reinterpret_cast<const uint4*>(k_base + r * D)[col_c];
float vals[8];
unpack_bf16x8(u, vals);
#pragma unroll
for (int j = 0; j < 8; ++j) {
acc[j] += vals[j];
}
}
}
#pragma unroll
for (int j = 0; j < 8; ++j) {
acc[j] += __shfl_xor_sync(0xffffffff, acc[j], 16);
if constexpr (CHUNKS == 8) {
acc[j] += __shfl_xor_sync(0xffffffff, acc[j], 8);
}
}
__shared__ __align__(16) float smem_red[8][D + 4];
int d0 = col_c * 8;
if (lane < CHUNKS) {
reinterpret_cast<float4*>(&smem_red[warp][d0])[0] = make_float4(acc[0], acc[1], acc[2], acc[3]);
reinterpret_cast<float4*>(&smem_red[warp][d0])[1] = make_float4(acc[4], acc[5], acc[6], acc[7]);
}
__syncthreads();
if (warp == 0 && lane < CHUNKS) {
float sum[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
#pragma unroll
for (int rg = 0; rg < 8; ++rg) {
float4 v0 = reinterpret_cast<const float4*>(&smem_red[rg][d0])[0];
float4 v1 = reinterpret_cast<const float4*>(&smem_red[rg][d0])[1];
sum[0] += v0.x; sum[1] += v0.y; sum[2] += v0.z; sum[3] += v0.w;
sum[4] += v1.x; sum[5] += v1.y; sum[6] += v1.z; sum[7] += v1.w;
}
float inv_len = 1.0f / static_cast<float>(valid_rows);
float avg[8];
#pragma unroll
for (int j = 0; j < 8; ++j) {
avg[j] = sum[j] * inv_len;
}
size_t out_off = static_cast<size_t>(bh * n_blocks + bi) * D + d0;
float* out_f = K_mean_fp32 + out_off;
reinterpret_cast<float4*>(out_f)[0] = make_float4(avg[0], avg[1], avg[2], avg[3]);
reinterpret_cast<float4*>(out_f)[1] = make_float4(avg[4], avg[5], avg[6], avg[7]);
if constexpr (D == 128) {
uint4 packed_bf = pack_bf16x8(avg);
reinterpret_cast<uint4*>(K_mean_bf16 + out_off)[0] = packed_bf;
float avg_bf[8];
unpack_bf16x8(packed_bf, avg_bf);
float err_sq = 0.f, avg_sq = 0.f;
#pragma unroll
for (int j = 0; j < 8; ++j) {
float diff = avg[j] - avg_bf[j];
err_sq = fmaf(diff, diff, err_sq);
avg_sq = fmaf(avg[j], avg[j], avg_sq);
}
constexpr uint32_t m_grp = 0x0000ffffu;
err_sq += __shfl_xor_sync(m_grp, err_sq, 1);
err_sq += __shfl_xor_sync(m_grp, err_sq, 2);
err_sq += __shfl_xor_sync(m_grp, err_sq, 4);
err_sq += __shfl_xor_sync(m_grp, err_sq, 8);
avg_sq += __shfl_xor_sync(m_grp, avg_sq, 1);
avg_sq += __shfl_xor_sync(m_grp, avg_sq, 2);
avg_sq += __shfl_xor_sync(m_grp, avg_sq, 4);
avg_sq += __shfl_xor_sync(m_grp, avg_sq, 8);
if (tid == 0) {
K_mean_err[bh * n_blocks + bi] = sqrtf(err_sq) * 1.01f + sqrtf(avg_sq) * 1e-5f;
}
}
}
}
// ---------------------------------------------------------------------------
// Kernel 2: Zero-Spill Local Attention + Adaptive SMEM / BF16 Tensor-Core
// Top-16 Prefilter with Exact Cauchy-Schwarz Fast-Skip
// ---------------------------------------------------------------------------
template <int D>
union LocalSharedBuf {
uint4 kv[64 * (D / 8)];
float kmean16[D][20];
float kmean32[D][(D == 64) ? 36 : 20];
int bcount_base[512];
};
template <int D, int MIN_BLOCKS_PER_SM>
__global__ __launch_bounds__(128, MIN_BLOCKS_PER_SM) void local_attn_score_select_kernel(
const __nv_bfloat16* __restrict__ Q,
const __nv_bfloat16* __restrict__ K,
const __nv_bfloat16* __restrict__ V,
const float* __restrict__ K_mean_fp32,
const __nv_bfloat16* __restrict__ K_mean_bf16,
const float* __restrict__ K_mean_err,
__nv_bfloat16* __restrict__ O,
float2* __restrict__ ml_all,
uint8_t* __restrict__ remote_cnt,
int* __restrict__ bucket_counts,
uint32_t* __restrict__ bucket_items,
uint2* __restrict__ g_work_queue,
uint32_t* __restrict__ g_counters,
int S,
int n_blocks,
int S_stride,
float scale
) {
constexpr int CHUNKS = D / 8;
constexpr int K_STEPS = D / 16;
constexpr int D_TILES = D / 8;
constexpr int MAX_SMEM_BUCKETS = 256;
constexpr int CHUNK_Q = (D == 64) ? 128 : 256;
extern __shared__ __align__(128) char smem_local_raw[];
uint4* sQ = reinterpret_cast<uint4*>(smem_local_raw);
LocalSharedBuf<D>* sBuf = reinterpret_cast<LocalSharedBuf<D>*>(sQ + 64 * CHUNKS);
int* s_bcount_sep = (D == 64) ? reinterpret_cast<int*>(sBuf + 1) : nullptr;
int* s_bbase_sep = (D == 64) ? (s_bcount_sep + MAX_SMEM_BUCKETS) : nullptr;
int qb = (n_blocks - 1) - static_cast<int>(blockIdx.x);
int bh = blockIdx.y;
int tid = threadIdx.x;
int warp = tid >> 5;
int lane = tid & 31;
int groupID = lane >> 2;
int t_in_g = lane & 3;
int s0 = qb * BLOCK_SIZE;
int M = min(BLOCK_SIZE, S - s0);
int r0 = warp * 16 + groupID;
int r1 = r0 + 8;
const __nv_bfloat16* q_gmem = Q + static_cast<size_t>(bh) * S * D;
const __nv_bfloat16* k_gmem = K + static_cast<size_t>(bh) * S * D;
const __nv_bfloat16* v_gmem = V + static_cast<size_t>(bh) * S * D;
if constexpr (D == 64) {
if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) {
for (int b = tid; b < qb - 1; b += 128) {
s_bcount_sep[b] = 0;
}
}
}
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < M) {
cp_async_16B(smem_u32(&sQ[r * CHUNKS + cs]), &q_gmem[(s0 + r) * D + c * 8]);
if (qb == 0) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
}
} else {
sQ[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
if (qb == 0) {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
}
cp_async_commit();
cp_async_wait<0>();
__syncthreads();
// -----------------------------------------------------------------------
// PHASE 1: Score [0 .. qb-1] -> Only 16 regs (my_tv, my_ti) survive!
// -----------------------------------------------------------------------
float my_tv[8];
int my_ti[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
my_tv[i] = -1e30f;
my_ti[i] = -1;
}
if (qb >= 1) {
if (qb <= 16) {
// Fast Path A1 (1 <= qb <= 16): Single 16-col SMEM FP32 tile
const float* km_fp32 = K_mean_fp32 + static_cast<size_t>(bh * n_blocks) * D;
constexpr int F4_PER_ROW = D / 4;
for (int idx = tid; idx < 16 * F4_PER_ROW; idx += 128) {
int br = idx / F4_PER_ROW;
int dc4 = idx % F4_PER_ROW;
int d = dc4 * 4;
float4 fv = (br < qb)
? reinterpret_cast<const float4*>(km_fp32)[br * F4_PER_ROW + dc4]
: make_float4(0.f, 0.f, 0.f, 0.f);
sBuf->kmean16[d + 0][br] = fv.x;
sBuf->kmean16[d + 1][br] = fv.y;
sBuf->kmean16[d + 2][br] = fv.z;
sBuf->kmean16[d + 3][br] = fv.w;
}
__syncthreads();
float cv0[4] = {0.f, 0.f, 0.f, 0.f};
float cv1[4] = {0.f, 0.f, 0.f, 0.f};
#pragma unroll 2
for (int dc = 0; dc < CHUNKS; ++dc) {
uint4 uq0 = sQ[r0 * CHUNKS + (dc ^ (r0 & 7))];
uint4 uq1 = sQ[r1 * CHUNKS + (dc ^ (r1 & 7))];
float qv0[8], qv1[8];
unpack_bf16x8(uq0, qv0);
unpack_bf16x8(uq1, qv1);
#pragma unroll
for (int j = 0; j < 8; ++j) {
int d = dc * 8 + j;
float q0v = qv0[j];
float q1v = qv1[j];
float4 km = *reinterpret_cast<const float4*>(&sBuf->kmean16[d][t_in_g * 4]);
cv0[0] = fmaf(q0v, km.x, cv0[0]); cv1[0] = fmaf(q1v, km.x, cv1[0]);
cv0[1] = fmaf(q0v, km.y, cv0[1]); cv1[1] = fmaf(q1v, km.y, cv1[1]);
cv0[2] = fmaf(q0v, km.z, cv0[2]); cv1[2] = fmaf(q1v, km.z, cv1[2]);
cv0[3] = fmaf(q0v, km.w, cv0[3]); cv1[3] = fmaf(q1v, km.w, cv1[3]);
}
}
__syncthreads();
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < M) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
} else {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
int ci[4];
#pragma unroll
for (int k = 0; k < 4; ++k) {
int br = t_in_g * 4 + k;
if (br < qb) {
cv0[k] *= scale;
cv1[k] *= scale;
ci[k] = br;
} else {
cv0[k] = -1e30f;
cv1[k] = -1e30f;
ci[k] = -1;
}
}
bool is_odd = ((t_in_g & 1) != 0);
float send_v[4];
#pragma unroll
for (int k = 0; k < 4; ++k) send_v[k] = is_odd ? cv0[k] : cv1[k];
#pragma unroll
for (int k = 0; k < 4; ++k) {
float recv_v = __shfl_xor_sync(0xffffffff, send_v[k], 1, 4);
int recv_i = __shfl_xor_sync(0xffffffff, ci[k], 1, 4);
my_tv[k] = is_odd ? recv_v : cv0[k];
my_ti[k] = is_odd ? recv_i : ci[k];
my_tv[4 + k] = is_odd ? cv1[k] : recv_v;
my_ti[4 + k] = is_odd ? ci[k] : recv_i;
}
bitonic_sort8_desc(my_tv, my_ti);
float other_v[8];
int other_i[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
other_v[i] = __shfl_xor_sync(0xffffffff, my_tv[i], 2, 4);
other_i[i] = __shfl_xor_sync(0xffffffff, my_ti[i], 2, 4);
}
merge_into_top8(my_tv, my_ti, other_v, other_i);
} else if constexpr (D == 64) {
// Fast Path A2 (D == 64, qb > 16): Tiled 32-col SMEM FP32 exact scoring
// Each lane t_in_g in {0..3} owns disjoint blocks (b_tile + t_in_g*8 + 0..7),
// so zero warp shuffles occur inside the b_tile loop!
float lv0[8], lv1[8];
int li0[8], li1[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
lv0[i] = -1e30f; li0[i] = -1;
lv1[i] = -1e30f; li1[i] = -1;
}
const float* km_fp32 = K_mean_fp32 + static_cast<size_t>(bh * n_blocks) * D;
constexpr int F4_PER_ROW = D / 4;
#pragma unroll 1
for (int b_tile = 0; b_tile < qb; b_tile += 32) {
int b_cnt = min(32, qb - b_tile);
for (int idx = tid; idx < 32 * F4_PER_ROW; idx += 128) {
int br = idx / F4_PER_ROW;
int dc4 = idx % F4_PER_ROW;
int d = dc4 * 4;
float4 fv = (br < b_cnt)
? reinterpret_cast<const float4*>(km_fp32)[(b_tile + br) * F4_PER_ROW + dc4]
: make_float4(0.f, 0.f, 0.f, 0.f);
sBuf->kmean32[d + 0][br] = fv.x;
sBuf->kmean32[d + 1][br] = fv.y;
sBuf->kmean32[d + 2][br] = fv.z;
sBuf->kmean32[d + 3][br] = fv.w;
}
__syncthreads();
float cv0[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
float cv1[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
#pragma unroll 2
for (int dc = 0; dc < CHUNKS; ++dc) {
uint4 uq0 = sQ[r0 * CHUNKS + (dc ^ (r0 & 7))];
uint4 uq1 = sQ[r1 * CHUNKS + (dc ^ (r1 & 7))];
float qv0[8], qv1[8];
unpack_bf16x8(uq0, qv0);
unpack_bf16x8(uq1, qv1);
#pragma unroll
for (int j = 0; j < 8; ++j) {
int d = dc * 8 + j;
float q0v = qv0[j];
float q1v = qv1[j];
float4 km0 = *reinterpret_cast<const float4*>(&sBuf->kmean32[d][t_in_g * 8]);
float4 km1 = *reinterpret_cast<const float4*>(&sBuf->kmean32[d][t_in_g * 8 + 4]);
cv0[0] = fmaf(q0v, km0.x, cv0[0]); cv1[0] = fmaf(q1v, km0.x, cv1[0]);
cv0[1] = fmaf(q0v, km0.y, cv0[1]); cv1[1] = fmaf(q1v, km0.y, cv1[1]);
cv0[2] = fmaf(q0v, km0.z, cv0[2]); cv1[2] = fmaf(q1v, km0.z, cv1[2]);
cv0[3] = fmaf(q0v, km0.w, cv0[3]); cv1[3] = fmaf(q1v, km0.w, cv1[3]);
cv0[4] = fmaf(q0v, km1.x, cv0[4]); cv1[4] = fmaf(q1v, km1.x, cv1[4]);
cv0[5] = fmaf(q0v, km1.y, cv0[5]); cv1[5] = fmaf(q1v, km1.y, cv1[5]);
cv0[6] = fmaf(q0v, km1.z, cv0[6]); cv1[6] = fmaf(q1v, km1.z, cv1[6]);
cv0[7] = fmaf(q0v, km1.w, cv0[7]); cv1[7] = fmaf(q1v, km1.w, cv1[7]);
}
}
int ci0[8], ci1[8];
#pragma unroll
for (int k = 0; k < 8; ++k) {
int br = t_in_g * 8 + k;
if (br < b_cnt) {
int gbi = b_tile + br;
cv0[k] *= scale;
cv1[k] *= scale;
ci0[k] = gbi;
ci1[k] = gbi;
} else {
cv0[k] = -1e30f; ci0[k] = -1;
cv1[k] = -1e30f; ci1[k] = -1;
}
}
bitonic_sort8_desc(cv0, ci0);
bitonic_sort8_desc(cv1, ci1);
if (b_tile == 0) {
#pragma unroll
for (int k = 0; k < 8; ++k) {
lv0[k] = cv0[k]; li0[k] = ci0[k];
lv1[k] = cv1[k]; li1[k] = ci1[k];
}
} else {
merge_into_top8(lv0, li0, cv0, ci0);
merge_into_top8(lv1, li1, cv1, ci1);
}
__syncthreads();
}
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < M) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
} else {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
// Simultaneous 2-step quad merge: lanes 0,1 merge r0 while lanes 2,3 merge r1!
bool high_half = (t_in_g >= 2);
float send_v[8];
int send_i[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
send_v[i] = high_half ? lv0[i] : lv1[i];
send_i[i] = high_half ? li0[i] : li1[i];
my_tv[i] = high_half ? lv1[i] : lv0[i];
my_ti[i] = high_half ? li1[i] : li0[i];
}
float recv_v[8];
int recv_i[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
recv_v[i] = __shfl_xor_sync(0xffffffff, send_v[i], 2, 4);
recv_i[i] = __shfl_xor_sync(0xffffffff, send_i[i], 2, 4);
}
merge_into_top8(my_tv, my_ti, recv_v, recv_i);
#pragma unroll
for (int i = 0; i < 8; ++i) {
recv_v[i] = __shfl_xor_sync(0xffffffff, my_tv[i], 1, 4);
recv_i[i] = __shfl_xor_sync(0xffffffff, my_ti[i], 1, 4);
}
merge_into_top8(my_tv, my_ti, recv_v, recv_i);
#pragma unroll
for (int i = 0; i < 8; ++i) {
float r1_v = __shfl_xor_sync(0xffffffff, my_tv[i], 2, 4);
int r1_i = __shfl_xor_sync(0xffffffff, my_ti[i], 2, 4);
if (t_in_g == 1) {
my_tv[i] = r1_v;
my_ti[i] = r1_i;
}
}
} else {
// Fast Path B (D == 128, qb > 16): BF16 Tensor-Core Top-16 Prefilter
// + Per-Candidate Exact Cauchy-Schwarz Fast-Skip Rescore
float lv0[8], lv1[8];
int li0[8], li1[8];
#pragma unroll
for (int i = 0; i < 8; ++i) {
lv0[i] = -1e30f; li0[i] = -1;
lv1[i] = -1e30f; li1[i] = -1;
}
const __nv_bfloat16* km_bf16 = K_mean_bf16 + static_cast<size_t>(bh * n_blocks) * D;
#pragma unroll 1
for (int b_tile = 0; b_tile < qb; b_tile += 64) {
int b_cnt = min(64, qb - b_tile);
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < b_cnt) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &km_bf16[(b_tile + r) * D + c * 8]);
} else {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
cp_async_wait<0>();
__syncthreads();
float km_acc[8][4];
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
km_acc[nt][0] = km_acc[nt][1] = km_acc[nt][2] = km_acc[nt][3] = 0.f;
}
#pragma unroll
for (int ks = 0; ks < K_STEPS; ++ks) {
uint32_t q0, q1, q2, q3;
{
int rq = warp * 16 + (lane & 15);
int cq = ks * 2 + (lane >> 4);
uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]);
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3)
: "r"(q_addr));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
int c = ks * 2 + ((lane >> 3) & 1);
int cs = c ^ (n_row & 7);
uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]);
uint32_t b00, b01, b10, b11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(km_acc[np * 2][0]), "=f"(km_acc[np * 2][1]), "=f"(km_acc[np * 2][2]), "=f"(km_acc[np * 2][3])
: "r"(q0), "r"(q1), "r"(q2), "r"(q3),
"r"(b00), "r"(b01),
"f"(km_acc[np * 2][0]), "f"(km_acc[np * 2][1]), "f"(km_acc[np * 2][2]), "f"(km_acc[np * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(km_acc[np * 2 + 1][0]), "=f"(km_acc[np * 2 + 1][1]), "=f"(km_acc[np * 2 + 1][2]), "=f"(km_acc[np * 2 + 1][3])
: "r"(q0), "r"(q1), "r"(q2), "r"(q3),
"r"(b10), "r"(b11),
"f"(km_acc[np * 2 + 1][0]), "f"(km_acc[np * 2 + 1][1]), "f"(km_acc[np * 2 + 1][2]), "f"(km_acc[np * 2 + 1][3]));
}
}
#pragma unroll
for (int half = 0; half < 2; ++half) {
float hv[8];
int hi[8];
#pragma unroll
for (int nt = 0; nt < 4; ++nt) {
int col0 = (half * 4 + nt) * 8 + t_in_g * 2;
int col1 = col0 + 1;
hv[nt * 2 + 0] = (col0 < b_cnt) ? km_acc[half * 4 + nt][0] : -1e30f;
hi[nt * 2 + 0] = (col0 < b_cnt) ? (b_tile + col0) : -1;
hv[nt * 2 + 1] = (col1 < b_cnt) ? km_acc[half * 4 + nt][1] : -1e30f;
hi[nt * 2 + 1] = (col1 < b_cnt) ? (b_tile + col1) : -1;
}
bitonic_sort8_desc(hv, hi);
merge_into_top8(lv0, li0, hv, hi);
#pragma unroll
for (int nt = 0; nt < 4; ++nt) {
int col0 = (half * 4 + nt) * 8 + t_in_g * 2;
int col1 = col0 + 1;
hv[nt * 2 + 0] = (col0 < b_cnt) ? km_acc[half * 4 + nt][2] : -1e30f;
hi[nt * 2 + 0] = (col0 < b_cnt) ? (b_tile + col0) : -1;
hv[nt * 2 + 1] = (col1 < b_cnt) ? km_acc[half * 4 + nt][3] : -1e30f;
hi[nt * 2 + 1] = (col1 < b_cnt) ? (b_tile + col1) : -1;
}
bitonic_sort8_desc(hv, hi);
merge_into_top8(lv1, li1, hv, hi);
}
__syncthreads();
}
// Immediately prefetch K_qb -> sBuf->kv while running quad_merge_top16 + rescore!
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < M) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
} else {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
quad_merge_top16(lv0, li0, t_in_g);
quad_merge_top16(lv1, li1, t_in_g);
constexpr int F16_STEPS = D / 16;
uint2 q0_u2[F16_STEPS];
uint2 q1_u2[F16_STEPS];
float q0_sq = 0.f, q1_sq = 0.f;
#pragma unroll
for (int s = 0; s < F16_STEPS; ++s) {
int c = s * 2 + (t_in_g >> 1);
int sub = t_in_g & 1;
uint2 u0 = reinterpret_cast<const uint2*>(&sQ[r0 * CHUNKS + (c ^ (r0 & 7))])[sub];
uint2 u1 = reinterpret_cast<const uint2*>(&sQ[r1 * CHUNKS + (c ^ (r1 & 7))])[sub];
q0_u2[s] = u0;
q1_u2[s] = u1;
float2 f0a = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u0.x));
float2 f0b = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u0.y));
q0_sq = fmaf(f0a.x, f0a.x, fmaf(f0a.y, f0a.y, fmaf(f0b.x, f0b.x, fmaf(f0b.y, f0b.y, q0_sq))));
float2 f1a = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u1.x));
float2 f1b = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u1.y));
q1_sq = fmaf(f1a.x, f1a.x, fmaf(f1a.y, f1a.y, fmaf(f1b.x, f1b.x, fmaf(f1b.y, f1b.y, q1_sq))));
}
q0_sq += __shfl_xor_sync(0xffffffff, q0_sq, 1);
q0_sq += __shfl_xor_sync(0xffffffff, q0_sq, 2);
q1_sq += __shfl_xor_sync(0xffffffff, q1_sq, 1);
q1_sq += __shfl_xor_sync(0xffffffff, q1_sq, 2);
float max_k_err = 0.f;
const float* k_err_base = K_mean_err + bh * n_blocks;
for (int b = lane; b < qb; b += 32) {
max_k_err = fmaxf(max_k_err, __ldg(&k_err_base[b]));
}
#pragma unroll
for (int mask = 16; mask >= 1; mask >>= 1) {
max_k_err = fmaxf(max_k_err, __shfl_xor_sync(0xffffffff, max_k_err, mask));
}
float two_delta0 = 2.0f * sqrtf(q0_sq) * max_k_err;
float two_delta1 = 2.0f * sqrtf(q1_sq) * max_k_err;
float v7_0 = __shfl_sync(0xffffffff, lv0[7], 0, 4);
float v7_1 = __shfl_sync(0xffffffff, lv1[7], 0, 4);
uint32_t quad_mask = 0xfu << (groupID * 4);
const float* km_fp32 = K_mean_fp32 + static_cast<size_t>(bh * n_blocks) * D;
#pragma unroll 1
for (int step = 0; step < 16; ++step) {
int src_lane = (step < 8) ? 0 : 1;
int b0 = __shfl_sync(0xffffffff, li0[0], src_lane, 4);
float hat0 = __shfl_sync(0xffffffff, lv0[0], src_lane, 4);
int b1 = __shfl_sync(0xffffffff, li1[0], src_lane, 4);
float hat1 = __shfl_sync(0xffffffff, lv1[0], src_lane, 4);
if ((t_in_g == 0 && step < 8) || (t_in_g == 1 && step >= 8)) {
#pragma unroll
for (int j = 0; j < 7; ++j) {
li0[j] = li0[j + 1]; lv0[j] = lv0[j + 1];
li1[j] = li1[j + 1]; lv1[j] = lv1[j + 1];
}
}
bool sure_out0 = (b0 < 0) || (step >= 8 && (v7_0 - hat0 > two_delta0));
bool sure_in0 = (!sure_out0) && (step < 7 && (hat0 - v7_0 > two_delta0));
bool need0 = (!sure_out0) && (!sure_in0);
bool sure_out1 = (b1 < 0) || (step >= 8 && (v7_1 - hat1 > two_delta1));
bool sure_in1 = (!sure_out1) && (step < 7 && (hat1 - v7_1 > two_delta1));
bool need1 = (!sure_out1) && (!sure_in1);
float score0 = hat0 * scale;
if (need0) {
float dot0 = 0.f;
const float4* km0 = reinterpret_cast<const float4*>(km_fp32 + static_cast<size_t>(b0) * D);
#pragma unroll
for (int s = 0; s < F16_STEPS; ++s) {
float4 kv = km0[s * 4 + t_in_g];
float2 q_lo = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q0_u2[s].x));
float2 q_hi = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q0_u2[s].y));
dot0 = fmaf(q_lo.x, kv.x, dot0);
dot0 = fmaf(q_lo.y, kv.y, dot0);
dot0 = fmaf(q_hi.x, kv.z, dot0);
dot0 = fmaf(q_hi.y, kv.w, dot0);
}
dot0 += __shfl_xor_sync(quad_mask, dot0, 1);
dot0 += __shfl_xor_sync(quad_mask, dot0, 2);
score0 = dot0 * scale;
}
float score1 = hat1 * scale;
if (need1) {
float dot1 = 0.f;
const float4* km1 = reinterpret_cast<const float4*>(km_fp32 + static_cast<size_t>(b1) * D);
#pragma unroll
for (int s = 0; s < F16_STEPS; ++s) {
float4 kv = km1[s * 4 + t_in_g];
float2 q_lo = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q1_u2[s].x));
float2 q_hi = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q1_u2[s].y));
dot1 = fmaf(q_lo.x, kv.x, dot1);
dot1 = fmaf(q_lo.y, kv.y, dot1);
dot1 = fmaf(q_hi.x, kv.z, dot1);
dot1 = fmaf(q_hi.y, kv.w, dot1);
}
dot1 += __shfl_xor_sync(quad_mask, dot1, 1);
dot1 += __shfl_xor_sync(quad_mask, dot1, 2);
score1 = dot1 * scale;
}
bool my_keep = (t_in_g == 0) ? (!sure_out0) : (!sure_out1);
if (t_in_g < 2 && my_keep) {
int my_b = (t_in_g == 0) ? b0 : b1;
float my_sc = (t_in_g == 0) ? score0 : score1;
insert_one_top8(my_tv, my_ti, my_sc, my_b);
}
}
}
cp_async_wait<0>();
__syncthreads();
}
// -----------------------------------------------------------------------
// PHASE 2: Diagonal Attention [qb] + imp_diag Top-8 Insert + Bucket Emit
// (Load q0..q3 per ks from sQ so q_frag[K_STEPS][4] is NOT held live!)
// -----------------------------------------------------------------------
float s_acc[8][4];
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
s_acc[nt][0] = s_acc[nt][1] = s_acc[nt][2] = s_acc[nt][3] = 0.f;
}
#pragma unroll
for (int ks = 0; ks < K_STEPS; ++ks) {
uint32_t q0, q1, q2, q3;
{
int rq = warp * 16 + (lane & 15);
int cq = ks * 2 + (lane >> 4);
uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]);
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3)
: "r"(q_addr));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
int c = ks * 2 + ((lane >> 3) & 1);
int cs = c ^ (n_row & 7);
uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]);
uint32_t b00, b01, b10, b11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(s_acc[np * 2][0]), "=f"(s_acc[np * 2][1]), "=f"(s_acc[np * 2][2]), "=f"(s_acc[np * 2][3])
: "r"(q0), "r"(q1), "r"(q2), "r"(q3),
"r"(b00), "r"(b01),
"f"(s_acc[np * 2][0]), "f"(s_acc[np * 2][1]), "f"(s_acc[np * 2][2]), "f"(s_acc[np * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(s_acc[np * 2 + 1][0]), "=f"(s_acc[np * 2 + 1][1]), "=f"(s_acc[np * 2 + 1][2]), "=f"(s_acc[np * 2 + 1][3])
: "r"(q0), "r"(q1), "r"(q2), "r"(q3),
"r"(b10), "r"(b11),
"f"(s_acc[np * 2 + 1][0]), "f"(s_acc[np * 2 + 1][1]), "f"(s_acc[np * 2 + 1][2]), "f"(s_acc[np * 2 + 1][3]));
}
}
__syncthreads();
if constexpr (D == 64) {
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < M) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(s0 + r) * D + c * 8]);
} else {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
} else {
if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) {
for (int b = tid; b < qb - 1; b += 128) {
sBuf->bcount_base[b] = 0;
}
}
}
float diag_sum0 = 0.f, diag_sum1 = 0.f;
float m0 = -1e30f, m1 = -1e30f;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
int col0 = nt * 8 + t_in_g * 2;
int col1 = col0 + 1;
float v00 = s_acc[nt][0] * scale;
float v01 = s_acc[nt][1] * scale;
float v10 = s_acc[nt][2] * scale;
float v11 = s_acc[nt][3] * scale;
if (col0 <= r0) { diag_sum0 += v00; m0 = fmaxf(m0, v00); s_acc[nt][0] = v00; } else { s_acc[nt][0] = -1e30f; }
if (col1 <= r0) { diag_sum0 += v01; m0 = fmaxf(m0, v01); s_acc[nt][1] = v01; } else { s_acc[nt][1] = -1e30f; }
if (col0 <= r1) { diag_sum1 += v10; m1 = fmaxf(m1, v10); s_acc[nt][2] = v10; } else { s_acc[nt][2] = -1e30f; }
if (col1 <= r1) { diag_sum1 += v11; m1 = fmaxf(m1, v11); s_acc[nt][3] = v11; } else { s_acc[nt][3] = -1e30f; }
}
diag_sum0 += __shfl_xor_sync(0xffffffff, diag_sum0, 1);
diag_sum0 += __shfl_xor_sync(0xffffffff, diag_sum0, 2);
diag_sum1 += __shfl_xor_sync(0xffffffff, diag_sum1, 1);
diag_sum1 += __shfl_xor_sync(0xffffffff, diag_sum1, 2);
float imp_diag0 = diag_sum0 / static_cast<float>(r0 + 1);
float imp_diag1 = diag_sum1 / static_cast<float>(r1 + 1);
m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 1));
m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 2));
m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 1));
m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 2));
float l0 = 0.f, l1 = 0.f;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
s_acc[nt][0] = __expf(s_acc[nt][0] - m0);
s_acc[nt][1] = __expf(s_acc[nt][1] - m0);
s_acc[nt][2] = __expf(s_acc[nt][2] - m1);
s_acc[nt][3] = __expf(s_acc[nt][3] - m1);
l0 += s_acc[nt][0] + s_acc[nt][1];
l1 += s_acc[nt][2] + s_acc[nt][3];
}
l0 += __shfl_xor_sync(0xffffffff, l0, 1);
l0 += __shfl_xor_sync(0xffffffff, l0, 2);
l1 += __shfl_xor_sync(0xffffffff, l1, 1);
l1 += __shfl_xor_sync(0xffffffff, l1, 2);
uint32_t p_frag[4][4];
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
__nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][0], s_acc[ks * 2][1]));
__nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][2], s_acc[ks * 2][3]));
__nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][0], s_acc[ks * 2 + 1][1]));
__nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][2], s_acc[ks * 2 + 1][3]));
p_frag[ks][0] = *reinterpret_cast<uint32_t*>(&v0);
p_frag[ks][1] = *reinterpret_cast<uint32_t*>(&v1);
p_frag[ks][2] = *reinterpret_cast<uint32_t*>(&v2);
p_frag[ks][3] = *reinterpret_cast<uint32_t*>(&v3);
}
bool prev_in_top0 = false, prev_in_top1 = false;
int cnt0 = 0, cnt1 = 0;
if (qb >= 1) {
if (t_in_g < 2) {
insert_one_top8(my_tv, my_ti, (t_in_g == 0) ? imp_diag0 : imp_diag1, qb);
}
bool my_prev = false;
int my_cnt = 0;
#pragma unroll
for (int i = 0; i < 8; ++i) {
if (my_ti[i] == qb - 1) my_prev = true;
if (my_ti[i] >= 0 && my_ti[i] < qb - 1) my_cnt++;
}
prev_in_top0 = __shfl_sync(0xffffffff, my_prev, 0, 4);
prev_in_top1 = __shfl_sync(0xffffffff, my_prev, 1, 4);
cnt0 = __shfl_sync(0xffffffff, my_cnt, 0, 4);
cnt1 = __shfl_sync(0xffffffff, my_cnt, 1, 4);
if (qb >= 2) {
int my_r = (t_in_g == 0) ? r0 : r1;
int my_lpos[8] = {0, 0, 0, 0, 0, 0, 0, 0};
if (qb - 1 <= MAX_SMEM_BUCKETS) {
int* s_bc = (D == 64) ? s_bcount_sep : sBuf->bcount_base;
int* s_bb = (D == 64) ? s_bbase_sep : (sBuf->bcount_base + MAX_SMEM_BUCKETS);
if constexpr (D != 64) {
__syncthreads();
}
if (t_in_g < 2 && my_r < M) {
#pragma unroll
for (int i = 0; i < 8; ++i) {
int bi = my_ti[i];
if (bi >= 0 && bi < qb - 1) {
my_lpos[i] = atomicAdd(&s_bc[bi], 1);
}
}
}
__syncthreads();
for (int b = tid; b < qb - 1; b += 128) {
int c_blk = s_bc[b];
if (c_blk > 0) {
int bucket_id = bh * n_blocks + b;
int gbase = atomicAdd(&bucket_counts[bucket_id * BCOUNT_STRIDE], c_blk);
s_bb[b] = gbase;
int first_chunk = (gbase + CHUNK_Q - 1) & ~(CHUNK_Q - 1);
for (int p = first_chunk; p < gbase + c_blk; p += CHUNK_Q) {
uint32_t wid = atomicAdd(&g_counters[0], 1u);
g_work_queue[wid] = make_uint2(static_cast<uint32_t>(bucket_id), static_cast<uint32_t>(p));
}
}
}
__syncthreads();
if (t_in_g < 2 && my_r < M) {
int slot = 0;
uint32_t t_global = static_cast<uint32_t>(s0 + my_r);
#pragma unroll
for (int i = 0; i < 8; ++i) {
int bi = my_ti[i];
if (bi >= 0 && bi < qb - 1) {
int bucket_id = bh * n_blocks + bi;
int pos = s_bb[bi] + my_lpos[i];
bucket_items[static_cast<size_t>(bucket_id) * S_stride + pos] =
(t_global << 4) | static_cast<uint32_t>(slot);
slot++;
}
}
}
} else {
constexpr int chunk_mask = CHUNK_Q - 1;
if (t_in_g < 2 && my_r < M) {
int slot = 0;
uint32_t t_global = static_cast<uint32_t>(s0 + my_r);
#pragma unroll
for (int i = 0; i < 8; ++i) {
int bi = my_ti[i];
if (bi >= 0 && bi < qb - 1) {
int bucket_id = bh * n_blocks + bi;
int pos = atomicAdd(&bucket_counts[bucket_id * BCOUNT_STRIDE], 1);
bucket_items[static_cast<size_t>(bucket_id) * S_stride + pos] =
(t_global << 4) | static_cast<uint32_t>(slot);
if ((pos & chunk_mask) == 0) {
uint32_t wid = atomicAdd(&g_counters[0], 1u);
g_work_queue[wid] = make_uint2(static_cast<uint32_t>(bucket_id), static_cast<uint32_t>(pos));
}
slot++;
}
}
}
}
}
}
if constexpr (D == 64) {
cp_async_wait<0>();
__syncthreads();
} else {
if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) {
__syncthreads();
}
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
if (r < M) {
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(s0 + r) * D + c * 8]);
} else {
sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
cp_async_wait<0>();
__syncthreads();
}
// -----------------------------------------------------------------------
// PHASE 3: P_curr @ V_qb + [qb - 1] Attention + Bank-Conflict-Free Store
// -----------------------------------------------------------------------
float o_acc[D_TILES][4];
#pragma unroll
for (int dt = 0; dt < D_TILES; ++dt) {
o_acc[dt][0] = o_acc[dt][1] = o_acc[dt][2] = o_acc[dt][3] = 0.f;
}
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
#pragma unroll
for (int dp = 0; dp < D / 16; ++dp) {
int r = ks * 16 + (lane & 15);
int c = dp * 2 + (lane >> 4);
int cs = c ^ (r & 7);
uint32_t addr = smem_u32(&sBuf->kv[r * CHUNKS + cs]);
uint32_t v00, v01, v10, v11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3])
: "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
"r"(v00), "r"(v01),
"f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3])
: "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
"r"(v10), "r"(v11),
"f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3]));
}
}
if (qb >= 1) {
__syncthreads();
int sp0 = (qb - 1) * BLOCK_SIZE;
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(sp0 + r) * D + c * 8]);
}
cp_async_commit();
cp_async_wait<0>();
__syncthreads();
float sp_acc[8][4];
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
sp_acc[nt][0] = sp_acc[nt][1] = sp_acc[nt][2] = sp_acc[nt][3] = 0.f;
}
#pragma unroll
for (int ks = 0; ks < K_STEPS; ++ks) {
uint32_t q0, q1, q2, q3;
{
int rq = warp * 16 + (lane & 15);
int cq = ks * 2 + (lane >> 4);
uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]);
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3)
: "r"(q_addr));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
int c = ks * 2 + ((lane >> 3) & 1);
int cs = c ^ (n_row & 7);
uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]);
uint32_t b00, b01, b10, b11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(sp_acc[np * 2][0]), "=f"(sp_acc[np * 2][1]), "=f"(sp_acc[np * 2][2]), "=f"(sp_acc[np * 2][3])
: "r"(q0), "r"(q1), "r"(q2), "r"(q3),
"r"(b00), "r"(b01),
"f"(sp_acc[np * 2][0]), "f"(sp_acc[np * 2][1]), "f"(sp_acc[np * 2][2]), "f"(sp_acc[np * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(sp_acc[np * 2 + 1][0]), "=f"(sp_acc[np * 2 + 1][1]), "=f"(sp_acc[np * 2 + 1][2]), "=f"(sp_acc[np * 2 + 1][3])
: "r"(q0), "r"(q1), "r"(q2), "r"(q3),
"r"(b10), "r"(b11),
"f"(sp_acc[np * 2 + 1][0]), "f"(sp_acc[np * 2 + 1][1]), "f"(sp_acc[np * 2 + 1][2]), "f"(sp_acc[np * 2 + 1][3]));
}
}
__syncthreads();
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(sp0 + r) * D + c * 8]);
}
cp_async_commit();
float mp0 = m0, mp1 = m1;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
int col0 = nt * 8 + t_in_g * 2;
int col1 = col0 + 1;
float v00 = (prev_in_top0 || col0 > r0) ? (sp_acc[nt][0] * scale) : -1e30f;
float v01 = (prev_in_top0 || col1 > r0) ? (sp_acc[nt][1] * scale) : -1e30f;
float v10 = (prev_in_top1 || col0 > r1) ? (sp_acc[nt][2] * scale) : -1e30f;
float v11 = (prev_in_top1 || col1 > r1) ? (sp_acc[nt][3] * scale) : -1e30f;
sp_acc[nt][0] = v00; sp_acc[nt][1] = v01;
sp_acc[nt][2] = v10; sp_acc[nt][3] = v11;
mp0 = fmaxf(mp0, fmaxf(v00, v01));
mp1 = fmaxf(mp1, fmaxf(v10, v11));
}
mp0 = fmaxf(mp0, __shfl_xor_sync(0xffffffff, mp0, 1));
mp0 = fmaxf(mp0, __shfl_xor_sync(0xffffffff, mp0, 2));
mp1 = fmaxf(mp1, __shfl_xor_sync(0xffffffff, mp1, 1));
mp1 = fmaxf(mp1, __shfl_xor_sync(0xffffffff, mp1, 2));
float alpha0 = __expf(m0 - mp0);
float alpha1 = __expf(m1 - mp1);
m0 = mp0;
m1 = mp1;
#pragma unroll
for (int dt = 0; dt < D_TILES; ++dt) {
o_acc[dt][0] *= alpha0; o_acc[dt][1] *= alpha0;
o_acc[dt][2] *= alpha1; o_acc[dt][3] *= alpha1;
}
float lp0 = 0.f, lp1 = 0.f;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
sp_acc[nt][0] = __expf(sp_acc[nt][0] - m0);
sp_acc[nt][1] = __expf(sp_acc[nt][1] - m0);
sp_acc[nt][2] = __expf(sp_acc[nt][2] - m1);
sp_acc[nt][3] = __expf(sp_acc[nt][3] - m1);
lp0 += sp_acc[nt][0] + sp_acc[nt][1];
lp1 += sp_acc[nt][2] + sp_acc[nt][3];
}
lp0 += __shfl_xor_sync(0xffffffff, lp0, 1);
lp0 += __shfl_xor_sync(0xffffffff, lp0, 2);
lp1 += __shfl_xor_sync(0xffffffff, lp1, 1);
lp1 += __shfl_xor_sync(0xffffffff, lp1, 2);
l0 = l0 * alpha0 + lp0;
l1 = l1 * alpha1 + lp1;
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
__nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2][0], sp_acc[ks * 2][1]));
__nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2][2], sp_acc[ks * 2][3]));
__nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2 + 1][0], sp_acc[ks * 2 + 1][1]));
__nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2 + 1][2], sp_acc[ks * 2 + 1][3]));
p_frag[ks][0] = *reinterpret_cast<uint32_t*>(&v0);
p_frag[ks][1] = *reinterpret_cast<uint32_t*>(&v1);
p_frag[ks][2] = *reinterpret_cast<uint32_t*>(&v2);
p_frag[ks][3] = *reinterpret_cast<uint32_t*>(&v3);
}
cp_async_wait<0>();
__syncthreads();
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
#pragma unroll
for (int dp = 0; dp < D / 16; ++dp) {
int r = ks * 16 + (lane & 15);
int c = dp * 2 + (lane >> 4);
int cs = c ^ (r & 7);
uint32_t addr = smem_u32(&sBuf->kv[r * CHUNKS + cs]);
uint32_t v00, v01, v10, v11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3])
: "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
"r"(v00), "r"(v01),
"f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3])
: "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
"r"(v10), "r"(v11),
"f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3]));
}
}
}
float inv_l0 = 1.0f / l0;
float inv_l1 = 1.0f / l1;
__syncthreads();
#pragma unroll
for (int dt = 0; dt < D_TILES; ++dt) {
int dts0 = dt ^ (r0 & 7);
int dts1 = dt ^ (r1 & 7);
reinterpret_cast<__nv_bfloat162*>(&sQ[r0 * CHUNKS + dts0])[t_in_g] =
__float22bfloat162_rn(make_float2(o_acc[dt][0] * inv_l0, o_acc[dt][1] * inv_l0));
reinterpret_cast<__nv_bfloat162*>(&sQ[r1 * CHUNKS + dts1])[t_in_g] =
__float22bfloat162_rn(make_float2(o_acc[dt][2] * inv_l1, o_acc[dt][3] * inv_l1));
}
if (t_in_g == 0) {
if (r0 < M) {
size_t gt0 = static_cast<size_t>(bh) * S + s0 + r0;
remote_cnt[gt0] = static_cast<uint8_t>(cnt0);
if (cnt0 > 0) {
ml_all[gt0 * 16 + 0] = make_float2(m0, l0);
}
}
if (r1 < M) {
size_t gt1 = static_cast<size_t>(bh) * S + s0 + r1;
remote_cnt[gt1] = static_cast<uint8_t>(cnt1);
if (cnt1 > 0) {
ml_all[gt1 * 16 + 0] = make_float2(m1, l1);
}
}
}
__syncthreads();
__nv_bfloat16* o_blk = O + (static_cast<size_t>(bh) * S + s0) * D;
for (int i = tid; i < M * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
reinterpret_cast<uint4*>(o_blk)[i] = sQ[r * CHUNKS + (c ^ (r & 7))];
}
}
// ---------------------------------------------------------------------------
// Kernel 3: Warp-Asynchronous High-Occupancy Persistent Remote Attention
// ---------------------------------------------------------------------------
template <int D>
__global__ __launch_bounds__(128, (D == 64 ? 4 : 2)) void remote_kv_attn_kernel(
const __nv_bfloat16* __restrict__ Q,
const __nv_bfloat16* __restrict__ K,
const __nv_bfloat16* __restrict__ V,
__nv_bfloat16* __restrict__ partial_O,
float2* __restrict__ ml_all,
const int* __restrict__ bucket_counts,
const uint32_t* __restrict__ bucket_items,
const uint2* __restrict__ g_work_queue,
uint32_t* __restrict__ g_counters,
int S,
int n_blocks,
int S_stride,
float scale
) {
constexpr int CHUNKS = D / 8;
constexpr int K_STEPS = D / 16;
constexpr int D_TILES = D / 8;
constexpr int CHUNK_Q = (D == 64) ? 128 : 256;
extern __shared__ __align__(128) char smem_raw[];
uint4* sK = reinterpret_cast<uint4*>(smem_raw);
uint4* sV = sK + 64 * CHUNKS;
uint4* sQ = sV + 64 * CHUNKS;
int tid = threadIdx.x;
int warp = tid >> 5;
int lane = tid & 31;
int groupID = lane >> 2;
int t_in_g = lane & 3;
uint32_t total_w = __ldg(&g_counters[0]);
if (blockIdx.x >= total_w) return;
bool first_iter = true;
while (true) {
uint32_t* s_meta = reinterpret_cast<uint32_t*>(sQ);
if (first_iter) {
first_iter = false;
if (tid == 0) {
uint2 w = __ldg(&g_work_queue[blockIdx.x]);
s_meta[1] = w.x;
s_meta[2] = w.y;
s_meta[3] = static_cast<uint32_t>(__ldg(&bucket_counts[w.x * BCOUNT_STRIDE]));
}
__syncthreads();
} else {
if (tid == 0) {
uint32_t wid = gridDim.x + atomicAdd(&g_counters[1], 1u);
if (wid < total_w) {
s_meta[0] = wid;
uint2 w = __ldg(&g_work_queue[wid]);
s_meta[1] = w.x;
s_meta[2] = w.y;
s_meta[3] = static_cast<uint32_t>(__ldg(&bucket_counts[w.x * BCOUNT_STRIDE]));
} else {
s_meta[0] = 0xffffffffu;
}
}
__syncthreads();
if (s_meta[0] == 0xffffffffu) break;
}
int bucket_id = static_cast<int>(s_meta[1]);
int chunk_start = static_cast<int>(s_meta[2]);
int s_bucket_total = static_cast<int>(s_meta[3]);
int chunk_len = min(CHUNK_Q, s_bucket_total - chunk_start);
int bh = bucket_id / n_blocks;
int bi = bucket_id % n_blocks;
int k_s0 = bi * BLOCK_SIZE;
const __nv_bfloat16* q_head = Q + static_cast<size_t>(bh) * S * D;
const __nv_bfloat16* k_blk = K + (static_cast<size_t>(bh) * S + k_s0) * D;
const __nv_bfloat16* v_blk = V + (static_cast<size_t>(bh) * S + k_s0) * D;
const uint32_t* b_items = bucket_items + static_cast<size_t>(bucket_id) * S_stride + chunk_start;
int r_load = lane & 15;
int q_base0 = warp * 16;
uint32_t my_pk16 = (q_base0 + r_load < chunk_len) ? __ldg(&b_items[q_base0 + r_load]) : 0u;
for (int i = tid; i < 64 * CHUNKS; i += 128) {
int r = i / CHUNKS;
int c = i % CHUNKS;
int cs = c ^ (r & 7);
cp_async_16B(smem_u32(&sK[r * CHUNKS + cs]), &k_blk[r * D + c * 8]);
cp_async_16B(smem_u32(&sV[r * CHUNKS + cs]), &v_blk[r * D + c * 8]);
}
cp_async_commit();
cp_async_wait<0>();
__syncthreads();
uint4* sQ_w = sQ + warp * (16 * CHUNKS);
for (int q_base = q_base0; q_base < chunk_len; q_base += 64) {
int w_m = min(16, chunk_len - q_base);
if (q_base > q_base0) {
my_pk16 = (r_load < w_m) ? __ldg(&b_items[q_base + r_load]) : 0u;
}
#pragma unroll
for (int it = 0; it < (16 * CHUNKS) / 32; ++it) {
int idx = it * 32 + lane;
int r = idx / CHUNKS;
int c = idx % CHUNKS;
int cs = c ^ (r & 7);
uint32_t pk_r = __shfl_sync(0xffffffff, my_pk16, r);
if (r < w_m) {
uint32_t t_idx = pk_r >> 4;
cp_async_16B(smem_u32(&sQ_w[r * CHUNKS + cs]), &q_head[t_idx * D + c * 8]);
} else {
sQ_w[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
}
}
cp_async_commit();
cp_async_wait<0>();
__syncwarp();
uint32_t q_frag[K_STEPS][4];
#pragma unroll
for (int ks = 0; ks < K_STEPS; ++ks) {
int r = lane & 15;
int c = ks * 2 + (lane >> 4);
int cs = c ^ (r & 7);
uint32_t addr = smem_u32(&sQ_w[r * CHUNKS + cs]);
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(q_frag[ks][0]), "=r"(q_frag[ks][1]), "=r"(q_frag[ks][2]), "=r"(q_frag[ks][3])
: "r"(addr));
}
float s_acc[8][4];
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
s_acc[nt][0] = s_acc[nt][1] = s_acc[nt][2] = s_acc[nt][3] = 0.f;
}
#pragma unroll
for (int ks = 0; ks < K_STEPS; ++ks) {
#pragma unroll
for (int np = 0; np < 4; ++np) {
int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
int c = ks * 2 + ((lane >> 3) & 1);
int cs = c ^ (n_row & 7);
uint32_t addr = smem_u32(&sK[n_row * CHUNKS + cs]);
uint32_t b00, b01, b10, b11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(s_acc[np * 2][0]), "=f"(s_acc[np * 2][1]), "=f"(s_acc[np * 2][2]), "=f"(s_acc[np * 2][3])
: "r"(q_frag[ks][0]), "r"(q_frag[ks][1]), "r"(q_frag[ks][2]), "r"(q_frag[ks][3]),
"r"(b00), "r"(b01),
"f"(s_acc[np * 2][0]), "f"(s_acc[np * 2][1]), "f"(s_acc[np * 2][2]), "f"(s_acc[np * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(s_acc[np * 2 + 1][0]), "=f"(s_acc[np * 2 + 1][1]), "=f"(s_acc[np * 2 + 1][2]), "=f"(s_acc[np * 2 + 1][3])
: "r"(q_frag[ks][0]), "r"(q_frag[ks][1]), "r"(q_frag[ks][2]), "r"(q_frag[ks][3]),
"r"(b10), "r"(b11),
"f"(s_acc[np * 2 + 1][0]), "f"(s_acc[np * 2 + 1][1]), "f"(s_acc[np * 2 + 1][2]), "f"(s_acc[np * 2 + 1][3]));
}
}
float m0 = -1e30f, m1 = -1e30f;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
s_acc[nt][0] *= scale; s_acc[nt][1] *= scale;
s_acc[nt][2] *= scale; s_acc[nt][3] *= scale;
m0 = fmaxf(m0, fmaxf(s_acc[nt][0], s_acc[nt][1]));
m1 = fmaxf(m1, fmaxf(s_acc[nt][2], s_acc[nt][3]));
}
m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 1));
m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 2));
m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 1));
m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 2));
float l0 = 0.f, l1 = 0.f;
#pragma unroll
for (int nt = 0; nt < 8; ++nt) {
s_acc[nt][0] = __expf(s_acc[nt][0] - m0);
s_acc[nt][1] = __expf(s_acc[nt][1] - m0);
s_acc[nt][2] = __expf(s_acc[nt][2] - m1);
s_acc[nt][3] = __expf(s_acc[nt][3] - m1);
l0 += s_acc[nt][0] + s_acc[nt][1];
l1 += s_acc[nt][2] + s_acc[nt][3];
}
l0 += __shfl_xor_sync(0xffffffff, l0, 1);
l0 += __shfl_xor_sync(0xffffffff, l0, 2);
l1 += __shfl_xor_sync(0xffffffff, l1, 1);
l1 += __shfl_xor_sync(0xffffffff, l1, 2);
uint32_t p_frag[4][4];
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
__nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][0], s_acc[ks * 2][1]));
__nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][2], s_acc[ks * 2][3]));
__nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][0], s_acc[ks * 2 + 1][1]));
__nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][2], s_acc[ks * 2 + 1][3]));
p_frag[ks][0] = *reinterpret_cast<uint32_t*>(&v0);
p_frag[ks][1] = *reinterpret_cast<uint32_t*>(&v1);
p_frag[ks][2] = *reinterpret_cast<uint32_t*>(&v2);
p_frag[ks][3] = *reinterpret_cast<uint32_t*>(&v3);
}
float o_acc[D_TILES][4];
#pragma unroll
for (int dt = 0; dt < D_TILES; ++dt) {
o_acc[dt][0] = o_acc[dt][1] = o_acc[dt][2] = o_acc[dt][3] = 0.f;
}
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
#pragma unroll
for (int dp = 0; dp < D / 16; ++dp) {
int r = ks * 16 + (lane & 15);
int c = dp * 2 + (lane >> 4);
int cs = c ^ (r & 7);
uint32_t addr = smem_u32(&sV[r * CHUNKS + cs]);
uint32_t v00, v01, v10, v11;
asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11)
: "r"(addr));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3])
: "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
"r"(v00), "r"(v01),
"f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3]));
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
: "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3])
: "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
"r"(v10), "r"(v11),
"f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3]));
}
}
__syncwarp();
float inv_l0 = 1.0f / l0;
float inv_l1 = 1.0f / l1;
int r0_w = groupID;
int r1_w = groupID + 8;
#pragma unroll
for (int dt = 0; dt < D_TILES; ++dt) {
int dts0 = dt ^ r0_w;
int dts1 = dt ^ (r1_w & 7);
reinterpret_cast<__nv_bfloat162*>(&sQ_w[r0_w * CHUNKS + dts0])[t_in_g] =
__float22bfloat162_rn(make_float2(o_acc[dt][0] * inv_l0, o_acc[dt][1] * inv_l0));
reinterpret_cast<__nv_bfloat162*>(&sQ_w[r1_w * CHUNKS + dts1])[t_in_g] =
__float22bfloat162_rn(make_float2(o_acc[dt][2] * inv_l1, o_acc[dt][3] * inv_l1));
}
uint32_t pk0 = __shfl_sync(0xffffffff, my_pk16, r0_w);
uint32_t pk1 = __shfl_sync(0xffffffff, my_pk16, r1_w);
if (t_in_g == 0) {
if (r0_w < w_m) {
size_t gt = static_cast<size_t>(bh) * S + (pk0 >> 4);
uint32_t slot = pk0 & 15u;
ml_all[gt * 16 + 1 + slot] = make_float2(m0, l0);
}
if (r1_w < w_m) {
size_t gt = static_cast<size_t>(bh) * S + (pk1 >> 4);
uint32_t slot = pk1 & 15u;
ml_all[gt * 16 + 1 + slot] = make_float2(m1, l1);
}
}
__syncwarp();
constexpr int ROWS_PER_STEP = 32 / CHUNKS;
int sub_r = lane / CHUNKS;
int sub_c = lane & (CHUNKS - 1);
#pragma unroll
for (int step = 0; step < 16; step += ROWS_PER_STEP) {
int rl = step + sub_r;
uint32_t pk = __shfl_sync(0xffffffff, my_pk16, rl);
if (rl < w_m) {
size_t gt = static_cast<size_t>(bh) * S + (pk >> 4);
uint32_t slot = pk & 15u;
uint4 val = sQ_w[rl * CHUNKS + (sub_c ^ (rl & 7))];
reinterpret_cast<uint4*>(partial_O + (gt * 8 + slot) * D)[sub_c] = val;
}
}
__syncwarp();
}
__syncthreads();
}
}
// ---------------------------------------------------------------------------
// Kernel 4: Unified High-Occupancy Combine Epilogue for t >= 128
// ---------------------------------------------------------------------------
template <int D>
__global__ __launch_bounds__(256) void combine_epilogue_kernel(
const __nv_bfloat16* __restrict__ partial_O,
const float2* __restrict__ ml_all,
const uint8_t* __restrict__ remote_cnt,
__nv_bfloat16* __restrict__ O,
int S
) {
constexpr int WIDTH = D / 8;
constexpr int Q_PER_CTA = 256 / WIDTH;
int tid = threadIdx.x;
int q_local = tid / WIDTH;
int sub_lane = tid & (WIDTH - 1);
int t = 128 + blockIdx.x * Q_PER_CTA + q_local;
if (t >= S) return;
int bh = blockIdx.y;
size_t gt = static_cast<size_t>(bh) * S + t;
int cnt = static_cast<int>(__ldg(&remote_cnt[gt]));
if (cnt == 0) return;
int total_parts = cnt + 1;
const float2* ml_ptr = ml_all + gt * 16;
float m_arr[9];
float w_arr[9];
float m_max = -1e30f;
#pragma unroll
for (int k = 0; k < 9; ++k) {
if (k < total_parts) {
float2 mv = __ldg(&ml_ptr[k]);
m_arr[k] = mv.x;
w_arr[k] = mv.y;
m_max = fmaxf(m_max, mv.x);
} else {
m_arr[k] = -1e30f;
w_arr[k] = 0.f;
}
}
float w_sum = 0.f;
#pragma unroll
for (int k = 0; k < 9; ++k) {
float w = (k < total_parts) ? (w_arr[k] * __expf(m_arr[k] - m_max)) : 0.f;
w_arr[k] = w;
w_sum += w;
}
float inv_w_sum = 1.0f / w_sum;
float acc[8];
uint4 u_loc = reinterpret_cast<const uint4*>(O + gt * D)[sub_lane];
unpack_bf16x8(u_loc, acc);
float w0 = w_arr[0] * inv_w_sum;
#pragma unroll
for (int j = 0; j < 8; ++j) {
acc[j] *= w0;
}
const uint4* po_u4 = reinterpret_cast<const uint4*>(partial_O + (gt * 8) * D) + sub_lane;
#pragma unroll
for (int slot = 0; slot < 8; ++slot) {
if (slot < cnt) {
float wk = w_arr[slot + 1] * inv_w_sum;
uint4 u = __ldg(&po_u4[slot * WIDTH]);
float vals[8];
unpack_bf16x8(u, vals);
#pragma unroll
for (int j = 0; j < 8; ++j) {
acc[j] = fmaf(wk, vals[j], acc[j]);
}
}
}
reinterpret_cast<uint4*>(O + gt * D)[sub_lane] = pack_bf16x8(acc);
}
// ---------------------------------------------------------------------------
// Cached Workspace Manager
// ---------------------------------------------------------------------------
struct Workspace {
void* ptr = nullptr;
size_t cap = 0;
void* ensure(size_t needed) {
if (needed > cap) {
if (ptr) cudaFree(ptr);
size_t alloc_sz = ((needed + (1 << 20) - 1) & ~((size_t)(1 << 20) - 1));
TORCH_CHECK(cudaMalloc(&ptr, alloc_sz) == cudaSuccess, "NSA workspace cudaMalloc failed");
cap = alloc_sz;
}
return ptr;
}
};
static Workspace g_ws[8];
inline size_t align256(size_t x) {
return (x + 255) & ~((size_t)255);
}
template <int D>
void launch_nsa_subbatch(
const __nv_bfloat16* q_ptr,
const __nv_bfloat16* k_ptr,
const __nv_bfloat16* v_ptr,
torch::Tensor& out,
c10::IntArrayRef out_sizes,
c10::Device out_dev,
size_t head_off,
int BH,
int S,
int dev,
cudaStream_t stream
) {
constexpr int CHUNK_Q = (D == 64) ? 128 : MAX_CHUNK_Q;
int n_blocks = (S + BLOCK_SIZE - 1) / BLOCK_SIZE;
int S_stride = (S + 31) & ~31;
int chunks_per_s = (S + CHUNK_Q - 1) / CHUNK_Q;
size_t sz_kmean_f = align256(static_cast<size_t>(BH) * n_blocks * D * sizeof(float));
size_t sz_kmean_bf = (D == 128) ? align256(static_cast<size_t>(BH) * n_blocks * D * sizeof(__nv_bfloat16)) : 0;
size_t sz_kmean_er = (D == 128) ? align256(static_cast<size_t>(BH) * n_blocks * sizeof(float)) : 0;
size_t sz_bcounts = align256(static_cast<size_t>(BH) * n_blocks * BCOUNT_STRIDE * sizeof(int));
size_t sz_cntrs = align256(2 * sizeof(uint32_t));
size_t sz_bitems = align256(static_cast<size_t>(BH) * n_blocks * S_stride * sizeof(uint32_t));
size_t sz_wqueue = align256(static_cast<size_t>(BH) * n_blocks * chunks_per_s * sizeof(uint2));
size_t sz_po = align256(static_cast<size_t>(BH) * S * 8 * D * sizeof(__nv_bfloat16));
size_t sz_ml = align256(static_cast<size_t>(BH) * S * 16 * sizeof(float2));
size_t sz_rcnt = align256(static_cast<size_t>(BH) * S * sizeof(uint8_t));
size_t total_bytes = sz_kmean_f + sz_kmean_bf + sz_kmean_er + sz_bcounts + sz_cntrs + sz_bitems + sz_wqueue + sz_po + sz_ml + sz_rcnt;
char* base = static_cast<char*>(g_ws[dev].ensure(total_bytes));
float* k_mean_f = reinterpret_cast<float*>(base); base += sz_kmean_f;
__nv_bfloat16* k_mean_b = (D == 128) ? reinterpret_cast<__nv_bfloat16*>(base) : nullptr; base += sz_kmean_bf;
float* k_mean_err = (D == 128) ? reinterpret_cast<float*>(base) : nullptr; base += sz_kmean_er;
int* bucket_counts = reinterpret_cast<int*>(base); base += sz_bcounts;
uint32_t* g_counters = reinterpret_cast<uint32_t*>(base); base += sz_cntrs;
uint32_t* bucket_items = reinterpret_cast<uint32_t*>(base); base += sz_bitems;
uint2* g_work_queue = reinterpret_cast<uint2*>(base); base += sz_wqueue;
__nv_bfloat16* part_O = reinterpret_cast<__nv_bfloat16*>(base); base += sz_po;
float2* ml_all = reinterpret_cast<float2*>(base); base += sz_ml;
uint8_t* remote_cnt = reinterpret_cast<uint8_t*>(base);
float scale = static_cast<float>(1.0 / std::sqrt(static_cast<double>(D)));
dim3 grid_blk(n_blocks, BH);
k_mean_and_init_kernel<D><<<grid_blk, 256, 0, stream>>>(
k_ptr, k_mean_f, k_mean_b, k_mean_err, bucket_counts, g_counters, S, n_blocks
);
if (!out.defined()) {
out = torch::Tensor(at::detail::empty_cuda(
out_sizes, at::ScalarType::BFloat16, out_dev, std::nullopt
));
}
__nv_bfloat16* o_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()) + head_off;
constexpr int CHUNKS = D / 8;
constexpr int MIN_BLK_LOCAL = (D == 64) ? 4 : 3;
constexpr size_t SMEM_LOCAL = 64 * CHUNKS * sizeof(uint4) + sizeof(LocalSharedBuf<D>)
+ ((D == 64) ? (2 * 256 * sizeof(int)) : 0);
constexpr size_t SMEM_REM = 3 * 64 * CHUNKS * sizeof(uint4);
static bool attr_set[8] = {false};
if (!attr_set[dev]) {
cudaFuncSetAttribute(
local_attn_score_select_kernel<D, MIN_BLK_LOCAL>,
cudaFuncAttributePreferredSharedMemoryCarveout,
cudaSharedmemCarveoutMaxShared
);
cudaFuncSetAttribute(
local_attn_score_select_kernel<D, MIN_BLK_LOCAL>,
cudaFuncAttributeMaxDynamicSharedMemorySize,
static_cast<int>(SMEM_LOCAL)
);
cudaFuncSetAttribute(
remote_kv_attn_kernel<D>,
cudaFuncAttributePreferredSharedMemoryCarveout,
cudaSharedmemCarveoutMaxShared
);
cudaFuncSetAttribute(
remote_kv_attn_kernel<D>,
cudaFuncAttributeMaxDynamicSharedMemorySize,
static_cast<int>(SMEM_REM)
);
attr_set[dev] = true;
}
local_attn_score_select_kernel<D, MIN_BLK_LOCAL><<<grid_blk, 128, SMEM_LOCAL, stream>>>(
q_ptr, k_ptr, v_ptr, k_mean_f, k_mean_b, k_mean_err, o_ptr, ml_all, remote_cnt,
bucket_counts, bucket_items, g_work_queue, g_counters,
S, n_blocks, S_stride, scale
);
if (S > 128) {
constexpr int PERSISTENT_CTAS = (D == 64) ? (188 * 4) : (188 * 2);
remote_kv_attn_kernel<D><<<PERSISTENT_CTAS, 128, SMEM_REM, stream>>>(
q_ptr, k_ptr, v_ptr, part_O, ml_all, bucket_counts, bucket_items,
g_work_queue, g_counters, S, n_blocks, S_stride, scale
);
constexpr int Q_PER_CTA = 256 / (D / 8);
int active_s = S - 128;
dim3 grid_epi((active_s + Q_PER_CTA - 1) / Q_PER_CTA, BH);
combine_epilogue_kernel<D><<<grid_epi, 256, 0, stream>>>(
part_O, ml_all, remote_cnt, o_ptr, S
);
}
}
template <int D>
void launch_nsa_l2_tiled(
const __nv_bfloat16* q_ptr,
const __nv_bfloat16* k_ptr,
const __nv_bfloat16* v_ptr,
torch::Tensor& out,
c10::IntArrayRef out_sizes,
c10::Device out_dev,
int BH,
int S,
int dev,
cudaStream_t stream
) {
size_t bytes_per_head = static_cast<size_t>(S) * D * 12 * sizeof(__nv_bfloat16);
int max_bh_l2 = static_cast<int>((112ULL * 1024 * 1024) / std::max<size_t>(1ULL, bytes_per_head));
int bh_step = (max_bh_l2 >= BH) ? BH : std::max(1, (BH + 1) / 2);
size_t head_stride = static_cast<size_t>(S) * D;
for (int bh0 = 0; bh0 < BH; bh0 += bh_step) {
int cur_bh = std::min(bh_step, BH - bh0);
size_t off = static_cast<size_t>(bh0) * head_stride;
launch_nsa_subbatch<D>(
q_ptr + off, k_ptr + off, v_ptr + off,
out, out_sizes, out_dev, off,
cur_bh, S, dev, stream
);
}
}
} // namespace
torch::Tensor nsa_forward_cuda(
const torch::Tensor& q_in,
const torch::Tensor& k_in,
const torch::Tensor& v_in
) {
TORCH_CHECK(q_in.is_cuda() && k_in.is_cuda() && v_in.is_cuda(), "q, k, v must be CUDA tensors");
TORCH_CHECK(q_in.scalar_type() == torch::kBFloat16, "q must be bfloat16");
torch::Tensor q_tmp, k_tmp, v_tmp;
const torch::Tensor& q = q_in.is_contiguous() ? q_in : (q_tmp = q_in.contiguous());
const torch::Tensor& k = k_in.is_contiguous() ? k_in : (k_tmp = k_in.contiguous());
const torch::Tensor& v = v_in.is_contiguous() ? v_in : (v_tmp = v_in.contiguous());
int B = static_cast<int>(q.size(0));
int H = static_cast<int>(q.size(1));
int S = static_cast<int>(q.size(2));
int D = static_cast<int>(q.size(3));
int BH = B * H;
int dev = q.get_device();
cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);
const __nv_bfloat16* q_ptr = reinterpret_cast<const __nv_bfloat16*>(q.data_ptr());
const __nv_bfloat16* k_ptr = reinterpret_cast<const __nv_bfloat16*>(k.data_ptr());
const __nv_bfloat16* v_ptr = reinterpret_cast<const __nv_bfloat16*>(v.data_ptr());
torch::Tensor out;
if (D == 64) {
launch_nsa_l2_tiled<64>(q_ptr, k_ptr, v_ptr, out, q.sizes(), q.device(), BH, S, dev, stream);
} else if (D == 128) {
launch_nsa_l2_tiled<128>(q_ptr, k_ptr, v_ptr, out, q.sizes(), q.device(), BH, S, dev, stream);
} else {
TORCH_CHECK(false, "Unsupported head dim D: ", D);
}
return out;
}
"""
_CPP_SOURCE = "torch::Tensor nsa_forward_cuda(const torch::Tensor& q, const torch::Tensor& k, const torch::Tensor& v);"
_ext = load_inline(
name="deepseek_nsa_sm120_v10",
cpp_sources=_CPP_SOURCE,
cuda_sources=_CUDA_SOURCE,
functions=["nsa_forward_cuda"],
extra_cuda_cflags=["-O3"],
)
class Model(nn.Module):
def __init__(self, B: int, H: int, S: int, D: int):
super().__init__()
self.B, self.H, self.S, self.D = B, H, S, D
self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16))
def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
return _ext.nsa_forward_cuda(q, k, v)
20261110_113716_agy_gemini-4-carbon-high_02_deepseek_nsa