KernelBench cuda · RTX PRO 6000

DeepSeek NSA Gemini 4 Carbon (High)

0.088 msgeomean latency across six shapes · lower is better

manually audited: clean

Gemini 4 Carbon wrote NSA block selection and sparse attention as four raw CUDA launches with inline-PTX bf16 mma.sync. A key-mean kernel stores fp32 block means of K; a per-query-block kernel scores every earlier block as q·mean(k) (exact fp32 FMA for D=64; for D=128 a bf16 tensor-core prefilter to 16 candidates, then fp32 rescoring wherever a Cauchy-Schwarz error bound cannot decide membership), takes the causal mean of the diagonal block from its tensor-core scores, keeps a top-8 with ties to the larger block index, attends the diagonal block and the previous block (whole if selected, else only the sliding-window tail), and scatters each remote selection into a per-key-block bucket. A persistent key-stationary kernel loads each selected K/V block once and attends all of its bucketed queries, and an epilogue merges the bf16 partials by log-sum-exp.

harnessagyagent session3h 35mtotal wall3h 41mcheck5mbenchmark2soutput tokens—gpu-lock wait1h 34mgpu-lock held26mregimecompute

Per-shape latencymilliseconds are the measured result; the dense-equivalent roofline is not a useful ceiling

1×16×2048×640.055 ms
1×16×4127×640.101 ms
1×8×8192×640.111 ms
1×8×8191×1280.208 ms
4×8×1024×640.048 ms
2×8×3000×640.075 ms

geomean(0.055 ms · 0.101 ms · 0.111 ms · 0.208 ms · 0.048 ms · 0.075 ms) = 0.088 ms

Kernel source (redacted)
from __future__ import annotations

import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

os.environ.setdefault("TORCH_CUDA_ARCH_LIST", "12.0a")

_CUDA_SOURCE = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <ATen/cuda/EmptyTensor.h>
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
#include <cmath>
#include <algorithm>

namespace {

constexpr int BLOCK_SIZE = 64;
constexpr int TOP_N_BLOCKS = 8;
constexpr int MAX_CHUNK_Q = 256;
constexpr int BCOUNT_STRIDE = 32; // 32 ints = 128B cache-line isolation per bucket

__device__ __forceinline__ uint32_t smem_u32(const void* ptr) {
    return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}

__device__ __forceinline__ void cp_async_16B(uint32_t dst_smem, const void* src_gmem) {
    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" :: "r"(dst_smem), "l"(src_gmem));
}

__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n");
}

template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}

__device__ __forceinline__ bool pair_gt(float va, int ia, float vb, int ib) {
    return (va > vb) || (va == vb && ia > ib);
}

__device__ __forceinline__ void cas_pair(float& va, int& ia, float& vb, int& ib, bool desc) {
    bool swap = desc ? pair_gt(vb, ib, va, ia) : pair_gt(va, ia, vb, ib);
    if (swap) {
        float tv = va; va = vb; vb = tv;
        int ti = ia; ia = ib; ib = ti;
    }
}

__device__ __forceinline__ void bitonic_merge8_desc(float v[8], int idx[8]) {
    cas_pair(v[0], idx[0], v[4], idx[4], true);
    cas_pair(v[1], idx[1], v[5], idx[5], true);
    cas_pair(v[2], idx[2], v[6], idx[6], true);
    cas_pair(v[3], idx[3], v[7], idx[7], true);

    cas_pair(v[0], idx[0], v[2], idx[2], true);
    cas_pair(v[1], idx[1], v[3], idx[3], true);
    cas_pair(v[4], idx[4], v[6], idx[6], true);
    cas_pair(v[5], idx[5], v[7], idx[7], true);

    cas_pair(v[0], idx[0], v[1], idx[1], true);
    cas_pair(v[2], idx[2], v[3], idx[3], true);
    cas_pair(v[4], idx[4], v[5], idx[5], true);
    cas_pair(v[6], idx[6], v[7], idx[7], true);
}

__device__ __forceinline__ void bitonic_sort8_desc(float v[8], int idx[8]) {
    cas_pair(v[0], idx[0], v[1], idx[1], true);
    cas_pair(v[2], idx[2], v[3], idx[3], false);
    cas_pair(v[4], idx[4], v[5], idx[5], true);
    cas_pair(v[6], idx[6], v[7], idx[7], false);

    cas_pair(v[0], idx[0], v[2], idx[2], true);
    cas_pair(v[1], idx[1], v[3], idx[3], true);
    cas_pair(v[0], idx[0], v[1], idx[1], true);
    cas_pair(v[2], idx[2], v[3], idx[3], true);

    cas_pair(v[4], idx[4], v[6], idx[6], false);
    cas_pair(v[5], idx[5], v[7], idx[7], false);
    cas_pair(v[4], idx[4], v[5], idx[5], false);
    cas_pair(v[6], idx[6], v[7], idx[7], false);

    bitonic_merge8_desc(v, idx);
}

__device__ __forceinline__ void merge_into_top8(float top_v[8], int top_i[8], const float cand_v[8], const int cand_i[8]) {
    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        float bv = cand_v[7 - i];
        int bi = cand_i[7 - i];
        if (pair_gt(bv, bi, top_v[i], top_i[i])) {
            top_v[i] = bv;
            top_i[i] = bi;
        }
    }
    bitonic_merge8_desc(top_v, top_i);
}

__device__ __forceinline__ void insert_one_top8(float top_v[8], int top_i[8], float cv, int ci) {
    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        if (pair_gt(cv, ci, top_v[i], top_i[i])) {
            float tv = top_v[i]; top_v[i] = cv; cv = tv;
            int ti = top_i[i]; top_i[i] = ci; ci = ti;
        }
    }
}

// Merge 4 sorted-desc 8-arrays across a 4-lane quad (t_in_g = 0..3) into Global Top-16:
// Upon return, t_in_g == 0 holds Top-16[0..7] and t_in_g == 1 holds Top-16[8..15] in (lv, li).
__device__ __forceinline__ void quad_merge_top16(float lv[8], int li[8], int t_in_g) {
    bool is_even = ((t_in_g & 1) == 0);
    float pv[8];
    int pi[8];

    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 1, 4);
        pi[i] = __shfl_xor_sync(0xffffffff, li[i], 1, 4);
    }
    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        float bv = pv[7 - i];
        int bi = pi[7 - i];
        bool b_gt_a = pair_gt(bv, bi, lv[i], li[i]);
        if (is_even ? b_gt_a : !b_gt_a) {
            lv[i] = bv;
            li[i] = bi;
        }
    }
    bitonic_merge8_desc(lv, li);

    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 3, 4);
        pi[i] = __shfl_xor_sync(0xffffffff, li[i], 3, 4);
    }
    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        float bv = pv[7 - i];
        int bi = pi[7 - i];
        if (pair_gt(bv, bi, lv[i], li[i])) {
            lv[i] = bv;
            li[i] = bi;
        }
    }

    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        pv[i] = __shfl_xor_sync(0xffffffff, lv[i], 1, 4);
        pi[i] = __shfl_xor_sync(0xffffffff, li[i], 1, 4);
    }
    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        bool p_gt_l = pair_gt(pv[i], pi[i], lv[i], li[i]);
        if (is_even ? p_gt_l : !p_gt_l) {
            lv[i] = pv[i];
            li[i] = pi[i];
        }
    }
    bitonic_merge8_desc(lv, li);
}

__device__ __forceinline__ void unpack_bf16x8(uint4 u, float out[8]) {
    const __nv_bfloat162* p = reinterpret_cast<const __nv_bfloat162*>(&u);
    #pragma unroll
    for (int i = 0; i < 4; ++i) {
        float2 f = __bfloat1622float2(p[i]);
        out[i * 2 + 0] = f.x;
        out[i * 2 + 1] = f.y;
    }
}

__device__ __forceinline__ uint4 pack_bf16x8(const float in[8]) {
    uint4 u;
    __nv_bfloat162* p = reinterpret_cast<__nv_bfloat162*>(&u);
    #pragma unroll
    for (int i = 0; i < 4; ++i) {
        p[i] = __float22bfloat162_rn(make_float2(in[i * 2 + 0], in[i * 2 + 1]));
    }
    return u;
}

// ---------------------------------------------------------------------------
// Kernel 1: High-MLP 256-thread per-block FP32 (+ BF16/err on D=128) key means
// ---------------------------------------------------------------------------
template <int D>
__global__ __launch_bounds__(256) void k_mean_and_init_kernel(
    const __nv_bfloat16* __restrict__ K,
    float* __restrict__ K_mean_fp32,
    __nv_bfloat16* __restrict__ K_mean_bf16,
    float* __restrict__ K_mean_err,
    int* __restrict__ bucket_counts,
    uint32_t* __restrict__ g_counters,
    int S,
    int n_blocks
) {
    constexpr int CHUNKS = D / 8;
    constexpr int ROW_GROUPS = 256 / CHUNKS;
    constexpr int ITERS_FULL = BLOCK_SIZE / ROW_GROUPS;

    int bi = blockIdx.x;
    int bh = blockIdx.y;
    int tid = threadIdx.x;
    int warp = tid >> 5;
    int lane = tid & 31;

    if (bi == 0 && bh == 0 && tid == 0) {
        g_counters[0] = 0u;
        g_counters[1] = 0u;
    }
    if (tid == 0) {
        bucket_counts[(bh * n_blocks + bi) * BCOUNT_STRIDE] = 0;
    }

    int s0 = bi * BLOCK_SIZE;
    int valid_rows = min(BLOCK_SIZE, S - s0);
    int col_c = tid & (CHUNKS - 1);
    int row_g = tid / CHUNKS;

    float acc[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
    const __nv_bfloat16* k_base = K + (static_cast<size_t>(bh) * S + s0) * D;

    if (valid_rows == BLOCK_SIZE) {
        #pragma unroll
        for (int it = 0; it < ITERS_FULL; ++it) {
            int r = row_g + it * ROW_GROUPS;
            uint4 u = reinterpret_cast<const uint4*>(k_base + r * D)[col_c];
            float vals[8];
            unpack_bf16x8(u, vals);
            #pragma unroll
            for (int j = 0; j < 8; ++j) {
                acc[j] += vals[j];
            }
        }
    } else {
        for (int r = row_g; r < valid_rows; r += ROW_GROUPS) {
            uint4 u = reinterpret_cast<const uint4*>(k_base + r * D)[col_c];
            float vals[8];
            unpack_bf16x8(u, vals);
            #pragma unroll
            for (int j = 0; j < 8; ++j) {
                acc[j] += vals[j];
            }
        }
    }

    #pragma unroll
    for (int j = 0; j < 8; ++j) {
        acc[j] += __shfl_xor_sync(0xffffffff, acc[j], 16);
        if constexpr (CHUNKS == 8) {
            acc[j] += __shfl_xor_sync(0xffffffff, acc[j], 8);
        }
    }

    __shared__ __align__(16) float smem_red[8][D + 4];
    int d0 = col_c * 8;
    if (lane < CHUNKS) {
        reinterpret_cast<float4*>(&smem_red[warp][d0])[0] = make_float4(acc[0], acc[1], acc[2], acc[3]);
        reinterpret_cast<float4*>(&smem_red[warp][d0])[1] = make_float4(acc[4], acc[5], acc[6], acc[7]);
    }
    __syncthreads();

    if (warp == 0 && lane < CHUNKS) {
        float sum[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
        #pragma unroll
        for (int rg = 0; rg < 8; ++rg) {
            float4 v0 = reinterpret_cast<const float4*>(&smem_red[rg][d0])[0];
            float4 v1 = reinterpret_cast<const float4*>(&smem_red[rg][d0])[1];
            sum[0] += v0.x; sum[1] += v0.y; sum[2] += v0.z; sum[3] += v0.w;
            sum[4] += v1.x; sum[5] += v1.y; sum[6] += v1.z; sum[7] += v1.w;
        }
        float inv_len = 1.0f / static_cast<float>(valid_rows);
        float avg[8];
        #pragma unroll
        for (int j = 0; j < 8; ++j) {
            avg[j] = sum[j] * inv_len;
        }
        size_t out_off = static_cast<size_t>(bh * n_blocks + bi) * D + d0;
        float* out_f = K_mean_fp32 + out_off;
        reinterpret_cast<float4*>(out_f)[0] = make_float4(avg[0], avg[1], avg[2], avg[3]);
        reinterpret_cast<float4*>(out_f)[1] = make_float4(avg[4], avg[5], avg[6], avg[7]);

        if constexpr (D == 128) {
            uint4 packed_bf = pack_bf16x8(avg);
            reinterpret_cast<uint4*>(K_mean_bf16 + out_off)[0] = packed_bf;

            float avg_bf[8];
            unpack_bf16x8(packed_bf, avg_bf);
            float err_sq = 0.f, avg_sq = 0.f;
            #pragma unroll
            for (int j = 0; j < 8; ++j) {
                float diff = avg[j] - avg_bf[j];
                err_sq = fmaf(diff, diff, err_sq);
                avg_sq = fmaf(avg[j], avg[j], avg_sq);
            }
            constexpr uint32_t m_grp = 0x0000ffffu;
            err_sq += __shfl_xor_sync(m_grp, err_sq, 1);
            err_sq += __shfl_xor_sync(m_grp, err_sq, 2);
            err_sq += __shfl_xor_sync(m_grp, err_sq, 4);
            err_sq += __shfl_xor_sync(m_grp, err_sq, 8);
            avg_sq += __shfl_xor_sync(m_grp, avg_sq, 1);
            avg_sq += __shfl_xor_sync(m_grp, avg_sq, 2);
            avg_sq += __shfl_xor_sync(m_grp, avg_sq, 4);
            avg_sq += __shfl_xor_sync(m_grp, avg_sq, 8);
            if (tid == 0) {
                K_mean_err[bh * n_blocks + bi] = sqrtf(err_sq) * 1.01f + sqrtf(avg_sq) * 1e-5f;
            }
        }
    }
}

// ---------------------------------------------------------------------------
// Kernel 2: Zero-Spill Local Attention + Adaptive SMEM / BF16 Tensor-Core
//           Top-16 Prefilter with Exact Cauchy-Schwarz Fast-Skip
// ---------------------------------------------------------------------------
template <int D>
union LocalSharedBuf {
    uint4 kv[64 * (D / 8)];
    float kmean16[D][20];
    float kmean32[D][(D == 64) ? 36 : 20];
    int bcount_base[512];
};

template <int D, int MIN_BLOCKS_PER_SM>
__global__ __launch_bounds__(128, MIN_BLOCKS_PER_SM) void local_attn_score_select_kernel(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    const float* __restrict__ K_mean_fp32,
    const __nv_bfloat16* __restrict__ K_mean_bf16,
    const float* __restrict__ K_mean_err,
    __nv_bfloat16* __restrict__ O,
    float2* __restrict__ ml_all,
    uint8_t* __restrict__ remote_cnt,
    int* __restrict__ bucket_counts,
    uint32_t* __restrict__ bucket_items,
    uint2* __restrict__ g_work_queue,
    uint32_t* __restrict__ g_counters,
    int S,
    int n_blocks,
    int S_stride,
    float scale
) {
    constexpr int CHUNKS = D / 8;
    constexpr int K_STEPS = D / 16;
    constexpr int D_TILES = D / 8;
    constexpr int MAX_SMEM_BUCKETS = 256;
    constexpr int CHUNK_Q = (D == 64) ? 128 : 256;

    extern __shared__ __align__(128) char smem_local_raw[];
    uint4* sQ = reinterpret_cast<uint4*>(smem_local_raw);
    LocalSharedBuf<D>* sBuf = reinterpret_cast<LocalSharedBuf<D>*>(sQ + 64 * CHUNKS);
    int* s_bcount_sep = (D == 64) ? reinterpret_cast<int*>(sBuf + 1) : nullptr;
    int* s_bbase_sep  = (D == 64) ? (s_bcount_sep + MAX_SMEM_BUCKETS) : nullptr;

    int qb = (n_blocks - 1) - static_cast<int>(blockIdx.x);
    int bh = blockIdx.y;
    int tid = threadIdx.x;
    int warp = tid >> 5;
    int lane = tid & 31;
    int groupID = lane >> 2;
    int t_in_g = lane & 3;

    int s0 = qb * BLOCK_SIZE;
    int M = min(BLOCK_SIZE, S - s0);
    int r0 = warp * 16 + groupID;
    int r1 = r0 + 8;

    const __nv_bfloat16* q_gmem = Q + static_cast<size_t>(bh) * S * D;
    const __nv_bfloat16* k_gmem = K + static_cast<size_t>(bh) * S * D;
    const __nv_bfloat16* v_gmem = V + static_cast<size_t>(bh) * S * D;

    if constexpr (D == 64) {
        if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) {
            for (int b = tid; b < qb - 1; b += 128) {
                s_bcount_sep[b] = 0;
            }
        }
    }

    for (int i = tid; i < 64 * CHUNKS; i += 128) {
        int r = i / CHUNKS;
        int c = i % CHUNKS;
        int cs = c ^ (r & 7);
        if (r < M) {
            cp_async_16B(smem_u32(&sQ[r * CHUNKS + cs]), &q_gmem[(s0 + r) * D + c * 8]);
            if (qb == 0) {
                cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
            }
        } else {
            sQ[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
            if (qb == 0) {
                sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
            }
        }
    }
    cp_async_commit();
    cp_async_wait<0>();
    __syncthreads();

    // -----------------------------------------------------------------------
    // PHASE 1: Score [0 .. qb-1] -> Only 16 regs (my_tv, my_ti) survive!
    // -----------------------------------------------------------------------
    float my_tv[8];
    int my_ti[8];
    #pragma unroll
    for (int i = 0; i < 8; ++i) {
        my_tv[i] = -1e30f;
        my_ti[i] = -1;
    }

    if (qb >= 1) {
        if (qb <= 16) {
            // Fast Path A1 (1 <= qb <= 16): Single 16-col SMEM FP32 tile
            const float* km_fp32 = K_mean_fp32 + static_cast<size_t>(bh * n_blocks) * D;
            constexpr int F4_PER_ROW = D / 4;
            for (int idx = tid; idx < 16 * F4_PER_ROW; idx += 128) {
                int br = idx / F4_PER_ROW;
                int dc4 = idx % F4_PER_ROW;
                int d = dc4 * 4;
                float4 fv = (br < qb)
                    ? reinterpret_cast<const float4*>(km_fp32)[br * F4_PER_ROW + dc4]
                    : make_float4(0.f, 0.f, 0.f, 0.f);
                sBuf->kmean16[d + 0][br] = fv.x;
                sBuf->kmean16[d + 1][br] = fv.y;
                sBuf->kmean16[d + 2][br] = fv.z;
                sBuf->kmean16[d + 3][br] = fv.w;
            }
            __syncthreads();

            float cv0[4] = {0.f, 0.f, 0.f, 0.f};
            float cv1[4] = {0.f, 0.f, 0.f, 0.f};
            #pragma unroll 2
            for (int dc = 0; dc < CHUNKS; ++dc) {
                uint4 uq0 = sQ[r0 * CHUNKS + (dc ^ (r0 & 7))];
                uint4 uq1 = sQ[r1 * CHUNKS + (dc ^ (r1 & 7))];
                float qv0[8], qv1[8];
                unpack_bf16x8(uq0, qv0);
                unpack_bf16x8(uq1, qv1);
                #pragma unroll
                for (int j = 0; j < 8; ++j) {
                    int d = dc * 8 + j;
                    float q0v = qv0[j];
                    float q1v = qv1[j];
                    float4 km = *reinterpret_cast<const float4*>(&sBuf->kmean16[d][t_in_g * 4]);
                    cv0[0] = fmaf(q0v, km.x, cv0[0]); cv1[0] = fmaf(q1v, km.x, cv1[0]);
                    cv0[1] = fmaf(q0v, km.y, cv0[1]); cv1[1] = fmaf(q1v, km.y, cv1[1]);
                    cv0[2] = fmaf(q0v, km.z, cv0[2]); cv1[2] = fmaf(q1v, km.z, cv1[2]);
                    cv0[3] = fmaf(q0v, km.w, cv0[3]); cv1[3] = fmaf(q1v, km.w, cv1[3]);
                }
            }
            __syncthreads();

            for (int i = tid; i < 64 * CHUNKS; i += 128) {
                int r = i / CHUNKS;
                int c = i % CHUNKS;
                int cs = c ^ (r & 7);
                if (r < M) {
                    cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
                } else {
                    sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
                }
            }
            cp_async_commit();

            int ci[4];
            #pragma unroll
            for (int k = 0; k < 4; ++k) {
                int br = t_in_g * 4 + k;
                if (br < qb) {
                    cv0[k] *= scale;
                    cv1[k] *= scale;
                    ci[k] = br;
                } else {
                    cv0[k] = -1e30f;
                    cv1[k] = -1e30f;
                    ci[k] = -1;
                }
            }

            bool is_odd = ((t_in_g & 1) != 0);
            float send_v[4];
            #pragma unroll
            for (int k = 0; k < 4; ++k) send_v[k] = is_odd ? cv0[k] : cv1[k];
            #pragma unroll
            for (int k = 0; k < 4; ++k) {
                float recv_v = __shfl_xor_sync(0xffffffff, send_v[k], 1, 4);
                int recv_i   = __shfl_xor_sync(0xffffffff, ci[k], 1, 4);
                my_tv[k]     = is_odd ? recv_v : cv0[k];
                my_ti[k]     = is_odd ? recv_i : ci[k];
                my_tv[4 + k] = is_odd ? cv1[k] : recv_v;
                my_ti[4 + k] = is_odd ? ci[k]  : recv_i;
            }
            bitonic_sort8_desc(my_tv, my_ti);

            float other_v[8];
            int other_i[8];
            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                other_v[i] = __shfl_xor_sync(0xffffffff, my_tv[i], 2, 4);
                other_i[i] = __shfl_xor_sync(0xffffffff, my_ti[i], 2, 4);
            }
            merge_into_top8(my_tv, my_ti, other_v, other_i);
        } else if constexpr (D == 64) {
            // Fast Path A2 (D == 64, qb > 16): Tiled 32-col SMEM FP32 exact scoring
            // Each lane t_in_g in {0..3} owns disjoint blocks (b_tile + t_in_g*8 + 0..7),
            // so zero warp shuffles occur inside the b_tile loop!
            float lv0[8], lv1[8];
            int li0[8], li1[8];
            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                lv0[i] = -1e30f; li0[i] = -1;
                lv1[i] = -1e30f; li1[i] = -1;
            }

            const float* km_fp32 = K_mean_fp32 + static_cast<size_t>(bh * n_blocks) * D;
            constexpr int F4_PER_ROW = D / 4;

            #pragma unroll 1
            for (int b_tile = 0; b_tile < qb; b_tile += 32) {
                int b_cnt = min(32, qb - b_tile);
                for (int idx = tid; idx < 32 * F4_PER_ROW; idx += 128) {
                    int br = idx / F4_PER_ROW;
                    int dc4 = idx % F4_PER_ROW;
                    int d = dc4 * 4;
                    float4 fv = (br < b_cnt)
                        ? reinterpret_cast<const float4*>(km_fp32)[(b_tile + br) * F4_PER_ROW + dc4]
                        : make_float4(0.f, 0.f, 0.f, 0.f);
                    sBuf->kmean32[d + 0][br] = fv.x;
                    sBuf->kmean32[d + 1][br] = fv.y;
                    sBuf->kmean32[d + 2][br] = fv.z;
                    sBuf->kmean32[d + 3][br] = fv.w;
                }
                __syncthreads();

                float cv0[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
                float cv1[8] = {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f};
                #pragma unroll 2
                for (int dc = 0; dc < CHUNKS; ++dc) {
                    uint4 uq0 = sQ[r0 * CHUNKS + (dc ^ (r0 & 7))];
                    uint4 uq1 = sQ[r1 * CHUNKS + (dc ^ (r1 & 7))];
                    float qv0[8], qv1[8];
                    unpack_bf16x8(uq0, qv0);
                    unpack_bf16x8(uq1, qv1);
                    #pragma unroll
                    for (int j = 0; j < 8; ++j) {
                        int d = dc * 8 + j;
                        float q0v = qv0[j];
                        float q1v = qv1[j];
                        float4 km0 = *reinterpret_cast<const float4*>(&sBuf->kmean32[d][t_in_g * 8]);
                        float4 km1 = *reinterpret_cast<const float4*>(&sBuf->kmean32[d][t_in_g * 8 + 4]);
                        cv0[0] = fmaf(q0v, km0.x, cv0[0]); cv1[0] = fmaf(q1v, km0.x, cv1[0]);
                        cv0[1] = fmaf(q0v, km0.y, cv0[1]); cv1[1] = fmaf(q1v, km0.y, cv1[1]);
                        cv0[2] = fmaf(q0v, km0.z, cv0[2]); cv1[2] = fmaf(q1v, km0.z, cv1[2]);
                        cv0[3] = fmaf(q0v, km0.w, cv0[3]); cv1[3] = fmaf(q1v, km0.w, cv1[3]);
                        cv0[4] = fmaf(q0v, km1.x, cv0[4]); cv1[4] = fmaf(q1v, km1.x, cv1[4]);
                        cv0[5] = fmaf(q0v, km1.y, cv0[5]); cv1[5] = fmaf(q1v, km1.y, cv1[5]);
                        cv0[6] = fmaf(q0v, km1.z, cv0[6]); cv1[6] = fmaf(q1v, km1.z, cv1[6]);
                        cv0[7] = fmaf(q0v, km1.w, cv0[7]); cv1[7] = fmaf(q1v, km1.w, cv1[7]);
                    }
                }

                int ci0[8], ci1[8];
                #pragma unroll
                for (int k = 0; k < 8; ++k) {
                    int br = t_in_g * 8 + k;
                    if (br < b_cnt) {
                        int gbi = b_tile + br;
                        cv0[k] *= scale;
                        cv1[k] *= scale;
                        ci0[k] = gbi;
                        ci1[k] = gbi;
                    } else {
                        cv0[k] = -1e30f; ci0[k] = -1;
                        cv1[k] = -1e30f; ci1[k] = -1;
                    }
                }
                bitonic_sort8_desc(cv0, ci0);
                bitonic_sort8_desc(cv1, ci1);
                if (b_tile == 0) {
                    #pragma unroll
                    for (int k = 0; k < 8; ++k) {
                        lv0[k] = cv0[k]; li0[k] = ci0[k];
                        lv1[k] = cv1[k]; li1[k] = ci1[k];
                    }
                } else {
                    merge_into_top8(lv0, li0, cv0, ci0);
                    merge_into_top8(lv1, li1, cv1, ci1);
                }
                __syncthreads();
            }

            for (int i = tid; i < 64 * CHUNKS; i += 128) {
                int r = i / CHUNKS;
                int c = i % CHUNKS;
                int cs = c ^ (r & 7);
                if (r < M) {
                    cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
                } else {
                    sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
                }
            }
            cp_async_commit();

            // Simultaneous 2-step quad merge: lanes 0,1 merge r0 while lanes 2,3 merge r1!
            bool high_half = (t_in_g >= 2);
            float send_v[8];
            int send_i[8];
            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                send_v[i] = high_half ? lv0[i] : lv1[i];
                send_i[i] = high_half ? li0[i] : li1[i];
                my_tv[i]  = high_half ? lv1[i] : lv0[i];
                my_ti[i]  = high_half ? li1[i] : li0[i];
            }
            float recv_v[8];
            int recv_i[8];
            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                recv_v[i] = __shfl_xor_sync(0xffffffff, send_v[i], 2, 4);
                recv_i[i] = __shfl_xor_sync(0xffffffff, send_i[i], 2, 4);
            }
            merge_into_top8(my_tv, my_ti, recv_v, recv_i);

            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                recv_v[i] = __shfl_xor_sync(0xffffffff, my_tv[i], 1, 4);
                recv_i[i] = __shfl_xor_sync(0xffffffff, my_ti[i], 1, 4);
            }
            merge_into_top8(my_tv, my_ti, recv_v, recv_i);

            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                float r1_v = __shfl_xor_sync(0xffffffff, my_tv[i], 2, 4);
                int r1_i   = __shfl_xor_sync(0xffffffff, my_ti[i], 2, 4);
                if (t_in_g == 1) {
                    my_tv[i] = r1_v;
                    my_ti[i] = r1_i;
                }
            }
        } else {
            // Fast Path B (D == 128, qb > 16): BF16 Tensor-Core Top-16 Prefilter
            // + Per-Candidate Exact Cauchy-Schwarz Fast-Skip Rescore
            float lv0[8], lv1[8];
            int li0[8], li1[8];
            #pragma unroll
            for (int i = 0; i < 8; ++i) {
                lv0[i] = -1e30f; li0[i] = -1;
                lv1[i] = -1e30f; li1[i] = -1;
            }

            const __nv_bfloat16* km_bf16 = K_mean_bf16 + static_cast<size_t>(bh * n_blocks) * D;
            #pragma unroll 1
            for (int b_tile = 0; b_tile < qb; b_tile += 64) {
                int b_cnt = min(64, qb - b_tile);
                for (int i = tid; i < 64 * CHUNKS; i += 128) {
                    int r = i / CHUNKS;
                    int c = i % CHUNKS;
                    int cs = c ^ (r & 7);
                    if (r < b_cnt) {
                        cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &km_bf16[(b_tile + r) * D + c * 8]);
                    } else {
                        sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
                    }
                }
                cp_async_commit();
                cp_async_wait<0>();
                __syncthreads();

                float km_acc[8][4];
                #pragma unroll
                for (int nt = 0; nt < 8; ++nt) {
                    km_acc[nt][0] = km_acc[nt][1] = km_acc[nt][2] = km_acc[nt][3] = 0.f;
                }
                #pragma unroll
                for (int ks = 0; ks < K_STEPS; ++ks) {
                    uint32_t q0, q1, q2, q3;
                    {
                        int rq = warp * 16 + (lane & 15);
                        int cq = ks * 2 + (lane >> 4);
                        uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]);
                        asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                            : "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3)
                            : "r"(q_addr));
                    }
                    #pragma unroll
                    for (int np = 0; np < 4; ++np) {
                        int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
                        int c = ks * 2 + ((lane >> 3) & 1);
                        int cs = c ^ (n_row & 7);
                        uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]);
                        uint32_t b00, b01, b10, b11;
                        asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                            : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
                            : "r"(addr));
                        asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                            : "=f"(km_acc[np * 2][0]), "=f"(km_acc[np * 2][1]), "=f"(km_acc[np * 2][2]), "=f"(km_acc[np * 2][3])
                            : "r"(q0), "r"(q1), "r"(q2), "r"(q3),
                              "r"(b00), "r"(b01),
                              "f"(km_acc[np * 2][0]), "f"(km_acc[np * 2][1]), "f"(km_acc[np * 2][2]), "f"(km_acc[np * 2][3]));
                        asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                            : "=f"(km_acc[np * 2 + 1][0]), "=f"(km_acc[np * 2 + 1][1]), "=f"(km_acc[np * 2 + 1][2]), "=f"(km_acc[np * 2 + 1][3])
                            : "r"(q0), "r"(q1), "r"(q2), "r"(q3),
                              "r"(b10), "r"(b11),
                              "f"(km_acc[np * 2 + 1][0]), "f"(km_acc[np * 2 + 1][1]), "f"(km_acc[np * 2 + 1][2]), "f"(km_acc[np * 2 + 1][3]));
                    }
                }

                #pragma unroll
                for (int half = 0; half < 2; ++half) {
                    float hv[8];
                    int hi[8];
                    #pragma unroll
                    for (int nt = 0; nt < 4; ++nt) {
                        int col0 = (half * 4 + nt) * 8 + t_in_g * 2;
                        int col1 = col0 + 1;
                        hv[nt * 2 + 0] = (col0 < b_cnt) ? km_acc[half * 4 + nt][0] : -1e30f;
                        hi[nt * 2 + 0] = (col0 < b_cnt) ? (b_tile + col0) : -1;
                        hv[nt * 2 + 1] = (col1 < b_cnt) ? km_acc[half * 4 + nt][1] : -1e30f;
                        hi[nt * 2 + 1] = (col1 < b_cnt) ? (b_tile + col1) : -1;
                    }
                    bitonic_sort8_desc(hv, hi);
                    merge_into_top8(lv0, li0, hv, hi);

                    #pragma unroll
                    for (int nt = 0; nt < 4; ++nt) {
                        int col0 = (half * 4 + nt) * 8 + t_in_g * 2;
                        int col1 = col0 + 1;
                        hv[nt * 2 + 0] = (col0 < b_cnt) ? km_acc[half * 4 + nt][2] : -1e30f;
                        hi[nt * 2 + 0] = (col0 < b_cnt) ? (b_tile + col0) : -1;
                        hv[nt * 2 + 1] = (col1 < b_cnt) ? km_acc[half * 4 + nt][3] : -1e30f;
                        hi[nt * 2 + 1] = (col1 < b_cnt) ? (b_tile + col1) : -1;
                    }
                    bitonic_sort8_desc(hv, hi);
                    merge_into_top8(lv1, li1, hv, hi);
                }
                __syncthreads();
            }

            // Immediately prefetch K_qb -> sBuf->kv while running quad_merge_top16 + rescore!
            for (int i = tid; i < 64 * CHUNKS; i += 128) {
                int r = i / CHUNKS;
                int c = i % CHUNKS;
                int cs = c ^ (r & 7);
                if (r < M) {
                    cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(s0 + r) * D + c * 8]);
                } else {
                    sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
                }
            }
            cp_async_commit();

            quad_merge_top16(lv0, li0, t_in_g);
            quad_merge_top16(lv1, li1, t_in_g);

            constexpr int F16_STEPS = D / 16;
            uint2 q0_u2[F16_STEPS];
            uint2 q1_u2[F16_STEPS];
            float q0_sq = 0.f, q1_sq = 0.f;
            #pragma unroll
            for (int s = 0; s < F16_STEPS; ++s) {
                int c = s * 2 + (t_in_g >> 1);
                int sub = t_in_g & 1;
                uint2 u0 = reinterpret_cast<const uint2*>(&sQ[r0 * CHUNKS + (c ^ (r0 & 7))])[sub];
                uint2 u1 = reinterpret_cast<const uint2*>(&sQ[r1 * CHUNKS + (c ^ (r1 & 7))])[sub];
                q0_u2[s] = u0;
                q1_u2[s] = u1;
                float2 f0a = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u0.x));
                float2 f0b = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u0.y));
                q0_sq = fmaf(f0a.x, f0a.x, fmaf(f0a.y, f0a.y, fmaf(f0b.x, f0b.x, fmaf(f0b.y, f0b.y, q0_sq))));
                float2 f1a = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u1.x));
                float2 f1b = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u1.y));
                q1_sq = fmaf(f1a.x, f1a.x, fmaf(f1a.y, f1a.y, fmaf(f1b.x, f1b.x, fmaf(f1b.y, f1b.y, q1_sq))));
            }
            q0_sq += __shfl_xor_sync(0xffffffff, q0_sq, 1);
            q0_sq += __shfl_xor_sync(0xffffffff, q0_sq, 2);
            q1_sq += __shfl_xor_sync(0xffffffff, q1_sq, 1);
            q1_sq += __shfl_xor_sync(0xffffffff, q1_sq, 2);
            float max_k_err = 0.f;
            const float* k_err_base = K_mean_err + bh * n_blocks;
            for (int b = lane; b < qb; b += 32) {
                max_k_err = fmaxf(max_k_err, __ldg(&k_err_base[b]));
            }
            #pragma unroll
            for (int mask = 16; mask >= 1; mask >>= 1) {
                max_k_err = fmaxf(max_k_err, __shfl_xor_sync(0xffffffff, max_k_err, mask));
            }
            float two_delta0 = 2.0f * sqrtf(q0_sq) * max_k_err;
            float two_delta1 = 2.0f * sqrtf(q1_sq) * max_k_err;
            float v7_0 = __shfl_sync(0xffffffff, lv0[7], 0, 4);
            float v7_1 = __shfl_sync(0xffffffff, lv1[7], 0, 4);
            uint32_t quad_mask = 0xfu << (groupID * 4);

            const float* km_fp32 = K_mean_fp32 + static_cast<size_t>(bh * n_blocks) * D;
            #pragma unroll 1
            for (int step = 0; step < 16; ++step) {
                int src_lane = (step < 8) ? 0 : 1;
                int b0     = __shfl_sync(0xffffffff, li0[0], src_lane, 4);
                float hat0 = __shfl_sync(0xffffffff, lv0[0], src_lane, 4);
                int b1     = __shfl_sync(0xffffffff, li1[0], src_lane, 4);
                float hat1 = __shfl_sync(0xffffffff, lv1[0], src_lane, 4);
                if ((t_in_g == 0 && step < 8) || (t_in_g == 1 && step >= 8)) {
                    #pragma unroll
                    for (int j = 0; j < 7; ++j) {
                        li0[j] = li0[j + 1]; lv0[j] = lv0[j + 1];
                        li1[j] = li1[j + 1]; lv1[j] = lv1[j + 1];
                    }
                }

                bool sure_out0 = (b0 < 0) || (step >= 8 && (v7_0 - hat0 > two_delta0));
                bool sure_in0  = (!sure_out0) && (step < 7 && (hat0 - v7_0 > two_delta0));
                bool need0     = (!sure_out0) && (!sure_in0);

                bool sure_out1 = (b1 < 0) || (step >= 8 && (v7_1 - hat1 > two_delta1));
                bool sure_in1  = (!sure_out1) && (step < 7 && (hat1 - v7_1 > two_delta1));
                bool need1     = (!sure_out1) && (!sure_in1);

                float score0 = hat0 * scale;
                if (need0) {
                    float dot0 = 0.f;
                    const float4* km0 = reinterpret_cast<const float4*>(km_fp32 + static_cast<size_t>(b0) * D);
                    #pragma unroll
                    for (int s = 0; s < F16_STEPS; ++s) {
                        float4 kv = km0[s * 4 + t_in_g];
                        float2 q_lo = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q0_u2[s].x));
                        float2 q_hi = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q0_u2[s].y));
                        dot0 = fmaf(q_lo.x, kv.x, dot0);
                        dot0 = fmaf(q_lo.y, kv.y, dot0);
                        dot0 = fmaf(q_hi.x, kv.z, dot0);
                        dot0 = fmaf(q_hi.y, kv.w, dot0);
                    }
                    dot0 += __shfl_xor_sync(quad_mask, dot0, 1);
                    dot0 += __shfl_xor_sync(quad_mask, dot0, 2);
                    score0 = dot0 * scale;
                }

                float score1 = hat1 * scale;
                if (need1) {
                    float dot1 = 0.f;
                    const float4* km1 = reinterpret_cast<const float4*>(km_fp32 + static_cast<size_t>(b1) * D);
                    #pragma unroll
                    for (int s = 0; s < F16_STEPS; ++s) {
                        float4 kv = km1[s * 4 + t_in_g];
                        float2 q_lo = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q1_u2[s].x));
                        float2 q_hi = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&q1_u2[s].y));
                        dot1 = fmaf(q_lo.x, kv.x, dot1);
                        dot1 = fmaf(q_lo.y, kv.y, dot1);
                        dot1 = fmaf(q_hi.x, kv.z, dot1);
                        dot1 = fmaf(q_hi.y, kv.w, dot1);
                    }
                    dot1 += __shfl_xor_sync(quad_mask, dot1, 1);
                    dot1 += __shfl_xor_sync(quad_mask, dot1, 2);
                    score1 = dot1 * scale;
                }

                bool my_keep = (t_in_g == 0) ? (!sure_out0) : (!sure_out1);
                if (t_in_g < 2 && my_keep) {
                    int my_b    = (t_in_g == 0) ? b0 : b1;
                    float my_sc = (t_in_g == 0) ? score0 : score1;
                    insert_one_top8(my_tv, my_ti, my_sc, my_b);
                }
            }
        }

        cp_async_wait<0>();
        __syncthreads();
    }

    // -----------------------------------------------------------------------
    // PHASE 2: Diagonal Attention [qb] + imp_diag Top-8 Insert + Bucket Emit
    //          (Load q0..q3 per ks from sQ so q_frag[K_STEPS][4] is NOT held live!)
    // -----------------------------------------------------------------------
    float s_acc[8][4];
    #pragma unroll
    for (int nt = 0; nt < 8; ++nt) {
        s_acc[nt][0] = s_acc[nt][1] = s_acc[nt][2] = s_acc[nt][3] = 0.f;
    }
    #pragma unroll
    for (int ks = 0; ks < K_STEPS; ++ks) {
        uint32_t q0, q1, q2, q3;
        {
            int rq = warp * 16 + (lane & 15);
            int cq = ks * 2 + (lane >> 4);
            uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]);
            asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                : "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3)
                : "r"(q_addr));
        }
        #pragma unroll
        for (int np = 0; np < 4; ++np) {
            int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
            int c = ks * 2 + ((lane >> 3) & 1);
            int cs = c ^ (n_row & 7);
            uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]);
            uint32_t b00, b01, b10, b11;
            asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
                : "r"(addr));
            asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                : "=f"(s_acc[np * 2][0]), "=f"(s_acc[np * 2][1]), "=f"(s_acc[np * 2][2]), "=f"(s_acc[np * 2][3])
                : "r"(q0), "r"(q1), "r"(q2), "r"(q3),
                  "r"(b00), "r"(b01),
                  "f"(s_acc[np * 2][0]), "f"(s_acc[np * 2][1]), "f"(s_acc[np * 2][2]), "f"(s_acc[np * 2][3]));
            asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                : "=f"(s_acc[np * 2 + 1][0]), "=f"(s_acc[np * 2 + 1][1]), "=f"(s_acc[np * 2 + 1][2]), "=f"(s_acc[np * 2 + 1][3])
                : "r"(q0), "r"(q1), "r"(q2), "r"(q3),
                  "r"(b10), "r"(b11),
                  "f"(s_acc[np * 2 + 1][0]), "f"(s_acc[np * 2 + 1][1]), "f"(s_acc[np * 2 + 1][2]), "f"(s_acc[np * 2 + 1][3]));
        }
    }
    __syncthreads();

    if constexpr (D == 64) {
        for (int i = tid; i < 64 * CHUNKS; i += 128) {
            int r = i / CHUNKS;
            int c = i % CHUNKS;
            int cs = c ^ (r & 7);
            if (r < M) {
                cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(s0 + r) * D + c * 8]);
            } else {
                sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
            }
        }
        cp_async_commit();
    } else {
        if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) {
            for (int b = tid; b < qb - 1; b += 128) {
                sBuf->bcount_base[b] = 0;
            }
        }
    }

    float diag_sum0 = 0.f, diag_sum1 = 0.f;
    float m0 = -1e30f, m1 = -1e30f;
    #pragma unroll
    for (int nt = 0; nt < 8; ++nt) {
        int col0 = nt * 8 + t_in_g * 2;
        int col1 = col0 + 1;
        float v00 = s_acc[nt][0] * scale;
        float v01 = s_acc[nt][1] * scale;
        float v10 = s_acc[nt][2] * scale;
        float v11 = s_acc[nt][3] * scale;

        if (col0 <= r0) { diag_sum0 += v00; m0 = fmaxf(m0, v00); s_acc[nt][0] = v00; } else { s_acc[nt][0] = -1e30f; }
        if (col1 <= r0) { diag_sum0 += v01; m0 = fmaxf(m0, v01); s_acc[nt][1] = v01; } else { s_acc[nt][1] = -1e30f; }
        if (col0 <= r1) { diag_sum1 += v10; m1 = fmaxf(m1, v10); s_acc[nt][2] = v10; } else { s_acc[nt][2] = -1e30f; }
        if (col1 <= r1) { diag_sum1 += v11; m1 = fmaxf(m1, v11); s_acc[nt][3] = v11; } else { s_acc[nt][3] = -1e30f; }
    }

    diag_sum0 += __shfl_xor_sync(0xffffffff, diag_sum0, 1);
    diag_sum0 += __shfl_xor_sync(0xffffffff, diag_sum0, 2);
    diag_sum1 += __shfl_xor_sync(0xffffffff, diag_sum1, 1);
    diag_sum1 += __shfl_xor_sync(0xffffffff, diag_sum1, 2);
    float imp_diag0 = diag_sum0 / static_cast<float>(r0 + 1);
    float imp_diag1 = diag_sum1 / static_cast<float>(r1 + 1);

    m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 1));
    m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 2));
    m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 1));
    m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 2));

    float l0 = 0.f, l1 = 0.f;
    #pragma unroll
    for (int nt = 0; nt < 8; ++nt) {
        s_acc[nt][0] = __expf(s_acc[nt][0] - m0);
        s_acc[nt][1] = __expf(s_acc[nt][1] - m0);
        s_acc[nt][2] = __expf(s_acc[nt][2] - m1);
        s_acc[nt][3] = __expf(s_acc[nt][3] - m1);
        l0 += s_acc[nt][0] + s_acc[nt][1];
        l1 += s_acc[nt][2] + s_acc[nt][3];
    }
    l0 += __shfl_xor_sync(0xffffffff, l0, 1);
    l0 += __shfl_xor_sync(0xffffffff, l0, 2);
    l1 += __shfl_xor_sync(0xffffffff, l1, 1);
    l1 += __shfl_xor_sync(0xffffffff, l1, 2);

    uint32_t p_frag[4][4];
    #pragma unroll
    for (int ks = 0; ks < 4; ++ks) {
        __nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][0], s_acc[ks * 2][1]));
        __nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][2], s_acc[ks * 2][3]));
        __nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][0], s_acc[ks * 2 + 1][1]));
        __nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][2], s_acc[ks * 2 + 1][3]));
        p_frag[ks][0] = *reinterpret_cast<uint32_t*>(&v0);
        p_frag[ks][1] = *reinterpret_cast<uint32_t*>(&v1);
        p_frag[ks][2] = *reinterpret_cast<uint32_t*>(&v2);
        p_frag[ks][3] = *reinterpret_cast<uint32_t*>(&v3);
    }

    bool prev_in_top0 = false, prev_in_top1 = false;
    int cnt0 = 0, cnt1 = 0;
    if (qb >= 1) {
        if (t_in_g < 2) {
            insert_one_top8(my_tv, my_ti, (t_in_g == 0) ? imp_diag0 : imp_diag1, qb);
        }
        bool my_prev = false;
        int my_cnt = 0;
        #pragma unroll
        for (int i = 0; i < 8; ++i) {
            if (my_ti[i] == qb - 1) my_prev = true;
            if (my_ti[i] >= 0 && my_ti[i] < qb - 1) my_cnt++;
        }
        prev_in_top0 = __shfl_sync(0xffffffff, my_prev, 0, 4);
        prev_in_top1 = __shfl_sync(0xffffffff, my_prev, 1, 4);
        cnt0 = __shfl_sync(0xffffffff, my_cnt, 0, 4);
        cnt1 = __shfl_sync(0xffffffff, my_cnt, 1, 4);

        if (qb >= 2) {
            int my_r = (t_in_g == 0) ? r0 : r1;
            int my_lpos[8] = {0, 0, 0, 0, 0, 0, 0, 0};

            if (qb - 1 <= MAX_SMEM_BUCKETS) {
                int* s_bc = (D == 64) ? s_bcount_sep : sBuf->bcount_base;
                int* s_bb = (D == 64) ? s_bbase_sep  : (sBuf->bcount_base + MAX_SMEM_BUCKETS);
                if constexpr (D != 64) {
                    __syncthreads();
                }
                if (t_in_g < 2 && my_r < M) {
                    #pragma unroll
                    for (int i = 0; i < 8; ++i) {
                        int bi = my_ti[i];
                        if (bi >= 0 && bi < qb - 1) {
                            my_lpos[i] = atomicAdd(&s_bc[bi], 1);
                        }
                    }
                }
                __syncthreads();

                for (int b = tid; b < qb - 1; b += 128) {
                    int c_blk = s_bc[b];
                    if (c_blk > 0) {
                        int bucket_id = bh * n_blocks + b;
                        int gbase = atomicAdd(&bucket_counts[bucket_id * BCOUNT_STRIDE], c_blk);
                        s_bb[b] = gbase;
                        int first_chunk = (gbase + CHUNK_Q - 1) & ~(CHUNK_Q - 1);
                        for (int p = first_chunk; p < gbase + c_blk; p += CHUNK_Q) {
                            uint32_t wid = atomicAdd(&g_counters[0], 1u);
                            g_work_queue[wid] = make_uint2(static_cast<uint32_t>(bucket_id), static_cast<uint32_t>(p));
                        }
                    }
                }
                __syncthreads();

                if (t_in_g < 2 && my_r < M) {
                    int slot = 0;
                    uint32_t t_global = static_cast<uint32_t>(s0 + my_r);
                    #pragma unroll
                    for (int i = 0; i < 8; ++i) {
                        int bi = my_ti[i];
                        if (bi >= 0 && bi < qb - 1) {
                            int bucket_id = bh * n_blocks + bi;
                            int pos = s_bb[bi] + my_lpos[i];
                            bucket_items[static_cast<size_t>(bucket_id) * S_stride + pos] =
                                (t_global << 4) | static_cast<uint32_t>(slot);
                            slot++;
                        }
                    }
                }
            } else {
                constexpr int chunk_mask = CHUNK_Q - 1;
                if (t_in_g < 2 && my_r < M) {
                    int slot = 0;
                    uint32_t t_global = static_cast<uint32_t>(s0 + my_r);
                    #pragma unroll
                    for (int i = 0; i < 8; ++i) {
                        int bi = my_ti[i];
                        if (bi >= 0 && bi < qb - 1) {
                            int bucket_id = bh * n_blocks + bi;
                            int pos = atomicAdd(&bucket_counts[bucket_id * BCOUNT_STRIDE], 1);
                            bucket_items[static_cast<size_t>(bucket_id) * S_stride + pos] =
                                (t_global << 4) | static_cast<uint32_t>(slot);
                            if ((pos & chunk_mask) == 0) {
                                uint32_t wid = atomicAdd(&g_counters[0], 1u);
                                g_work_queue[wid] = make_uint2(static_cast<uint32_t>(bucket_id), static_cast<uint32_t>(pos));
                            }
                            slot++;
                        }
                    }
                }
            }
        }
    }

    if constexpr (D == 64) {
        cp_async_wait<0>();
        __syncthreads();
    } else {
        if (qb >= 2 && qb - 1 <= MAX_SMEM_BUCKETS) {
            __syncthreads();
        }
        for (int i = tid; i < 64 * CHUNKS; i += 128) {
            int r = i / CHUNKS;
            int c = i % CHUNKS;
            int cs = c ^ (r & 7);
            if (r < M) {
                cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(s0 + r) * D + c * 8]);
            } else {
                sBuf->kv[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
            }
        }
        cp_async_commit();
        cp_async_wait<0>();
        __syncthreads();
    }

    // -----------------------------------------------------------------------
    // PHASE 3: P_curr @ V_qb + [qb - 1] Attention + Bank-Conflict-Free Store
    // -----------------------------------------------------------------------
    float o_acc[D_TILES][4];
    #pragma unroll
    for (int dt = 0; dt < D_TILES; ++dt) {
        o_acc[dt][0] = o_acc[dt][1] = o_acc[dt][2] = o_acc[dt][3] = 0.f;
    }
    #pragma unroll
    for (int ks = 0; ks < 4; ++ks) {
        #pragma unroll
        for (int dp = 0; dp < D / 16; ++dp) {
            int r = ks * 16 + (lane & 15);
            int c = dp * 2 + (lane >> 4);
            int cs = c ^ (r & 7);
            uint32_t addr = smem_u32(&sBuf->kv[r * CHUNKS + cs]);
            uint32_t v00, v01, v10, v11;
            asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                : "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11)
                : "r"(addr));
            asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                : "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3])
                : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
                  "r"(v00), "r"(v01),
                  "f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3]));
            asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                : "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3])
                : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
                  "r"(v10), "r"(v11),
                  "f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3]));
        }
    }

    if (qb >= 1) {
        __syncthreads();
        int sp0 = (qb - 1) * BLOCK_SIZE;
        for (int i = tid; i < 64 * CHUNKS; i += 128) {
            int r = i / CHUNKS;
            int c = i % CHUNKS;
            int cs = c ^ (r & 7);
            cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &k_gmem[(sp0 + r) * D + c * 8]);
        }
        cp_async_commit();
        cp_async_wait<0>();
        __syncthreads();

        float sp_acc[8][4];
        #pragma unroll
        for (int nt = 0; nt < 8; ++nt) {
            sp_acc[nt][0] = sp_acc[nt][1] = sp_acc[nt][2] = sp_acc[nt][3] = 0.f;
        }
        #pragma unroll
        for (int ks = 0; ks < K_STEPS; ++ks) {
            uint32_t q0, q1, q2, q3;
            {
                int rq = warp * 16 + (lane & 15);
                int cq = ks * 2 + (lane >> 4);
                uint32_t q_addr = smem_u32(&sQ[rq * CHUNKS + (cq ^ (rq & 7))]);
                asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                    : "=r"(q0), "=r"(q1), "=r"(q2), "=r"(q3)
                    : "r"(q_addr));
            }
            #pragma unroll
            for (int np = 0; np < 4; ++np) {
                int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
                int c = ks * 2 + ((lane >> 3) & 1);
                int cs = c ^ (n_row & 7);
                uint32_t addr = smem_u32(&sBuf->kv[n_row * CHUNKS + cs]);
                uint32_t b00, b01, b10, b11;
                asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                    : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
                    : "r"(addr));
                asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                    : "=f"(sp_acc[np * 2][0]), "=f"(sp_acc[np * 2][1]), "=f"(sp_acc[np * 2][2]), "=f"(sp_acc[np * 2][3])
                    : "r"(q0), "r"(q1), "r"(q2), "r"(q3),
                      "r"(b00), "r"(b01),
                      "f"(sp_acc[np * 2][0]), "f"(sp_acc[np * 2][1]), "f"(sp_acc[np * 2][2]), "f"(sp_acc[np * 2][3]));
                asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                    : "=f"(sp_acc[np * 2 + 1][0]), "=f"(sp_acc[np * 2 + 1][1]), "=f"(sp_acc[np * 2 + 1][2]), "=f"(sp_acc[np * 2 + 1][3])
                    : "r"(q0), "r"(q1), "r"(q2), "r"(q3),
                      "r"(b10), "r"(b11),
                      "f"(sp_acc[np * 2 + 1][0]), "f"(sp_acc[np * 2 + 1][1]), "f"(sp_acc[np * 2 + 1][2]), "f"(sp_acc[np * 2 + 1][3]));
            }
        }
        __syncthreads();

        for (int i = tid; i < 64 * CHUNKS; i += 128) {
            int r = i / CHUNKS;
            int c = i % CHUNKS;
            int cs = c ^ (r & 7);
            cp_async_16B(smem_u32(&sBuf->kv[r * CHUNKS + cs]), &v_gmem[(sp0 + r) * D + c * 8]);
        }
        cp_async_commit();

        float mp0 = m0, mp1 = m1;
        #pragma unroll
        for (int nt = 0; nt < 8; ++nt) {
            int col0 = nt * 8 + t_in_g * 2;
            int col1 = col0 + 1;
            float v00 = (prev_in_top0 || col0 > r0) ? (sp_acc[nt][0] * scale) : -1e30f;
            float v01 = (prev_in_top0 || col1 > r0) ? (sp_acc[nt][1] * scale) : -1e30f;
            float v10 = (prev_in_top1 || col0 > r1) ? (sp_acc[nt][2] * scale) : -1e30f;
            float v11 = (prev_in_top1 || col1 > r1) ? (sp_acc[nt][3] * scale) : -1e30f;
            sp_acc[nt][0] = v00; sp_acc[nt][1] = v01;
            sp_acc[nt][2] = v10; sp_acc[nt][3] = v11;
            mp0 = fmaxf(mp0, fmaxf(v00, v01));
            mp1 = fmaxf(mp1, fmaxf(v10, v11));
        }
        mp0 = fmaxf(mp0, __shfl_xor_sync(0xffffffff, mp0, 1));
        mp0 = fmaxf(mp0, __shfl_xor_sync(0xffffffff, mp0, 2));
        mp1 = fmaxf(mp1, __shfl_xor_sync(0xffffffff, mp1, 1));
        mp1 = fmaxf(mp1, __shfl_xor_sync(0xffffffff, mp1, 2));

        float alpha0 = __expf(m0 - mp0);
        float alpha1 = __expf(m1 - mp1);
        m0 = mp0;
        m1 = mp1;

        #pragma unroll
        for (int dt = 0; dt < D_TILES; ++dt) {
            o_acc[dt][0] *= alpha0; o_acc[dt][1] *= alpha0;
            o_acc[dt][2] *= alpha1; o_acc[dt][3] *= alpha1;
        }

        float lp0 = 0.f, lp1 = 0.f;
        #pragma unroll
        for (int nt = 0; nt < 8; ++nt) {
            sp_acc[nt][0] = __expf(sp_acc[nt][0] - m0);
            sp_acc[nt][1] = __expf(sp_acc[nt][1] - m0);
            sp_acc[nt][2] = __expf(sp_acc[nt][2] - m1);
            sp_acc[nt][3] = __expf(sp_acc[nt][3] - m1);
            lp0 += sp_acc[nt][0] + sp_acc[nt][1];
            lp1 += sp_acc[nt][2] + sp_acc[nt][3];
        }
        lp0 += __shfl_xor_sync(0xffffffff, lp0, 1);
        lp0 += __shfl_xor_sync(0xffffffff, lp0, 2);
        lp1 += __shfl_xor_sync(0xffffffff, lp1, 1);
        lp1 += __shfl_xor_sync(0xffffffff, lp1, 2);
        l0 = l0 * alpha0 + lp0;
        l1 = l1 * alpha1 + lp1;

        #pragma unroll
        for (int ks = 0; ks < 4; ++ks) {
            __nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2][0], sp_acc[ks * 2][1]));
            __nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2][2], sp_acc[ks * 2][3]));
            __nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2 + 1][0], sp_acc[ks * 2 + 1][1]));
            __nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(sp_acc[ks * 2 + 1][2], sp_acc[ks * 2 + 1][3]));
            p_frag[ks][0] = *reinterpret_cast<uint32_t*>(&v0);
            p_frag[ks][1] = *reinterpret_cast<uint32_t*>(&v1);
            p_frag[ks][2] = *reinterpret_cast<uint32_t*>(&v2);
            p_frag[ks][3] = *reinterpret_cast<uint32_t*>(&v3);
        }

        cp_async_wait<0>();
        __syncthreads();

        #pragma unroll
        for (int ks = 0; ks < 4; ++ks) {
            #pragma unroll
            for (int dp = 0; dp < D / 16; ++dp) {
                int r = ks * 16 + (lane & 15);
                int c = dp * 2 + (lane >> 4);
                int cs = c ^ (r & 7);
                uint32_t addr = smem_u32(&sBuf->kv[r * CHUNKS + cs]);
                uint32_t v00, v01, v10, v11;
                asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                    : "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11)
                    : "r"(addr));
                asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                    : "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3])
                    : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
                      "r"(v00), "r"(v01),
                      "f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3]));
                asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                    : "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3])
                    : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
                      "r"(v10), "r"(v11),
                      "f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3]));
            }
        }
    }

    float inv_l0 = 1.0f / l0;
    float inv_l1 = 1.0f / l1;
    __syncthreads();
    #pragma unroll
    for (int dt = 0; dt < D_TILES; ++dt) {
        int dts0 = dt ^ (r0 & 7);
        int dts1 = dt ^ (r1 & 7);
        reinterpret_cast<__nv_bfloat162*>(&sQ[r0 * CHUNKS + dts0])[t_in_g] =
            __float22bfloat162_rn(make_float2(o_acc[dt][0] * inv_l0, o_acc[dt][1] * inv_l0));
        reinterpret_cast<__nv_bfloat162*>(&sQ[r1 * CHUNKS + dts1])[t_in_g] =
            __float22bfloat162_rn(make_float2(o_acc[dt][2] * inv_l1, o_acc[dt][3] * inv_l1));
    }

    if (t_in_g == 0) {
        if (r0 < M) {
            size_t gt0 = static_cast<size_t>(bh) * S + s0 + r0;
            remote_cnt[gt0] = static_cast<uint8_t>(cnt0);
            if (cnt0 > 0) {
                ml_all[gt0 * 16 + 0] = make_float2(m0, l0);
            }
        }
        if (r1 < M) {
            size_t gt1 = static_cast<size_t>(bh) * S + s0 + r1;
            remote_cnt[gt1] = static_cast<uint8_t>(cnt1);
            if (cnt1 > 0) {
                ml_all[gt1 * 16 + 0] = make_float2(m1, l1);
            }
        }
    }
    __syncthreads();

    __nv_bfloat16* o_blk = O + (static_cast<size_t>(bh) * S + s0) * D;
    for (int i = tid; i < M * CHUNKS; i += 128) {
        int r = i / CHUNKS;
        int c = i % CHUNKS;
        reinterpret_cast<uint4*>(o_blk)[i] = sQ[r * CHUNKS + (c ^ (r & 7))];
    }
}

// ---------------------------------------------------------------------------
// Kernel 3: Warp-Asynchronous High-Occupancy Persistent Remote Attention
// ---------------------------------------------------------------------------
template <int D>
__global__ __launch_bounds__(128, (D == 64 ? 4 : 2)) void remote_kv_attn_kernel(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    __nv_bfloat16* __restrict__ partial_O,
    float2* __restrict__ ml_all,
    const int* __restrict__ bucket_counts,
    const uint32_t* __restrict__ bucket_items,
    const uint2* __restrict__ g_work_queue,
    uint32_t* __restrict__ g_counters,
    int S,
    int n_blocks,
    int S_stride,
    float scale
) {
    constexpr int CHUNKS = D / 8;
    constexpr int K_STEPS = D / 16;
    constexpr int D_TILES = D / 8;
    constexpr int CHUNK_Q = (D == 64) ? 128 : 256;

    extern __shared__ __align__(128) char smem_raw[];
    uint4* sK = reinterpret_cast<uint4*>(smem_raw);
    uint4* sV = sK + 64 * CHUNKS;
    uint4* sQ = sV + 64 * CHUNKS;

    int tid = threadIdx.x;
    int warp = tid >> 5;
    int lane = tid & 31;
    int groupID = lane >> 2;
    int t_in_g = lane & 3;

    uint32_t total_w = __ldg(&g_counters[0]);
    if (blockIdx.x >= total_w) return;

    bool first_iter = true;
    while (true) {
        uint32_t* s_meta = reinterpret_cast<uint32_t*>(sQ);
        if (first_iter) {
            first_iter = false;
            if (tid == 0) {
                uint2 w = __ldg(&g_work_queue[blockIdx.x]);
                s_meta[1] = w.x;
                s_meta[2] = w.y;
                s_meta[3] = static_cast<uint32_t>(__ldg(&bucket_counts[w.x * BCOUNT_STRIDE]));
            }
            __syncthreads();
        } else {
            if (tid == 0) {
                uint32_t wid = gridDim.x + atomicAdd(&g_counters[1], 1u);
                if (wid < total_w) {
                    s_meta[0] = wid;
                    uint2 w = __ldg(&g_work_queue[wid]);
                    s_meta[1] = w.x;
                    s_meta[2] = w.y;
                    s_meta[3] = static_cast<uint32_t>(__ldg(&bucket_counts[w.x * BCOUNT_STRIDE]));
                } else {
                    s_meta[0] = 0xffffffffu;
                }
            }
            __syncthreads();
            if (s_meta[0] == 0xffffffffu) break;
        }

        int bucket_id = static_cast<int>(s_meta[1]);
        int chunk_start = static_cast<int>(s_meta[2]);
        int s_bucket_total = static_cast<int>(s_meta[3]);
        int chunk_len = min(CHUNK_Q, s_bucket_total - chunk_start);

        int bh = bucket_id / n_blocks;
        int bi = bucket_id % n_blocks;
        int k_s0 = bi * BLOCK_SIZE;

        const __nv_bfloat16* q_head = Q + static_cast<size_t>(bh) * S * D;
        const __nv_bfloat16* k_blk = K + (static_cast<size_t>(bh) * S + k_s0) * D;
        const __nv_bfloat16* v_blk = V + (static_cast<size_t>(bh) * S + k_s0) * D;
        const uint32_t* b_items = bucket_items + static_cast<size_t>(bucket_id) * S_stride + chunk_start;

        int r_load = lane & 15;
        int q_base0 = warp * 16;
        uint32_t my_pk16 = (q_base0 + r_load < chunk_len) ? __ldg(&b_items[q_base0 + r_load]) : 0u;

        for (int i = tid; i < 64 * CHUNKS; i += 128) {
            int r = i / CHUNKS;
            int c = i % CHUNKS;
            int cs = c ^ (r & 7);
            cp_async_16B(smem_u32(&sK[r * CHUNKS + cs]), &k_blk[r * D + c * 8]);
            cp_async_16B(smem_u32(&sV[r * CHUNKS + cs]), &v_blk[r * D + c * 8]);
        }
        cp_async_commit();
        cp_async_wait<0>();
        __syncthreads();

        uint4* sQ_w = sQ + warp * (16 * CHUNKS);
        for (int q_base = q_base0; q_base < chunk_len; q_base += 64) {
            int w_m = min(16, chunk_len - q_base);
            if (q_base > q_base0) {
                my_pk16 = (r_load < w_m) ? __ldg(&b_items[q_base + r_load]) : 0u;
            }

            #pragma unroll
            for (int it = 0; it < (16 * CHUNKS) / 32; ++it) {
                int idx = it * 32 + lane;
                int r = idx / CHUNKS;
                int c = idx % CHUNKS;
                int cs = c ^ (r & 7);
                uint32_t pk_r = __shfl_sync(0xffffffff, my_pk16, r);
                if (r < w_m) {
                    uint32_t t_idx = pk_r >> 4;
                    cp_async_16B(smem_u32(&sQ_w[r * CHUNKS + cs]), &q_head[t_idx * D + c * 8]);
                } else {
                    sQ_w[r * CHUNKS + cs] = make_uint4(0, 0, 0, 0);
                }
            }
            cp_async_commit();
            cp_async_wait<0>();
            __syncwarp();

            uint32_t q_frag[K_STEPS][4];
            #pragma unroll
            for (int ks = 0; ks < K_STEPS; ++ks) {
                int r = lane & 15;
                int c = ks * 2 + (lane >> 4);
                int cs = c ^ (r & 7);
                uint32_t addr = smem_u32(&sQ_w[r * CHUNKS + cs]);
                asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                    : "=r"(q_frag[ks][0]), "=r"(q_frag[ks][1]), "=r"(q_frag[ks][2]), "=r"(q_frag[ks][3])
                    : "r"(addr));
            }

            float s_acc[8][4];
            #pragma unroll
            for (int nt = 0; nt < 8; ++nt) {
                s_acc[nt][0] = s_acc[nt][1] = s_acc[nt][2] = s_acc[nt][3] = 0.f;
            }
            #pragma unroll
            for (int ks = 0; ks < K_STEPS; ++ks) {
                #pragma unroll
                for (int np = 0; np < 4; ++np) {
                    int n_row = np * 16 + ((lane >> 4) * 8) + (lane & 7);
                    int c = ks * 2 + ((lane >> 3) & 1);
                    int cs = c ^ (n_row & 7);
                    uint32_t addr = smem_u32(&sK[n_row * CHUNKS + cs]);
                    uint32_t b00, b01, b10, b11;
                    asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                        : "=r"(b00), "=r"(b01), "=r"(b10), "=r"(b11)
                        : "r"(addr));
                    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                        : "=f"(s_acc[np * 2][0]), "=f"(s_acc[np * 2][1]), "=f"(s_acc[np * 2][2]), "=f"(s_acc[np * 2][3])
                        : "r"(q_frag[ks][0]), "r"(q_frag[ks][1]), "r"(q_frag[ks][2]), "r"(q_frag[ks][3]),
                          "r"(b00), "r"(b01),
                          "f"(s_acc[np * 2][0]), "f"(s_acc[np * 2][1]), "f"(s_acc[np * 2][2]), "f"(s_acc[np * 2][3]));
                    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                        : "=f"(s_acc[np * 2 + 1][0]), "=f"(s_acc[np * 2 + 1][1]), "=f"(s_acc[np * 2 + 1][2]), "=f"(s_acc[np * 2 + 1][3])
                        : "r"(q_frag[ks][0]), "r"(q_frag[ks][1]), "r"(q_frag[ks][2]), "r"(q_frag[ks][3]),
                          "r"(b10), "r"(b11),
                          "f"(s_acc[np * 2 + 1][0]), "f"(s_acc[np * 2 + 1][1]), "f"(s_acc[np * 2 + 1][2]), "f"(s_acc[np * 2 + 1][3]));
                }
            }

            float m0 = -1e30f, m1 = -1e30f;
            #pragma unroll
            for (int nt = 0; nt < 8; ++nt) {
                s_acc[nt][0] *= scale; s_acc[nt][1] *= scale;
                s_acc[nt][2] *= scale; s_acc[nt][3] *= scale;
                m0 = fmaxf(m0, fmaxf(s_acc[nt][0], s_acc[nt][1]));
                m1 = fmaxf(m1, fmaxf(s_acc[nt][2], s_acc[nt][3]));
            }
            m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 1));
            m0 = fmaxf(m0, __shfl_xor_sync(0xffffffff, m0, 2));
            m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 1));
            m1 = fmaxf(m1, __shfl_xor_sync(0xffffffff, m1, 2));

            float l0 = 0.f, l1 = 0.f;
            #pragma unroll
            for (int nt = 0; nt < 8; ++nt) {
                s_acc[nt][0] = __expf(s_acc[nt][0] - m0);
                s_acc[nt][1] = __expf(s_acc[nt][1] - m0);
                s_acc[nt][2] = __expf(s_acc[nt][2] - m1);
                s_acc[nt][3] = __expf(s_acc[nt][3] - m1);
                l0 += s_acc[nt][0] + s_acc[nt][1];
                l1 += s_acc[nt][2] + s_acc[nt][3];
            }
            l0 += __shfl_xor_sync(0xffffffff, l0, 1);
            l0 += __shfl_xor_sync(0xffffffff, l0, 2);
            l1 += __shfl_xor_sync(0xffffffff, l1, 1);
            l1 += __shfl_xor_sync(0xffffffff, l1, 2);

            uint32_t p_frag[4][4];
            #pragma unroll
            for (int ks = 0; ks < 4; ++ks) {
                __nv_bfloat162 v0 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][0], s_acc[ks * 2][1]));
                __nv_bfloat162 v1 = __float22bfloat162_rn(make_float2(s_acc[ks * 2][2], s_acc[ks * 2][3]));
                __nv_bfloat162 v2 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][0], s_acc[ks * 2 + 1][1]));
                __nv_bfloat162 v3 = __float22bfloat162_rn(make_float2(s_acc[ks * 2 + 1][2], s_acc[ks * 2 + 1][3]));
                p_frag[ks][0] = *reinterpret_cast<uint32_t*>(&v0);
                p_frag[ks][1] = *reinterpret_cast<uint32_t*>(&v1);
                p_frag[ks][2] = *reinterpret_cast<uint32_t*>(&v2);
                p_frag[ks][3] = *reinterpret_cast<uint32_t*>(&v3);
            }

            float o_acc[D_TILES][4];
            #pragma unroll
            for (int dt = 0; dt < D_TILES; ++dt) {
                o_acc[dt][0] = o_acc[dt][1] = o_acc[dt][2] = o_acc[dt][3] = 0.f;
            }
            #pragma unroll
            for (int ks = 0; ks < 4; ++ks) {
                #pragma unroll
                for (int dp = 0; dp < D / 16; ++dp) {
                    int r = ks * 16 + (lane & 15);
                    int c = dp * 2 + (lane >> 4);
                    int cs = c ^ (r & 7);
                    uint32_t addr = smem_u32(&sV[r * CHUNKS + cs]);
                    uint32_t v00, v01, v10, v11;
                    asm volatile("ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0,%1,%2,%3}, [%4];\n"
                        : "=r"(v00), "=r"(v01), "=r"(v10), "=r"(v11)
                        : "r"(addr));
                    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                        : "=f"(o_acc[dp * 2][0]), "=f"(o_acc[dp * 2][1]), "=f"(o_acc[dp * 2][2]), "=f"(o_acc[dp * 2][3])
                        : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
                          "r"(v00), "r"(v01),
                          "f"(o_acc[dp * 2][0]), "f"(o_acc[dp * 2][1]), "f"(o_acc[dp * 2][2]), "f"(o_acc[dp * 2][3]));
                    asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%10,%11,%12,%13};\n"
                        : "=f"(o_acc[dp * 2 + 1][0]), "=f"(o_acc[dp * 2 + 1][1]), "=f"(o_acc[dp * 2 + 1][2]), "=f"(o_acc[dp * 2 + 1][3])
                        : "r"(p_frag[ks][0]), "r"(p_frag[ks][1]), "r"(p_frag[ks][2]), "r"(p_frag[ks][3]),
                          "r"(v10), "r"(v11),
                          "f"(o_acc[dp * 2 + 1][0]), "f"(o_acc[dp * 2 + 1][1]), "f"(o_acc[dp * 2 + 1][2]), "f"(o_acc[dp * 2 + 1][3]));
                }
            }

            __syncwarp();
            float inv_l0 = 1.0f / l0;
            float inv_l1 = 1.0f / l1;
            int r0_w = groupID;
            int r1_w = groupID + 8;
            #pragma unroll
            for (int dt = 0; dt < D_TILES; ++dt) {
                int dts0 = dt ^ r0_w;
                int dts1 = dt ^ (r1_w & 7);
                reinterpret_cast<__nv_bfloat162*>(&sQ_w[r0_w * CHUNKS + dts0])[t_in_g] =
                    __float22bfloat162_rn(make_float2(o_acc[dt][0] * inv_l0, o_acc[dt][1] * inv_l0));
                reinterpret_cast<__nv_bfloat162*>(&sQ_w[r1_w * CHUNKS + dts1])[t_in_g] =
                    __float22bfloat162_rn(make_float2(o_acc[dt][2] * inv_l1, o_acc[dt][3] * inv_l1));
            }

            uint32_t pk0 = __shfl_sync(0xffffffff, my_pk16, r0_w);
            uint32_t pk1 = __shfl_sync(0xffffffff, my_pk16, r1_w);
            if (t_in_g == 0) {
                if (r0_w < w_m) {
                    size_t gt = static_cast<size_t>(bh) * S + (pk0 >> 4);
                    uint32_t slot = pk0 & 15u;
                    ml_all[gt * 16 + 1 + slot] = make_float2(m0, l0);
                }
                if (r1_w < w_m) {
                    size_t gt = static_cast<size_t>(bh) * S + (pk1 >> 4);
                    uint32_t slot = pk1 & 15u;
                    ml_all[gt * 16 + 1 + slot] = make_float2(m1, l1);
                }
            }
            __syncwarp();

            constexpr int ROWS_PER_STEP = 32 / CHUNKS;
            int sub_r = lane / CHUNKS;
            int sub_c = lane & (CHUNKS - 1);
            #pragma unroll
            for (int step = 0; step < 16; step += ROWS_PER_STEP) {
                int rl = step + sub_r;
                uint32_t pk = __shfl_sync(0xffffffff, my_pk16, rl);
                if (rl < w_m) {
                    size_t gt = static_cast<size_t>(bh) * S + (pk >> 4);
                    uint32_t slot = pk & 15u;
                    uint4 val = sQ_w[rl * CHUNKS + (sub_c ^ (rl & 7))];
                    reinterpret_cast<uint4*>(partial_O + (gt * 8 + slot) * D)[sub_c] = val;
                }
            }
            __syncwarp();
        }
        __syncthreads();
    }
}

// ---------------------------------------------------------------------------
// Kernel 4: Unified High-Occupancy Combine Epilogue for t >= 128
// ---------------------------------------------------------------------------
template <int D>
__global__ __launch_bounds__(256) void combine_epilogue_kernel(
    const __nv_bfloat16* __restrict__ partial_O,
    const float2* __restrict__ ml_all,
    const uint8_t* __restrict__ remote_cnt,
    __nv_bfloat16* __restrict__ O,
    int S
) {
    constexpr int WIDTH = D / 8;
    constexpr int Q_PER_CTA = 256 / WIDTH;

    int tid = threadIdx.x;
    int q_local = tid / WIDTH;
    int sub_lane = tid & (WIDTH - 1);
    int t = 128 + blockIdx.x * Q_PER_CTA + q_local;
    if (t >= S) return;

    int bh = blockIdx.y;
    size_t gt = static_cast<size_t>(bh) * S + t;
    int cnt = static_cast<int>(__ldg(&remote_cnt[gt]));
    if (cnt == 0) return;

    int total_parts = cnt + 1;
    const float2* ml_ptr = ml_all + gt * 16;

    float m_arr[9];
    float w_arr[9];
    float m_max = -1e30f;
    #pragma unroll
    for (int k = 0; k < 9; ++k) {
        if (k < total_parts) {
            float2 mv = __ldg(&ml_ptr[k]);
            m_arr[k] = mv.x;
            w_arr[k] = mv.y;
            m_max = fmaxf(m_max, mv.x);
        } else {
            m_arr[k] = -1e30f;
            w_arr[k] = 0.f;
        }
    }

    float w_sum = 0.f;
    #pragma unroll
    for (int k = 0; k < 9; ++k) {
        float w = (k < total_parts) ? (w_arr[k] * __expf(m_arr[k] - m_max)) : 0.f;
        w_arr[k] = w;
        w_sum += w;
    }
    float inv_w_sum = 1.0f / w_sum;

    float acc[8];
    uint4 u_loc = reinterpret_cast<const uint4*>(O + gt * D)[sub_lane];
    unpack_bf16x8(u_loc, acc);
    float w0 = w_arr[0] * inv_w_sum;
    #pragma unroll
    for (int j = 0; j < 8; ++j) {
        acc[j] *= w0;
    }

    const uint4* po_u4 = reinterpret_cast<const uint4*>(partial_O + (gt * 8) * D) + sub_lane;
    #pragma unroll
    for (int slot = 0; slot < 8; ++slot) {
        if (slot < cnt) {
            float wk = w_arr[slot + 1] * inv_w_sum;
            uint4 u = __ldg(&po_u4[slot * WIDTH]);
            float vals[8];
            unpack_bf16x8(u, vals);
            #pragma unroll
            for (int j = 0; j < 8; ++j) {
                acc[j] = fmaf(wk, vals[j], acc[j]);
            }
        }
    }

    reinterpret_cast<uint4*>(O + gt * D)[sub_lane] = pack_bf16x8(acc);
}

// ---------------------------------------------------------------------------
// Cached Workspace Manager
// ---------------------------------------------------------------------------
struct Workspace {
    void* ptr = nullptr;
    size_t cap = 0;

    void* ensure(size_t needed) {
        if (needed > cap) {
            if (ptr) cudaFree(ptr);
            size_t alloc_sz = ((needed + (1 << 20) - 1) & ~((size_t)(1 << 20) - 1));
            TORCH_CHECK(cudaMalloc(&ptr, alloc_sz) == cudaSuccess, "NSA workspace cudaMalloc failed");
            cap = alloc_sz;
        }
        return ptr;
    }
};

static Workspace g_ws[8];

inline size_t align256(size_t x) {
    return (x + 255) & ~((size_t)255);
}

template <int D>
void launch_nsa_subbatch(
    const __nv_bfloat16* q_ptr,
    const __nv_bfloat16* k_ptr,
    const __nv_bfloat16* v_ptr,
    torch::Tensor& out,
    c10::IntArrayRef out_sizes,
    c10::Device out_dev,
    size_t head_off,
    int BH,
    int S,
    int dev,
    cudaStream_t stream
) {
    constexpr int CHUNK_Q = (D == 64) ? 128 : MAX_CHUNK_Q;
    int n_blocks = (S + BLOCK_SIZE - 1) / BLOCK_SIZE;
    int S_stride = (S + 31) & ~31;
    int chunks_per_s = (S + CHUNK_Q - 1) / CHUNK_Q;

    size_t sz_kmean_f  = align256(static_cast<size_t>(BH) * n_blocks * D * sizeof(float));
    size_t sz_kmean_bf = (D == 128) ? align256(static_cast<size_t>(BH) * n_blocks * D * sizeof(__nv_bfloat16)) : 0;
    size_t sz_kmean_er = (D == 128) ? align256(static_cast<size_t>(BH) * n_blocks * sizeof(float)) : 0;
    size_t sz_bcounts  = align256(static_cast<size_t>(BH) * n_blocks * BCOUNT_STRIDE * sizeof(int));
    size_t sz_cntrs    = align256(2 * sizeof(uint32_t));
    size_t sz_bitems   = align256(static_cast<size_t>(BH) * n_blocks * S_stride * sizeof(uint32_t));
    size_t sz_wqueue   = align256(static_cast<size_t>(BH) * n_blocks * chunks_per_s * sizeof(uint2));
    size_t sz_po       = align256(static_cast<size_t>(BH) * S * 8 * D * sizeof(__nv_bfloat16));
    size_t sz_ml       = align256(static_cast<size_t>(BH) * S * 16 * sizeof(float2));
    size_t sz_rcnt     = align256(static_cast<size_t>(BH) * S * sizeof(uint8_t));

    size_t total_bytes = sz_kmean_f + sz_kmean_bf + sz_kmean_er + sz_bcounts + sz_cntrs + sz_bitems + sz_wqueue + sz_po + sz_ml + sz_rcnt;
    char* base = static_cast<char*>(g_ws[dev].ensure(total_bytes));

    float* k_mean_f         = reinterpret_cast<float*>(base); base += sz_kmean_f;
    __nv_bfloat16* k_mean_b = (D == 128) ? reinterpret_cast<__nv_bfloat16*>(base) : nullptr; base += sz_kmean_bf;
    float* k_mean_err       = (D == 128) ? reinterpret_cast<float*>(base) : nullptr; base += sz_kmean_er;
    int* bucket_counts      = reinterpret_cast<int*>(base); base += sz_bcounts;
    uint32_t* g_counters    = reinterpret_cast<uint32_t*>(base); base += sz_cntrs;
    uint32_t* bucket_items  = reinterpret_cast<uint32_t*>(base); base += sz_bitems;
    uint2* g_work_queue     = reinterpret_cast<uint2*>(base); base += sz_wqueue;
    __nv_bfloat16* part_O   = reinterpret_cast<__nv_bfloat16*>(base); base += sz_po;
    float2* ml_all          = reinterpret_cast<float2*>(base); base += sz_ml;
    uint8_t* remote_cnt     = reinterpret_cast<uint8_t*>(base);

    float scale = static_cast<float>(1.0 / std::sqrt(static_cast<double>(D)));

    dim3 grid_blk(n_blocks, BH);
    k_mean_and_init_kernel<D><<<grid_blk, 256, 0, stream>>>(
        k_ptr, k_mean_f, k_mean_b, k_mean_err, bucket_counts, g_counters, S, n_blocks
    );

    if (!out.defined()) {
        out = torch::Tensor(at::detail::empty_cuda(
            out_sizes, at::ScalarType::BFloat16, out_dev, std::nullopt
        ));
    }
    __nv_bfloat16* o_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()) + head_off;

    constexpr int CHUNKS = D / 8;
    constexpr int MIN_BLK_LOCAL = (D == 64) ? 4 : 3;
    constexpr size_t SMEM_LOCAL = 64 * CHUNKS * sizeof(uint4) + sizeof(LocalSharedBuf<D>)
                                + ((D == 64) ? (2 * 256 * sizeof(int)) : 0);
    constexpr size_t SMEM_REM   = 3 * 64 * CHUNKS * sizeof(uint4);

    static bool attr_set[8] = {false};
    if (!attr_set[dev]) {
        cudaFuncSetAttribute(
            local_attn_score_select_kernel<D, MIN_BLK_LOCAL>,
            cudaFuncAttributePreferredSharedMemoryCarveout,
            cudaSharedmemCarveoutMaxShared
        );
        cudaFuncSetAttribute(
            local_attn_score_select_kernel<D, MIN_BLK_LOCAL>,
            cudaFuncAttributeMaxDynamicSharedMemorySize,
            static_cast<int>(SMEM_LOCAL)
        );
        cudaFuncSetAttribute(
            remote_kv_attn_kernel<D>,
            cudaFuncAttributePreferredSharedMemoryCarveout,
            cudaSharedmemCarveoutMaxShared
        );
        cudaFuncSetAttribute(
            remote_kv_attn_kernel<D>,
            cudaFuncAttributeMaxDynamicSharedMemorySize,
            static_cast<int>(SMEM_REM)
        );
        attr_set[dev] = true;
    }

    local_attn_score_select_kernel<D, MIN_BLK_LOCAL><<<grid_blk, 128, SMEM_LOCAL, stream>>>(
        q_ptr, k_ptr, v_ptr, k_mean_f, k_mean_b, k_mean_err, o_ptr, ml_all, remote_cnt,
        bucket_counts, bucket_items, g_work_queue, g_counters,
        S, n_blocks, S_stride, scale
    );

    if (S > 128) {
        constexpr int PERSISTENT_CTAS = (D == 64) ? (188 * 4) : (188 * 2);
        remote_kv_attn_kernel<D><<<PERSISTENT_CTAS, 128, SMEM_REM, stream>>>(
            q_ptr, k_ptr, v_ptr, part_O, ml_all, bucket_counts, bucket_items,
            g_work_queue, g_counters, S, n_blocks, S_stride, scale
        );

        constexpr int Q_PER_CTA = 256 / (D / 8);
        int active_s = S - 128;
        dim3 grid_epi((active_s + Q_PER_CTA - 1) / Q_PER_CTA, BH);
        combine_epilogue_kernel<D><<<grid_epi, 256, 0, stream>>>(
            part_O, ml_all, remote_cnt, o_ptr, S
        );
    }
}

template <int D>
void launch_nsa_l2_tiled(
    const __nv_bfloat16* q_ptr,
    const __nv_bfloat16* k_ptr,
    const __nv_bfloat16* v_ptr,
    torch::Tensor& out,
    c10::IntArrayRef out_sizes,
    c10::Device out_dev,
    int BH,
    int S,
    int dev,
    cudaStream_t stream
) {
    size_t bytes_per_head = static_cast<size_t>(S) * D * 12 * sizeof(__nv_bfloat16);
    int max_bh_l2 = static_cast<int>((112ULL * 1024 * 1024) / std::max<size_t>(1ULL, bytes_per_head));
    int bh_step = (max_bh_l2 >= BH) ? BH : std::max(1, (BH + 1) / 2);

    size_t head_stride = static_cast<size_t>(S) * D;
    for (int bh0 = 0; bh0 < BH; bh0 += bh_step) {
        int cur_bh = std::min(bh_step, BH - bh0);
        size_t off = static_cast<size_t>(bh0) * head_stride;
        launch_nsa_subbatch<D>(
            q_ptr + off, k_ptr + off, v_ptr + off,
            out, out_sizes, out_dev, off,
            cur_bh, S, dev, stream
        );
    }
}

} // namespace

torch::Tensor nsa_forward_cuda(
    const torch::Tensor& q_in,
    const torch::Tensor& k_in,
    const torch::Tensor& v_in
) {
    TORCH_CHECK(q_in.is_cuda() && k_in.is_cuda() && v_in.is_cuda(), "q, k, v must be CUDA tensors");
    TORCH_CHECK(q_in.scalar_type() == torch::kBFloat16, "q must be bfloat16");

    torch::Tensor q_tmp, k_tmp, v_tmp;
    const torch::Tensor& q = q_in.is_contiguous() ? q_in : (q_tmp = q_in.contiguous());
    const torch::Tensor& k = k_in.is_contiguous() ? k_in : (k_tmp = k_in.contiguous());
    const torch::Tensor& v = v_in.is_contiguous() ? v_in : (v_tmp = v_in.contiguous());

    int B = static_cast<int>(q.size(0));
    int H = static_cast<int>(q.size(1));
    int S = static_cast<int>(q.size(2));
    int D = static_cast<int>(q.size(3));
    int BH = B * H;

    int dev = q.get_device();
    cudaStream_t stream = at::cuda::getCurrentCUDAStream(dev);

    const __nv_bfloat16* q_ptr = reinterpret_cast<const __nv_bfloat16*>(q.data_ptr());
    const __nv_bfloat16* k_ptr = reinterpret_cast<const __nv_bfloat16*>(k.data_ptr());
    const __nv_bfloat16* v_ptr = reinterpret_cast<const __nv_bfloat16*>(v.data_ptr());

    torch::Tensor out;
    if (D == 64) {
        launch_nsa_l2_tiled<64>(q_ptr, k_ptr, v_ptr, out, q.sizes(), q.device(), BH, S, dev, stream);
    } else if (D == 128) {
        launch_nsa_l2_tiled<128>(q_ptr, k_ptr, v_ptr, out, q.sizes(), q.device(), BH, S, dev, stream);
    } else {
        TORCH_CHECK(false, "Unsupported head dim D: ", D);
    }

    return out;
}
"""

_CPP_SOURCE = "torch::Tensor nsa_forward_cuda(const torch::Tensor& q, const torch::Tensor& k, const torch::Tensor& v);"

_ext = load_inline(
    name="deepseek_nsa_sm120_v10",
    cpp_sources=_CPP_SOURCE,
    cuda_sources=_CUDA_SOURCE,
    functions=["nsa_forward_cuda"],
    extra_cuda_cflags=["-O3"],
)


class Model(nn.Module):
    def __init__(self, B: int, H: int, S: int, D: int):
        super().__init__()
        self.B, self.H, self.S, self.D = B, H, S, D
        self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16))

    def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
        return _ext.nsa_forward_cuda(q, k, v)

20261110_113716_agy_gemini-4-carbon-high_02_deepseek_nsa