KernelBench cuda · RTX PRO 6000

DeepSeek NSA Gemini 4 Neon (XHigh)

0.073 msgeomean latency across six shapes · lower is better

Gemini 4 Neon (XHigh) built a hand-written SM120 CUDA/PTX sparse attention pipeline with exact per-token NSA selection (fp32 block key means split into three bf16 planes for exact tensor-core scoring, causal diagonal mean, register top-8 heaps with exact reference tie-breaking to the larger block index) and a lock-free inverted-index key-stationary attention kernel that stores each (row, block) partial output as saturated signed int8 (pack_i8x4 via cvt.rni.sat.s8.f32) with a compact 8-byte (m, l, scale) header before a softmax-weighted merge. Small D=64 shapes (S <= 1024) take a fused row-stationary kernel with no int8 intermediate.

harnessagyagent session6h 32mtotal wall6h 38mcheck5mbenchmark1soutput tokens—gpu-lock wait24mgpu-lock held4h 8mregimecompute

Per-shape latencymilliseconds are the measured result; the dense-equivalent roofline is not a useful ceiling

1×16×2048×640.050 ms
1×16×4127×640.089 ms
1×8×8192×640.098 ms
1×8×8191×1280.158 ms
4×8×1024×640.035 ms
2×8×3000×640.063 ms

geomean(0.050 ms · 0.089 ms · 0.098 ms · 0.158 ms · 0.035 ms · 0.063 ms) = 0.073 ms

Kernel source (redacted)
"""DeepSeek NSA-inspired sparse attention in CUDA for RTX PRO 6000 (SM120).

Implements exact bench semantics:
  - Causal block importance = mean(q · k / sqrt(D)) over causal keys in 64-token blocks
  - Top-8 blocks unioned with 64-token causal sliding window
  - Exact FP32 block importance via zero-smem 128-bit uint4 burst K_mean (3xBF16 unified buffer)
    + parallel warp Hillis-Steele diagonal importance
  - Zero-SMEM Register-Direct 4-Thread Split Top-8 (0 stack, 0 spills across all fast-path kernels)
  - Two specialized SM120 BF16 Tensor Core MMA execution paths (software-pipelined ldmatrix, fused-FMA softmax):
      * Path A: 2-Warp-Group Pipelined BN=16 Exact 3xBF16 Split-KV Direct Fused Attention (n_blocks <= 16),
        with zero-stall startup overlap, early K_next prefetch, and folded LPT grid balancing
      * Path B: 4-Kernel Lock-Free Inverted-Index Block-Centric Sparse Attention:
          1. compute_k_mean_and_diag_kernel (zero-smem 128-bit uint4 burst K/Q + unified [BH, 3, nb, D] K_h)
          2. score_topk_and_scatter_64/128 (fused Q+K_h1[0] startup, exact 3xBF16 Top-8, 6-CTA/SM D=64 / 3-CTA/SM D=128)
          3. remote_block_attn_kernel (register-preloaded ql_b before QK MMA, overlapped Q_next prefetch,
             Causal-LPT bi-outer Phase 2, PTX cvt.rni.sat.s8+prmt MMA-native int8 o_partial)
          4. local_window_and_merge_64/128 (LPT local window, early K_next prefetch, __ldcs register-direct
             MMA-native merge with 4-wide batching & mls_partial preload on D=128, __stcs streaming output write)
"""
from __future__ import annotations

import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

BLOCK_SIZE = 64
TOP_N_BLOCKS = 8
SLIDING_WINDOW = 64

CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cuda_fp16.h>
#include <stdint.h>
#include <math.h>
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>

#define BLOCK_SIZE 64
#define TOP_N 8

struct __align__(4) i8x4 {
    int8_t x, y, z, w;
};

// -----------------------------------------------------------------------------
// Low-level SM120 PTX Primitives
// -----------------------------------------------------------------------------

__device__ __forceinline__ float fast_exp2f(float x) {
    float y;
    asm volatile("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x));
    return y;
}

__device__ __forceinline__ uint32_t pack_bf16x2(float a, float b) {
    __nv_bfloat162 v = __floats2bfloat162_rn(a, b);
    return *reinterpret_cast<const uint32_t*>(&v);
}

// 7-instruction hardware PTX float4 -> saturated signed int8x4 packed in uint32_t!
__device__ __forceinline__ uint32_t pack_i8x4(float a, float b, float c, float d) {
    uint32_t ra, rb, rc, rd, ab, cd, res;
    asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(ra) : "f"(a));
    asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(rb) : "f"(b));
    asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(rc) : "f"(c));
    asm("cvt.rni.sat.s8.f32 %0, %1;" : "=r"(rd) : "f"(d));
    asm("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(ab) : "r"(ra), "r"(rb));
    asm("prmt.b32 %0, %1, %2, 0x0040;" : "=r"(cd) : "r"(rc), "r"(rd));
    asm("prmt.b32 %0, %1, %2, 0x5410;" : "=r"(res) : "r"(ab), "r"(cd));
    return res;
}

__device__ __forceinline__ void bar_sync_wg(int wg) {
    if (wg == 0) {
        asm volatile("bar.sync 1, 128;\n");
    } else {
        asm volatile("bar.sync 2, 128;\n");
    }
}

// Compact 8-byte (m: fp32, l: fp16 in [1, 64], raw_s: bf16 exact fp32 exponent range)
__device__ __forceinline__ uint2 pack_mls(float m, float l, float raw_s) {
    uint32_t u_m = __float_as_uint(m);
    __half hl = __float2half_rn(l);
    __nv_bfloat16 bs = __float2bfloat16_rn(raw_s);
    uint16_t u_l = *reinterpret_cast<const uint16_t*>(&hl);
    uint16_t u_s = *reinterpret_cast<const uint16_t*>(&bs);
    return make_uint2(u_m, static_cast<uint32_t>(u_l) | (static_cast<uint32_t>(u_s) << 16));
}

__device__ __forceinline__ void unpack_mls(uint2 pk, float& m, float& l, float& raw_s) {
    m = __uint_as_float(pk.x);
    uint16_t u_l = static_cast<uint16_t>(pk.y & 0xFFFFu);
    uint16_t u_s = static_cast<uint16_t>(pk.y >> 16);
    l = __half2float(*reinterpret_cast<const __half*>(&u_l));
    raw_s = __bfloat162float(*reinterpret_cast<const __nv_bfloat16*>(&u_s));
}

// Order-preserving uint64 key: (v1 > v2) || (v1 == v2 && b1 > b2)
__device__ __forceinline__ uint64_t make_score_key(float val, int bi) {
    val = val + 0.0f;
    uint32_t u = __float_as_uint(val);
    uint32_t ord = (u & 0x80000000u) ? ~u : (u ^ 0x80000000u);
    return (static_cast<uint64_t>(ord) << 32) | static_cast<uint32_t>(bi);
}

__device__ __forceinline__ void swap_u64(uint64_t& a, uint64_t& b) {
    uint64_t t = a; a = b; b = t;
}

__device__ __forceinline__ void sift_down_3(uint64_t& h3, uint64_t& h7) {
    if (h3 > h7) swap_u64(h3, h7);
}

__device__ __forceinline__ void sift_down_1(uint64_t& h1, uint64_t& h3, uint64_t& h4, uint64_t& h7) {
    if (h3 < h4) {
        if (h1 > h3) { swap_u64(h1, h3); sift_down_3(h3, h7); }
    } else {
        if (h1 > h4) { swap_u64(h1, h4); }
    }
}

__device__ __forceinline__ void sift_down_2(uint64_t& h2, uint64_t& h5, uint64_t& h6) {
    if (h5 < h6) {
        if (h2 > h5) swap_u64(h2, h5);
    } else {
        if (h2 > h6) swap_u64(h2, h6);
    }
}

__device__ __forceinline__ void heap8_sift_root(
    uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3,
    uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7
) {
    if (h1 < h2) {
        if (h0 > h1) { swap_u64(h0, h1); sift_down_1(h1, h3, h4, h7); }
    } else {
        if (h0 > h2) { swap_u64(h0, h2); sift_down_2(h2, h5, h6); }
    }
}

__device__ __forceinline__ void heap8_build(
    uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3,
    uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7
) {
    sift_down_3(h3, h7);
    sift_down_1(h1, h3, h4, h7);
    sift_down_2(h2, h5, h6);
    heap8_sift_root(h0, h1, h2, h3, h4, h5, h6, h7);
}

__device__ __forceinline__ void heap8_push(
    uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3,
    uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7,
    uint64_t x
) {
    if (x > h0) {
        h0 = x;
        heap8_sift_root(h0, h1, h2, h3, h4, h5, h6, h7);
    }
}

__device__ __forceinline__ uint64_t pop_max4_9(
    uint64_t& k0, uint64_t& k1, uint64_t& k2, uint64_t& k3,
    uint64_t& k4, uint64_t& k5, uint64_t& k6, uint64_t& k7,
    uint64_t& k8
) {
    uint64_t m01 = (k0 > k1) ? k0 : k1;
    uint64_t m23 = (k2 > k3) ? k2 : k3;
    uint64_t m45 = (k4 > k5) ? k4 : k5;
    uint64_t m67 = (k6 > k7) ? k6 : k7;
    uint64_t m03 = (m01 > m23) ? m01 : m23;
    uint64_t m47 = (m45 > m67) ? m45 : m67;
    uint64_t lm  = (m03 > m47) ? m03 : m47;
    if (k8 > lm) lm = k8;

    uint64_t o1 = __shfl_xor_sync(0xffffffff, lm, 1);
    if (o1 > lm) lm = o1;
    uint64_t o2 = __shfl_xor_sync(0xffffffff, lm, 2);
    if (o2 > lm) lm = o2;

    uint32_t lm_lo = static_cast<uint32_t>(lm);
    if (static_cast<uint32_t>(k0) == lm_lo) k0 = 0ULL;
    if (static_cast<uint32_t>(k1) == lm_lo) k1 = 0ULL;
    if (static_cast<uint32_t>(k2) == lm_lo) k2 = 0ULL;
    if (static_cast<uint32_t>(k3) == lm_lo) k3 = 0ULL;
    if (static_cast<uint32_t>(k4) == lm_lo) k4 = 0ULL;
    if (static_cast<uint32_t>(k5) == lm_lo) k5 = 0ULL;
    if (static_cast<uint32_t>(k6) == lm_lo) k6 = 0ULL;
    if (static_cast<uint32_t>(k7) == lm_lo) k7 = 0ULL;
    if (static_cast<uint32_t>(k8) == lm_lo) k8 = 0ULL;

    return lm;
}

__device__ __forceinline__ uint64_t pop_max4_5(
    uint64_t& k0, uint64_t& k1, uint64_t& k2, uint64_t& k3, uint64_t& k4
) {
    uint64_t m01 = (k0 > k1) ? k0 : k1;
    uint64_t m23 = (k2 > k3) ? k2 : k3;
    uint64_t lm  = (m01 > m23) ? m01 : m23;
    if (k4 > lm) lm = k4;

    uint64_t o1 = __shfl_xor_sync(0xffffffff, lm, 1);
    if (o1 > lm) lm = o1;
    uint64_t o2 = __shfl_xor_sync(0xffffffff, lm, 2);
    if (o2 > lm) lm = o2;

    uint32_t lm_lo = static_cast<uint32_t>(lm);
    if (static_cast<uint32_t>(k0) == lm_lo) k0 = 0ULL;
    if (static_cast<uint32_t>(k1) == lm_lo) k1 = 0ULL;
    if (static_cast<uint32_t>(k2) == lm_lo) k2 = 0ULL;
    if (static_cast<uint32_t>(k3) == lm_lo) k3 = 0ULL;
    if (static_cast<uint32_t>(k4) == lm_lo) k4 = 0ULL;

    return lm;
}

__device__ __forceinline__ void cp_async_16(void* smem_ptr, const void* glob_ptr) {
    uint32_t smem_addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" :: "r"(smem_addr), "l"(glob_ptr));
}

__device__ __forceinline__ void cp_async_16_pred(void* smem_ptr, const void* glob_ptr, bool pred) {
    uint32_t smem_addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    int src_size = pred ? 16 : 0;
    asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;\n" :: "r"(smem_addr), "l"(glob_ptr), "r"(src_size));
}

__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n");
}

template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}

__device__ __forceinline__ void ldmatrix_x4(
    uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr
) {
    uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile(
        "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
        : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr)
    );
}

__device__ __forceinline__ void ldmatrix_x4_trans(
    uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, const void* smem_ptr
) {
    uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
    asm volatile(
        "ldmatrix.sync.aligned.m8n8.x4.trans.shared.b16 {%0, %1, %2, %3}, [%4];\n"
        : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(addr)
    );
}

__device__ __forceinline__ void mma_m16n8k16_bf16(
    float& d0, float& d1, float& d2, float& d3,
    uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
    uint32_t b0, uint32_t b1
) {
    asm volatile(
        "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
        "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
        : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
        : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)
    );
}

template <int D>
__device__ __forceinline__ int swz_chunk_idx(int row, int col_chunk) {
    constexpr int CHUNKS_PER_ROW = D / 8;
    return row * CHUNKS_PER_ROW + (col_chunk ^ (row & 7));
}

template <int D, int ROWS>
__device__ __forceinline__ void load_tile_swz_async(
    uint4* smem_tile,
    const __nv_bfloat16* glob_base,
    int row_start,
    int S,
    int tid,
    int num_threads
) {
    constexpr int CHUNKS_PER_ROW = D / 8;
    constexpr int TOTAL_CHUNKS = ROWS * CHUNKS_PER_ROW;
    const uint4* glob_u4 = reinterpret_cast<const uint4*>(glob_base);
    #pragma unroll
    for (int idx = tid; idx < TOTAL_CHUNKS; idx += num_threads) {
        int r = idx / CHUNKS_PER_ROW;
        int c = idx % CHUNKS_PER_ROW;
        int gr = row_start + r;
        int dst = swz_chunk_idx<D>(r, c);
        bool valid = (gr < S);
        const uint4* src_ptr = glob_u4 + (valid ? (int64_t(gr) * CHUNKS_PER_ROW + c) : 0);
        cp_async_16_pred(&smem_tile[dst], src_ptr, valid);
    }
}

// Software-pipelined QK 16x64 Tensor Core GEMM (double-buffered b, single a!)
template <int D>
__device__ __forceinline__ void warp_gemm_qk_16x64(
    float s[8][4],
    const uint4* Q_smem,
    const uint4* K_smem,
    int warp_id,
    int lane
) {
    #pragma unroll
    for (int ni = 0; ni < 8; ++ni) {
        s[ni][0] = 0.0f; s[ni][1] = 0.0f; s[ni][2] = 0.0f; s[ni][3] = 0.0f;
    }
    int q_row = warp_id * 16 + (lane & 15);
    int q_k_sub = (lane >> 4);
    int k_row_sub = ((lane >> 4) << 3) + (lane & 7);
    int k_k_sub = (lane >> 3) & 1;

    constexpr int K_STEPS = D / 16;
    uint32_t b[2][4];
    ldmatrix_x4(b[0][0], b[0][1], b[0][2], b[0][3], &K_smem[swz_chunk_idx<D>(k_row_sub, k_k_sub)]);

    #pragma unroll
    for (int ks = 0; ks < K_STEPS; ++ks) {
        int k_chunk = ks * 2;
        uint32_t a0, a1, a2, a3;
        ldmatrix_x4(a0, a1, a2, a3, &Q_smem[swz_chunk_idx<D>(q_row, k_chunk + q_k_sub)]);

        #pragma unroll
        for (int np = 0; np < 4; ++np) {
            int cur_b = np & 1;
            if (np + 1 < 4) {
                int k_row_n = (np + 1) * 16 + k_row_sub;
                ldmatrix_x4(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3],
                            &K_smem[swz_chunk_idx<D>(k_row_n, k_chunk + k_k_sub)]);
            } else if (ks + 1 < K_STEPS) {
                ldmatrix_x4(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3],
                            &K_smem[swz_chunk_idx<D>(k_row_sub, (ks + 1) * 2 + k_k_sub)]);
            }
            mma_m16n8k16_bf16(s[np * 2][0],     s[np * 2][1],     s[np * 2][2],     s[np * 2][3],
                              a0, a1, a2, a3, b[cur_b][0], b[cur_b][1]);
            mma_m16n8k16_bf16(s[np * 2 + 1][0], s[np * 2 + 1][1], s[np * 2 + 1][2], s[np * 2 + 1][3],
                              a0, a1, a2, a3, b[cur_b][2], b[cur_b][3]);
        }
    }
}

// Software-pipelined PV 16xD Tensor Core GEMM (double-buffered b!)
template <int D>
__device__ __forceinline__ void warp_gemm_pv_16xD(
    float o[D / 8][4],
    const uint32_t p_lo[8],
    const uint32_t p_hi[8],
    const uint4* V_smem,
    int lane
) {
    int v_row_sub = (lane & 15);
    int v_d_sub = (lane >> 4);
    constexpr int D_PAIRS = D / 16;

    uint32_t b[2][4];
    ldmatrix_x4_trans(b[0][0], b[0][1], b[0][2], b[0][3], &V_smem[swz_chunk_idx<D>(v_row_sub, v_d_sub)]);

    #pragma unroll
    for (int ks = 0; ks < 4; ++ks) {
        uint32_t a0 = p_lo[ks * 2];
        uint32_t a1 = p_hi[ks * 2];
        uint32_t a2 = p_lo[ks * 2 + 1];
        uint32_t a3 = p_hi[ks * 2 + 1];
        int v_row = ks * 16 + v_row_sub;

        #pragma unroll
        for (int dp = 0; dp < D_PAIRS; ++dp) {
            int cur_b = dp & 1;
            if (dp + 1 < D_PAIRS) {
                ldmatrix_x4_trans(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3],
                                  &V_smem[swz_chunk_idx<D>(v_row, (dp + 1) * 2 + v_d_sub)]);
            } else if (ks + 1 < 4) {
                ldmatrix_x4_trans(b[cur_b ^ 1][0], b[cur_b ^ 1][1], b[cur_b ^ 1][2], b[cur_b ^ 1][3],
                                  &V_smem[swz_chunk_idx<D>((ks + 1) * 16 + v_row_sub, v_d_sub)]);
            }
            mma_m16n8k16_bf16(o[dp * 2][0],     o[dp * 2][1],     o[dp * 2][2],     o[dp * 2][3],
                              a0, a1, a2, a3, b[cur_b][0], b[cur_b][1]);
            mma_m16n8k16_bf16(o[dp * 2 + 1][0], o[dp * 2 + 1][1], o[dp * 2 + 1][2], o[dp * 2 + 1][3],
                              a0, a1, a2, a3, b[cur_b][2], b[cur_b][3]);
        }
    }
}

__device__ __forceinline__ void unpack_u4_bf16(uint4 u, float v[8]) {
    float2 f0 = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u.x));
    float2 f1 = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u.y));
    float2 f2 = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u.z));
    float2 f3 = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&u.w));
    v[0] = f0.x; v[1] = f0.y; v[2] = f1.x; v[3] = f1.y;
    v[4] = f2.x; v[5] = f2.y; v[6] = f3.x; v[7] = f3.y;
}

// -----------------------------------------------------------------------------
// Kernel 1: Zero-smem_K Exact FP32 K_mean (unified [BH, 3, nb, D]) + Diag Importance
//   - D=64: 128-bit uint4 burst (K read ONCE into regs + 4-row parallel Hillis-Steele scan)
//   - D=128: 12-CTA/SM 64-bit uint2 pipelined streaming
// -----------------------------------------------------------------------------
template <int D>
__global__ void compute_k_mean_and_diag_kernel(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    __nv_bfloat16* __restrict__ K_h,
    float* __restrict__ imp_diag,
    uint32_t* __restrict__ d_counters,
    int S,
    int n_blocks,
    float scale
) {
    int bi = blockIdx.x;
    int bh = blockIdx.y;
    int tid = threadIdx.x;
    int warp_id = tid >> 5;
    int lane = tid & 31;

    if (bi == 0 && bh == 0 && tid < 3) {
        d_counters[tid] = 0u;
    }

    __shared__ float warp_sums[4 * D];
    __shared__ float smem_imp[BLOCK_SIZE];

    int row_start = bi * BLOCK_SIZE;
    int rows = min(BLOCK_SIZE, S - row_start);
    const __nv_bfloat16* K_blk = K + int64_t(bh) * S * D + int64_t(row_start) * D;
    const __nv_bfloat16* Q_blk = Q + int64_t(bh) * S * D + int64_t(row_start) * D;

    int r_beg = warp_id * 16;
    int r_end = min(rows, r_beg + 16);

    if constexpr (D == 64) {
        const uint4* gK_u4 = reinterpret_cast<const uint4*>(K_blk);
        const uint4* gQ_u4 = reinterpret_cast<const uint4*>(Q_blk);
        int sub_r = lane >> 3;  // 0..3 (4 rows per warp step)
        int chunk = lane & 7;   // 0..7 (8 bf16 per lane)
        bool do_q = (bi >= TOP_N);

        uint4 kr[4], qr[4];
        #pragma unroll
        for (int it = 0; it < 4; ++it) {
            int r = r_beg + it * 4 + sub_r;
            kr[it] = (r < rows) ? __ldg(&gK_u4[r * 8 + chunk]) : make_uint4(0u, 0u, 0u, 0u);
        }
        if (do_q) {
            #pragma unroll
            for (int it = 0; it < 4; ++it) {
                int r = r_beg + it * 4 + sub_r;
                qr[it] = (r < rows) ? __ldg(&gQ_u4[r * 8 + chunk]) : make_uint4(0u, 0u, 0u, 0u);
            }
        }

        float ws[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
        #pragma unroll
        for (int it = 0; it < 4; ++it) {
            float kv[8];
            unpack_u4_bf16(kr[it], kv);
            #pragma unroll
            for (int j = 0; j < 8; ++j) ws[j] += kv[j];
        }
        #pragma unroll
        for (int j = 0; j < 8; ++j) {
            ws[j] += __shfl_xor_sync(0xffffffff, ws[j], 8);
            ws[j] += __shfl_xor_sync(0xffffffff, ws[j], 16);
        }
        if (sub_r == 0) {
            #pragma unroll
            for (int j = 0; j < 8; ++j) warp_sums[warp_id * 64 + chunk * 8 + j] = ws[j];
        }
        __syncthreads();

        if (do_q) {
            float acc[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
            #pragma unroll
            for (int u = 0; u < 4; ++u) {
                if (u < warp_id) {
                    #pragma unroll
                    for (int j = 0; j < 8; ++j) acc[j] += warp_sums[u * 64 + chunk * 8 + j];
                }
            }
            #pragma unroll
            for (int it = 0; it < 4; ++it) {
                float kv[8], qv[8];
                unpack_u4_bf16(kr[it], kv);
                unpack_u4_bf16(qr[it], qv);
                float pfx[8];
                #pragma unroll
                for (int j = 0; j < 8; ++j) {
                    float x = kv[j];
                    float s1 = __shfl_up_sync(0xffffffff, x, 8);
                    if (sub_r >= 1) x += s1;
                    float s2 = __shfl_up_sync(0xffffffff, x, 16);
                    if (sub_r >= 2) x += s2;
                    pfx[j] = acc[j] + x;
                    acc[j] += __shfl_sync(0xffffffff, x, 24 + chunk);
                }
                float dot = 0.0f;
                #pragma unroll
                for (int j = 0; j < 8; ++j) dot = fmaf(pfx[j], qv[j], dot);
                dot += __shfl_xor_sync(0xffffffff, dot, 1);
                dot += __shfl_xor_sync(0xffffffff, dot, 2);
                dot += __shfl_xor_sync(0xffffffff, dot, 4);
                int r = r_beg + it * 4 + sub_r;
                if (chunk == 0 && r < rows) {
                    smem_imp[r] = dot * (scale * __frcp_rn(float(r + 1)));
                }
            }
            __syncwarp();
            if (lane < r_end - r_beg) {
                imp_diag[int64_t(bh) * S + row_start + r_beg + lane] = smem_imp[r_beg + lane];
            }
        }
    } else {
        // D == 128: 2-burst 128-bit uint4 (4 uint4 K+Q bursts, 2-row parallel Hillis-Steele scan)
        const uint4* gK_u4 = reinterpret_cast<const uint4*>(K_blk);
        const uint4* gQ_u4 = reinterpret_cast<const uint4*>(Q_blk);
        int sub_r = lane >> 4;  // 0..1
        int chunk = lane & 15;  // 0..15 (8 bf16 per lane)

        float ws[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
        #pragma unroll
        for (int half = 0; half < 2; ++half) {
            uint4 kr[4];
            #pragma unroll
            for (int it = 0; it < 4; ++it) {
                int r = r_beg + (half * 4 + it) * 2 + sub_r;
                kr[it] = (r < rows) ? __ldg(&gK_u4[r * 16 + chunk]) : make_uint4(0u, 0u, 0u, 0u);
            }
            #pragma unroll
            for (int it = 0; it < 4; ++it) {
                float kv[8];
                unpack_u4_bf16(kr[it], kv);
                #pragma unroll
                for (int j = 0; j < 8; ++j) ws[j] += kv[j];
            }
        }
        #pragma unroll
        for (int j = 0; j < 8; ++j) ws[j] += __shfl_xor_sync(0xffffffff, ws[j], 16);
        if (sub_r == 0) {
            #pragma unroll
            for (int j = 0; j < 8; ++j) warp_sums[warp_id * 128 + chunk * 8 + j] = ws[j];
        }
        __syncthreads();

        if (bi >= TOP_N) {
            float acc[8] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
            #pragma unroll
            for (int u = 0; u < 4; ++u) {
                if (u < warp_id) {
                    #pragma unroll
                    for (int j = 0; j < 8; ++j) acc[j] += warp_sums[u * 128 + chunk * 8 + j];
                }
            }
            #pragma unroll
            for (int half = 0; half < 2; ++half) {
                uint4 kr[4], qr[4];
                #pragma unroll
                for (int it = 0; it < 4; ++it) {
                    int r = r_beg + (half * 4 + it) * 2 + sub_r;
                    kr[it] = (r < rows) ? __ldg(&gK_u4[r * 16 + chunk]) : make_uint4(0u, 0u, 0u, 0u);
                    qr[it] = (r < rows) ? __ldg(&gQ_u4[r * 16 + chunk]) : make_uint4(0u, 0u, 0u, 0u);
                }
                #pragma unroll
                for (int it = 0; it < 4; ++it) {
                    float kv[8], qv[8];
                    unpack_u4_bf16(kr[it], kv);
                    unpack_u4_bf16(qr[it], qv);
                    float dot = 0.0f;
                    #pragma unroll
                    for (int j = 0; j < 8; ++j) {
                        float x = kv[j];
                        float s1 = __shfl_up_sync(0xffffffff, x, 16);
                        if (sub_r == 1) x += s1;
                        dot = fmaf(acc[j] + x, qv[j], dot);
                        acc[j] += __shfl_sync(0xffffffff, x, 16 + chunk);
                    }
                    dot += __shfl_xor_sync(0xffffffff, dot, 1);
                    dot += __shfl_xor_sync(0xffffffff, dot, 2);
                    dot += __shfl_xor_sync(0xffffffff, dot, 4);
                    dot += __shfl_xor_sync(0xffffffff, dot, 8);
                    int r = r_beg + (half * 4 + it) * 2 + sub_r;
                    if (chunk == 0 && r < rows) {
                        smem_imp[r] = dot * (scale * __frcp_rn(float(r + 1)));
                    }
                }
            }
            __syncwarp();
            if (lane < r_end - r_beg) {
                imp_diag[int64_t(bh) * S + row_start + r_beg + lane] = smem_imp[r_beg + lane];
            }
        }
    }

    if (bi < n_blocks - 1 && tid < D) {
        int64_t km_stride = int64_t(n_blocks) * D;
        int64_t km_base = (int64_t(bh) * 3 * n_blocks + bi) * D + tid;
        float total = warp_sums[0 * D + tid] + warp_sums[1 * D + tid]
                    + warp_sums[2 * D + tid] + warp_sums[3 * D + tid];
        float v = total * __frcp_rn(float(rows));
        __nv_bfloat16 h1 = __float2bfloat16_rn(v);
        float r1 = v - __bfloat162float(h1);
        __nv_bfloat16 h2 = __float2bfloat16_rn(r1);
        float r2 = r1 - __bfloat162float(h2);
        __nv_bfloat16 h3 = __float2bfloat16_rn(r2);
        K_h[km_base]                 = h1;
        K_h[km_base + km_stride]     = h2;
        K_h[km_base + 2 * km_stride] = h3;
    }
}

// -----------------------------------------------------------------------------
// Zero-SMEM Register-Direct Exact 3xBF16 Block Scoring -> Register 4-Thread Heaps
// Single unified K_h pointer ([BH, 3, nb, D]) saves 4 pointer registers!
// -----------------------------------------------------------------------------
template <int D>
__device__ __forceinline__ void score_3bf16_to_reg_heaps(
    uint64_t& h0, uint64_t& h1, uint64_t& h2, uint64_t& h3,
    uint64_t& h4, uint64_t& h5, uint64_t& h6, uint64_t& h7,
    uint64_t& g0, uint64_t& g1, uint64_t& g2, uint64_t& g3,
    uint64_t& g4, uint64_t& g5, uint64_t& g6, uint64_t& g7,
    uint4* K_dbl_smem,
    const uint4* Q_smem,
    const __nv_bfloat16* K_h_head,
    int n_blocks,
    int bi_cur,
    float scale,
    int tid,
    int warp_id,
    int lane
) {
    constexpr int CHUNKS_PER_ROW = D / 8;
    constexpr int TOTAL_CHUNKS = 32 * CHUNKS_PER_ROW;
    int cur_buf = 0;

    int q_row = warp_id * 16 + (lane & 15);
    int k_row_sub = ((lane >> 4) << 3) + (lane & 7);
    int tid_g = lane & 3;
    constexpr int K_STEPS = D / 16;
    int kh_stride_u4 = n_blocks * CHUNKS_PER_ROW;
    const uint4* kh_u4 = reinterpret_cast<const uint4*>(K_h_head);

    auto load_buf = [&](int buf_id, int plane_off_u4, int row_off) {
        uint4* dst_buf = K_dbl_smem + (buf_id ? TOTAL_CHUNKS : 0);
        const uint4* src_u4 = kh_u4 + plane_off_u4;
        #pragma unroll
        for (int idx = tid; idx < TOTAL_CHUNKS; idx += 128) {
            int r = idx / CHUNKS_PER_ROW;
            int c = idx % CHUNKS_PER_ROW;
            int gb = row_off + r;
            bool valid = (gb < bi_cur);
            const uint4* sp = src_u4 + (valid ? (gb * CHUNKS_PER_ROW + c) : 0);
            cp_async_16_pred(&dst_buf[swz_chunk_idx<D>(r, c)], sp, valid);
        }
        cp_async_commit();
    };

    auto mma_buf = [&](float s[4][4], int buf_id) {
        const uint4* kbuf = K_dbl_smem + (buf_id ? TOTAL_CHUNKS : 0);
        int q_k_sub = (lane >> 4);
        int k_k_sub = (lane >> 3) & 1;
        #pragma unroll
        for (int ks = 0; ks < K_STEPS; ++ks) {
            int k_chunk = ks * 2;
            uint32_t a0, a1, a2, a3, b0, b1, b2, b3, b4, b5, b6, b7;
            ldmatrix_x4(a0, a1, a2, a3, &Q_smem[swz_chunk_idx<D>(q_row, k_chunk + q_k_sub)]);
            ldmatrix_x4(b0, b1, b2, b3, &kbuf[swz_chunk_idx<D>(k_row_sub, k_chunk + k_k_sub)]);
            ldmatrix_x4(b4, b5, b6, b7, &kbuf[swz_chunk_idx<D>(16 + k_row_sub, k_chunk + k_k_sub)]);
            mma_m16n8k16_bf16(s[0][0], s[0][1], s[0][2], s[0][3], a0, a1, a2, a3, b0, b1);
            mma_m16n8k16_bf16(s[1][0], s[1][1], s[1][2], s[1][3], a0, a1, a2, a3, b2, b3);
            mma_m16n8k16_bf16(s[2][0], s[2][1], s[2][2], s[2][3], a0, a1, a2, a3, b4, b5);
            mma_m16n8k16_bf16(s[3][0], s[3][1], s[3][2], s[3][3], a0, a1, a2, a3, b6, b7);
        }
    };

    load_buf(0, 0, 0);

    for (int bi_tile = 0; bi_tile < bi_cur; bi_tile += 32) {
        float s[4][4];
        #pragma unroll
        for (int ni = 0; ni < 4; ++ni) {
            s[ni][0] = 0.0f; s[ni][1] = 0.0f; s[ni][2] = 0.0f; s[ni][3] = 0.0f;
        }

        load_buf(cur_buf ^ 1, kh_stride_u4, bi_tile);
        cp_async_wait<1>();
        __syncthreads();
        mma_buf(s, cur_buf);
        __syncthreads();

        load_buf(cur_buf, 2 * kh_stride_u4, bi_tile);
        cp_async_wait<1>();
        __syncthreads();
        mma_buf(s, cur_buf ^ 1);

        cp_async_wait<0>();
        __syncthreads();
        bool has_next = (bi_tile + 32 < bi_cur);
        if (has_next) {
            load_buf(cur_buf ^ 1, 0, bi_tile + 32);
        }
        mma_buf(s, cur_buf);

        int b_base = bi_tile + tid_g * 2;
        auto sc_key = [&](float v, int bi) -> uint64_t {
            if constexpr (D == 64) {
                return (bi < bi_cur) ? make_score_key(v, bi) : 0ULL;
            } else {
                return (bi < bi_cur) ? make_score_key(v * scale, bi) : 0ULL;
            }
        };

        if (bi_tile == 0) {
            h0 = sc_key(s[0][0], b_base + 0);  h1 = sc_key(s[0][1], b_base + 1);
            h2 = sc_key(s[1][0], b_base + 8);  h3 = sc_key(s[1][1], b_base + 9);
            h4 = sc_key(s[2][0], b_base + 16); h5 = sc_key(s[2][1], b_base + 17);
            h6 = sc_key(s[3][0], b_base + 24); h7 = sc_key(s[3][1], b_base + 25);

            g0 = sc_key(s[0][2], b_base + 0);  g1 = sc_key(s[0][3], b_base + 1);
            g2 = sc_key(s[1][2], b_base + 8);  g3 = sc_key(s[1][3], b_base + 9);
            g4 = sc_key(s[2][2], b_base + 16); g5 = sc_key(s[2][3], b_base + 17);
            g6 = sc_key(s[3][2], b_base + 24); g7 = sc_key(s[3][3], b_base + 25);
            if (has_next) {
                heap8_build(h0, h1, h2, h3, h4, h5, h6, h7);
                heap8_build(g0, g1, g2, g3, g4, g5, g6, g7);
            }
        } else {
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[0][0], b_base + 0));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[0][1], b_base + 1));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[1][0], b_base + 8));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[1][1], b_base + 9));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[2][0], b_base + 16));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[2][1], b_base + 17));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[3][0], b_base + 24));
            heap8_push(h0, h1, h2, h3, h4, h5, h6, h7, sc_key(s[3][1], b_base + 25));

            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[0][2], b_base + 0));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[0][3], b_base + 1));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[1][2], b_base + 8));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[1][3], b_base + 9));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[2][2], b_base + 16));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[2][3], b_base + 17));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[3][2], b_base + 24));
            heap8_push(g0, g1, g2, g3, g4, g5, g6, g7, sc_key(s[3][3], b_base + 25));
        }

        if (has_next) {
            __syncthreads(); // Ensure all warps done with cur_buf before next iter loads K_h2 into it!
            cur_buf ^= 1;
        }
    }
}

// -----------------------------------------------------------------------------
// Helper: Fused-FMA Online Softmax Prep
// -----------------------------------------------------------------------------
template <int D, bool DO_COL_MASK>
__device__ __forceinline__ void online_softmax_prep(
    float o[D / 8][4],
    uint32_t p_lo[8],
    uint32_t p_hi[8],
    float& m0, float& l0, bool& has0,
    float& m1, float& l1, bool& has1,
    float s[8][4],
    bool keep0, bool keep1,
    int r0, int r1,
    bool is_diag_block,
    bool full_prev0, bool full_prev1,
    float scale_log2e,
    int lane
) {
    int tid_g = lane & 3;
    float mb0 = -1e30f, mb1 = -1e30f;

    #pragma unroll
    for (int ni = 0; ni < 8; ++ni) {
        if constexpr (DO_COL_MASK) {
            int c0 = ni * 8 + tid_g * 2;
            int c1 = c0 + 1;
            if (is_diag_block) {
                if (c0 > r0) s[ni][0] = -1e30f;
                if (c1 > r0) s[ni][1] = -1e30f;
                if (c0 > r1) s[ni][2] = -1e30f;
                if (c1 > r1) s[ni][3] = -1e30f;
            } else {
                if (!full_prev0) {
                    if (c0 <= r0) s[ni][0] = -1e30f;
                    if (c1 <= r0) s[ni][1] = -1e30f;
                }
                if (!full_prev1) {
                    if (c0 <= r1) s[ni][2] = -1e30f;
                    if (c1 <= r1) s[ni][3] = -1e30f;
                }
            }
        }
        mb0 = fmaxf(mb0, fmaxf(s[ni][0], s[ni][1]));
        mb1 = fmaxf(mb1, fmaxf(s[ni][2], s[ni][3]));
    }

    mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 1));
    mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 2));
    mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 1));
    mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 2));

    bool eff0 = keep0 && (mb0 > -1e29f);
    bool eff1 = keep1 && (mb1 > -1e29f);
    float mb0_s = mb0 * scale_log2e;
    float mb1_s = mb1 * scale_log2e;

    float m0_new = eff0 ? (has0 ? fmaxf(m0, mb0_s) : mb0_s) : m0;
    float m1_new = eff1 ? (has1 ? fmaxf(m1, mb1_s) : mb1_s) : m1;

    float rescale0 = (eff0 && has0) ? fast_exp2f(m0 - m0_new) : 1.0f;
    float rescale1 = (eff1 && has1) ? fast_exp2f(m1 - m1_new) : 1.0f;

    if (eff0 && has0 && rescale0 != 1.0f) {
        #pragma unroll
        for (int di = 0; di < D / 8; ++di) {
            o[di][0] *= rescale0;
            o[di][1] *= rescale0;
        }
        l0 *= rescale0;
    }
    if (eff1 && has1 && rescale1 != 1.0f) {
        #pragma unroll
        for (int di = 0; di < D / 8; ++di) {
            o[di][2] *= rescale1;
            o[di][3] *= rescale1;
        }
        l1 *= rescale1;
    }

    float lb0 = 0.0f, lb1 = 0.0f;
    float neg_m0 = -m0_new;
    float neg_m1 = -m1_new;
    #pragma unroll
    for (int ni = 0; ni < 8; ++ni) {
        if (eff0) {
            float p0, p1;
            if constexpr (DO_COL_MASK) {
                p0 = (s[ni][0] > -1e29f) ? fast_exp2f(fmaf(s[ni][0], scale_log2e, neg_m0)) : 0.0f;
                p1 = (s[ni][1] > -1e29f) ? fast_exp2f(fmaf(s[ni][1], scale_log2e, neg_m0)) : 0.0f;
            } else {
                p0 = fast_exp2f(fmaf(s[ni][0], scale_log2e, neg_m0));
                p1 = fast_exp2f(fmaf(s[ni][1], scale_log2e, neg_m0));
            }
            lb0 += p0 + p1;
            p_lo[ni] = pack_bf16x2(p0, p1);
        } else {
            p_lo[ni] = 0u;
        }
        if (eff1) {
            float p2, p3;
            if constexpr (DO_COL_MASK) {
                p2 = (s[ni][2] > -1e29f) ? fast_exp2f(fmaf(s[ni][2], scale_log2e, neg_m1)) : 0.0f;
                p3 = (s[ni][3] > -1e29f) ? fast_exp2f(fmaf(s[ni][3], scale_log2e, neg_m1)) : 0.0f;
            } else {
                p2 = fast_exp2f(fmaf(s[ni][2], scale_log2e, neg_m1));
                p3 = fast_exp2f(fmaf(s[ni][3], scale_log2e, neg_m1));
            }
            lb1 += p2 + p3;
            p_hi[ni] = pack_bf16x2(p2, p3);
        } else {
            p_hi[ni] = 0u;
        }
    }

    lb0 += __shfl_xor_sync(0xffffffff, lb0, 1);
    lb0 += __shfl_xor_sync(0xffffffff, lb0, 2);
    lb1 += __shfl_xor_sync(0xffffffff, lb1, 1);
    lb1 += __shfl_xor_sync(0xffffffff, lb1, 2);
    if (eff0) {
        l0 = has0 ? (l0 + lb0) : lb0;
        m0 = m0_new;
        has0 = true;
    }
    if (eff1) {
        l1 = has1 ? (l1 + lb1) : lb1;
        m1 = m1_new;
        has1 = true;
    }
}

template <int D, bool DO_NORM>
__device__ __forceinline__ void store_O_bf16_via_smem(
    __nv_bfloat16* O_head,
    uint4* smem_stage,
    float o[D / 8][4],
    float l0, float l1,
    int bi_cur, int S,
    int warp_id, int lane
) {
    float inv_l0 = DO_NORM ? ((l0 > 0.0f) ? __frcp_rn(l0) : 0.0f) : 1.0f;
    float inv_l1 = DO_NORM ? ((l1 > 0.0f) ? __frcp_rn(l1) : 0.0f) : 1.0f;

    __nv_bfloat162* s2 = reinterpret_cast<__nv_bfloat162*>(smem_stage);
    int gid = lane >> 2;
    int tid_g = lane & 3;
    int r0 = warp_id * 16 + gid;
    int r1 = r0 + 8;

    #pragma unroll
    for (int di = 0; di < D / 8; ++di) {
        int swz0 = (di ^ (r0 & 7)) * 4 + tid_g;
        int swz1 = (di ^ (r1 & 7)) * 4 + tid_g;
        if constexpr (DO_NORM) {
            s2[r0 * (D / 2) + swz0] = __floats2bfloat162_rn(o[di][0] * inv_l0, o[di][1] * inv_l0);
            s2[r1 * (D / 2) + swz1] = __floats2bfloat162_rn(o[di][2] * inv_l1, o[di][3] * inv_l1);
        } else {
            s2[r0 * (D / 2) + swz0] = __floats2bfloat162_rn(o[di][0], o[di][1]);
            s2[r1 * (D / 2) + swz1] = __floats2bfloat162_rn(o[di][2], o[di][3]);
        }
    }
    __syncwarp();

    constexpr int CHUNKS_PER_ROW = D / 8;
    constexpr int WARP_CHUNKS = 16 * CHUNKS_PER_ROW;
    uint4* dst_u4 = reinterpret_cast<uint4*>(O_head);
    int base_row = bi_cur * BLOCK_SIZE + warp_id * 16;

    #pragma unroll
    for (int idx = lane; idx < WARP_CHUNKS; idx += 32) {
        int r = idx / CHUNKS_PER_ROW;
        int c = idx % CHUNKS_PER_ROW;
        int gr = base_row + r;
        if (gr < S) {
            int r_smem = warp_id * 16 + r;
            __stcs(&dst_u4[int64_t(gr) * CHUNKS_PER_ROW + c], smem_stage[swz_chunk_idx<D>(r_smem, c)]);
        }
    }
}

// =============================================================================
// PATH A (D=64): 2-Warp-Group Pipelined BN=16 Exact 3xBF16 Split-KV (n_blocks <= 16)
// Early K_next prefetch + folded LPT wave balancing + unmasked bi_cur < TOP_N
// =============================================================================
__global__ __launch_bounds__(256, 2) void fused_nsa_direct_bn16_splitkv_64(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    const __nv_bfloat16* __restrict__ K_h,
    const float* __restrict__ imp_diag,
    __nv_bfloat16* __restrict__ O,
    int BH,
    int S,
    int n_blocks,
    float scale
) {
    extern __shared__ char smem_raw[];
    constexpr int D = 64;
    constexpr int TILE_U4 = BLOCK_SIZE * (D / 8);
    uint4* Q_smem  = reinterpret_cast<uint4*>(smem_raw);
    uint4* K0_smem = Q_smem + TILE_U4;
    uint4* V0_smem = K0_smem + TILE_U4;
    uint4* K1_smem = V0_smem + TILE_U4;
    uint4* V1_smem = K1_smem + TILE_U4;

    int total_ctas = BH * n_blocks;
    int lin = blockIdx.x;
    int w1_end = min(376, total_ctas);
    int idx_lpt = (lin >= 188 && lin < w1_end) ? (188 + (w1_end - 1 - lin)) : lin;
    int bi_cur = (n_blocks - 1) - (idx_lpt / BH);
    int bh = idx_lpt % BH;
    int tid = threadIdx.x;
    int wg = tid >> 7;
    int ltid = tid & 127;
    int warp_id = ltid >> 5;
    int lane = ltid & 31;
    int gid = lane >> 2;
    int tid_g = lane & 3;
    int r0 = warp_id * 16 + gid;
    int r1 = r0 + 8;

    const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D;
    const __nv_bfloat16* K_head = K + int64_t(bh) * S * D;
    const __nv_bfloat16* V_head = V + int64_t(bh) * S * D;
    __nv_bfloat16* O_head = O + int64_t(bh) * S * D;

    load_tile_swz_async<D, BLOCK_SIZE>(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 256);

    uint32_t msk0 = 0u, msk1 = 0u;
    bool fp0 = false, fp1 = false;
    bool has_first = false;
    int bi_first = 0;
    uint32_t my_rem = 0u;

    if (bi_cur < TOP_N) {
        // Overlap K[0], K[1] in Group 0 with Q_smem, and V[0], V[1] in Group 1 -> ZERO startup stall!
        if (wg == 0) {
            load_tile_swz_async<D, BLOCK_SIZE>(K0_smem, K_head, 0, S, ltid, 128);
        } else if (bi_cur >= 1) {
            load_tile_swz_async<D, BLOCK_SIZE>(K1_smem, K_head, BLOCK_SIZE, S, ltid, 128);
        }
        cp_async_commit();
        if (wg == 0) {
            load_tile_swz_async<D, BLOCK_SIZE>(V0_smem, V_head, 0, S, ltid, 128);
        } else if (bi_cur >= 1) {
            load_tile_swz_async<D, BLOCK_SIZE>(V1_smem, V_head, BLOCK_SIZE, S, ltid, 128);
        }
        cp_async_commit();
        cp_async_wait<1>();
        __syncthreads();

        uint32_t all_m = (1u << (bi_cur + 1)) - 1u;
        msk0 = all_m; msk1 = all_m;
        fp0 = true; fp1 = true;
        if (wg == 0) {
            has_first = true;
            bi_first = 0;
            my_rem = all_m & 0x55555554u; // even blocks >= 2: 2, 4, 6
        } else if (bi_cur >= 1) {
            has_first = true;
            bi_first = 1;
            my_rem = all_m & 0xAAAAAAA8u; // odd blocks >= 3: 3, 5, 7
        }
    } else {
        int kh_stride_u4 = n_blocks * 8;
        const uint4* kh_u4 = reinterpret_cast<const uint4*>(K_h + int64_t(bh) * 3 * n_blocks * D);
        // All 256 threads load K_h1/h2/h3 into K0_smem (Group 0 alongside Q_smem)
        #pragma unroll
        for (int idx = tid; idx < 3 * 128; idx += 256) {
            int plane = idx >> 7;
            int off = idx & 127;
            int r = off >> 3, c = off & 7;
            bool valid = (r < bi_cur);
            const uint4* sp = kh_u4 + plane * kh_stride_u4 + (valid ? off : 0);
            cp_async_16_pred(&K0_smem[swz_chunk_idx<D>(plane * 16 + r, c)], sp, valid);
        }
        cp_async_commit();
        // Simultaneously prefetch guaranteed sliding-window tiles V[bi_cur-1], K[bi_cur], V[bi_cur] in Group 1!
        if (wg == 0) {
            load_tile_swz_async<D, BLOCK_SIZE>(V0_smem, V_head, (bi_cur - 1) * BLOCK_SIZE, S, ltid, 128);
        } else {
            load_tile_swz_async<D, BLOCK_SIZE>(K1_smem, K_head, bi_cur * BLOCK_SIZE, S, ltid, 128);
            load_tile_swz_async<D, BLOCK_SIZE>(V1_smem, V_head, bi_cur * BLOCK_SIZE, S, ltid, 128);
        }
        cp_async_commit();
        cp_async_wait<1>();
        __syncthreads();

        int q_row = warp_id * 16 + (lane & 15);
        int q_k_sub = (lane >> 4);
        int k_row_sub = ((lane >> 4) << 3) + (lane & 7);
        int k_k_sub = (lane >> 3) & 1;

        float sc00=0,sc01=0,sc02=0,sc03=0, sc10=0,sc11=0,sc12=0,sc13=0;
        #pragma unroll
        for (int ks = 0; ks < 4; ++ks) {
            int k_chunk = ks * 2;
            uint32_t a0,a1,a2,a3, b0,b1,b2,b3;
            ldmatrix_x4(a0,a1,a2,a3, &Q_smem[swz_chunk_idx<D>(q_row, k_chunk + q_k_sub)]);
            ldmatrix_x4(b0,b1,b2,b3, &K0_smem[swz_chunk_idx<D>(k_row_sub, k_chunk + k_k_sub)]);
            mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1);
            mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3);
            ldmatrix_x4(b0,b1,b2,b3, &K0_smem[swz_chunk_idx<D>(16+k_row_sub, k_chunk + k_k_sub)]);
            mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1);
            mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3);
            ldmatrix_x4(b0,b1,b2,b3, &K0_smem[swz_chunk_idx<D>(32+k_row_sub, k_chunk + k_k_sub)]);
            mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1);
            mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3);
        }

        // K0_smem is now free! Overlap K[bi_cur-1] load behind all 16 pop_max4_5 Top-8 iterations!
        __syncthreads();
        load_tile_swz_async<D, BLOCK_SIZE>(K0_smem, K_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 256);
        cp_async_commit();

        int b_base = tid_g * 2;
        uint64_t a0 = (b_base+0 < bi_cur) ? make_score_key(sc00, b_base+0) : 0ULL;
        uint64_t a1 = (b_base+1 < bi_cur) ? make_score_key(sc01, b_base+1) : 0ULL;
        uint64_t a2 = (b_base+8 < bi_cur) ? make_score_key(sc10, b_base+8) : 0ULL;
        uint64_t a3 = (b_base+9 < bi_cur) ? make_score_key(sc11, b_base+9) : 0ULL;
        uint64_t g0 = (b_base+0 < bi_cur) ? make_score_key(sc02, b_base+0) : 0ULL;
        uint64_t g1 = (b_base+1 < bi_cur) ? make_score_key(sc03, b_base+1) : 0ULL;
        uint64_t g2 = (b_base+8 < bi_cur) ? make_score_key(sc12, b_base+8) : 0ULL;
        uint64_t g3 = (b_base+9 < bi_cur) ? make_score_key(sc13, b_base+9) : 0ULL;

        const float* imp_head = imp_diag + int64_t(bh) * S;
        int t0 = bi_cur * BLOCK_SIZE + r0, t1 = bi_cur * BLOCK_SIZE + r1;
        bool v0 = (t0 < S), v1 = (t1 < S);
        uint64_t d0 = (tid_g == 0 && v0) ? make_score_key(imp_head[t0] * 8.0f, bi_cur) : 0ULL;
        uint64_t d1 = (tid_g == 0 && v1) ? make_score_key(imp_head[t1] * 8.0f, bi_cur) : 0ULL;

        msk0 = v0 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u;
        msk1 = v1 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u;
        #pragma unroll
        for (int u = 0; u < TOP_N; ++u) {
            uint64_t w0 = pop_max4_5(a0, a1, a2, a3, d0);
            if (v0 && w0 != 0ULL) { int b = int(w0 & 0xFFFFFFFFu); msk0 |= (1u << b); if (b == bi_cur - 1) fp0 = true; }
            uint64_t w1 = pop_max4_5(g0, g1, g2, g3, d1);
            if (v1 && w1 != 0ULL) { int b = int(w1 & 0xFFFFFFFFu); msk1 |= (1u << b); if (b == bi_cur - 1) fp1 = true; }
        }
        uint32_t my_u = msk0 | msk1;
        #pragma unroll
        for (int off = 16; off > 0; off >>= 1) my_u |= __shfl_xor_sync(0xffffffff, my_u, off);
        __shared__ uint32_t s_um[4];
        if (wg == 0 && lane == 0) s_um[warp_id] = my_u;
        cp_async_wait<0>();
        __syncthreads();
        uint32_t rem_mask = s_um[0] | s_um[1] | s_um[2] | s_um[3];

        // wg=0 starts on bi_cur-1 (already in K0/V0!), wg=1 starts on bi_cur (already in K1/V1!)
        has_first = true;
        bi_first = (wg == 0) ? (bi_cur - 1) : bi_cur;
        uint32_t extra = rem_mask & ~((1u << bi_cur) | (1u << (bi_cur - 1)));
        int parity = 0;
        while (extra != 0u) {
            int b = __ffs(extra) - 1;
            extra &= (extra - 1u);
            if ((parity & 1) == wg) my_rem |= (1u << b);
            parity++;
        }
    }

    uint4* K_my = (wg == 0) ? K0_smem : K1_smem;
    uint4* V_my = (wg == 0) ? V0_smem : V1_smem;

    float o[D / 8][4];
    #pragma unroll
    for (int di = 0; di < D / 8; ++di) { o[di][0]=0; o[di][1]=0; o[di][2]=0; o[di][3]=0; }
    float m0 = -1e30f, l0 = 0.0f; bool has0 = false;
    float m1 = -1e30f, l1 = 0.0f; bool has1 = false;
    const float scale_log2e = scale * 1.4426950408889634f;

    if (has_first) {
        int bi = bi_first;
        while (true) {
            float s[8][4];
            warp_gemm_qk_16x64<D>(s, Q_smem, K_my, warp_id, lane);

            bool has_next = (my_rem != 0u);
            int bi_next = 0;
            if (has_next) {
                bar_sync_wg(wg);
                bi_next = __ffs(my_rem) - 1;
                my_rem &= (my_rem - 1u);
                load_tile_swz_async<D, BLOCK_SIZE>(K_my, K_head, bi_next * BLOCK_SIZE, S, ltid, 128);
                cp_async_commit();
            }

            uint32_t p_lo[8], p_hi[8];
            bool keep0 = ((msk0 >> bi) & 1u) != 0u;
            bool keep1 = ((msk1 >> bi) & 1u) != 0u;
            bool need_mask = (bi == bi_cur) || (bi == bi_cur - 1 && bi_cur >= TOP_N);
            if (!need_mask) {
                online_softmax_prep<D, false>(
                    o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
                    s, keep0, keep1, r0, r1, false, false, false, scale_log2e, lane
                );
            } else {
                online_softmax_prep<D, true>(
                    o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
                    s, keep0, keep1, r0, r1, (bi == bi_cur), fp0, fp1, scale_log2e, lane
                );
            }

            if (has_next) {
                cp_async_wait<1>();
            } else {
                cp_async_wait<0>();
            }
            bar_sync_wg(wg);

            warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_my, lane);
            if (!has_next) break;
            cp_async_wait<0>();
            bar_sync_wg(wg);
            load_tile_swz_async<D, BLOCK_SIZE>(V_my, V_head, bi_next * BLOCK_SIZE, S, ltid, 128);
            cp_async_commit();
            bi = bi_next;
        }
    }

    if (bi_cur == 0) {
        if (wg == 0) store_O_bf16_via_smem<D, true>(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane);
        return;
    }

    __syncthreads();
    float* O1_smem = reinterpret_cast<float*>(K0_smem);
    float2* ml1_smem = reinterpret_cast<float2*>(K1_smem);
    if (wg == 1) {
        #pragma unroll
        for (int di = 0; di < D / 8; ++di) {
            int col0 = (di ^ (r0 & 7)) * 8 + tid_g * 2;
            int col1 = (di ^ (r1 & 7)) * 8 + tid_g * 2;
            O1_smem[r0 * D + col0]     = o[di][0];
            O1_smem[r0 * D + col0 + 1] = o[di][1];
            O1_smem[r1 * D + col1]     = o[di][2];
            O1_smem[r1 * D + col1 + 1] = o[di][3];
        }
        if (tid_g == 0) {
            ml1_smem[r0] = make_float2(m0, has0 ? l0 : 0.0f);
            ml1_smem[r1] = make_float2(m1, has1 ? l1 : 0.0f);
        }
    }
    __syncthreads();

    if (wg == 0) {
        float2 ml0_1 = ml1_smem[r0];
        float2 ml1_1 = ml1_smem[r1];
        bool h0_1 = (ml0_1.y > 0.0f);
        bool h1_1 = (ml1_1.y > 0.0f);
        float m0_fin = has0 ? (h0_1 ? fmaxf(m0, ml0_1.x) : m0) : ml0_1.x;
        float m1_fin = has1 ? (h1_1 ? fmaxf(m1, ml1_1.x) : m1) : ml1_1.x;
        float s0_0 = has0 ? fast_exp2f(m0 - m0_fin) : 0.0f;
        float s0_1 = h0_1 ? fast_exp2f(ml0_1.x - m0_fin) : 0.0f;
        float s1_0 = has1 ? fast_exp2f(m1 - m1_fin) : 0.0f;
        float s1_1 = h1_1 ? fast_exp2f(ml1_1.x - m1_fin) : 0.0f;
        float l0_fin = l0 * s0_0 + ml0_1.y * s0_1;
        float l1_fin = l1 * s1_0 + ml1_1.y * s1_1;
        #pragma unroll
        for (int di = 0; di < D / 8; ++di) {
            int col0 = (di ^ (r0 & 7)) * 8 + tid_g * 2;
            int col1 = (di ^ (r1 & 7)) * 8 + tid_g * 2;
            o[di][0] = o[di][0] * s0_0 + O1_smem[r0 * D + col0]     * s0_1;
            o[di][1] = o[di][1] * s0_0 + O1_smem[r0 * D + col0 + 1] * s0_1;
            o[di][2] = o[di][2] * s1_0 + O1_smem[r1 * D + col1]     * s1_1;
            o[di][3] = o[di][3] * s1_0 + O1_smem[r1 * D + col1 + 1] * s1_1;
        }
        store_O_bf16_via_smem<D, true>(O_head, Q_smem, o, l0_fin, l1_fin, bi_cur, S, warp_id, lane);
    }
}

// PATH A (D=128 edge-shape support)
__global__ __launch_bounds__(128, 2) void fused_nsa_direct_bn16_128(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    const __nv_bfloat16* __restrict__ K_h,
    const float* __restrict__ imp_diag,
    __nv_bfloat16* __restrict__ O,
    int S,
    int n_blocks,
    float scale
) {
    extern __shared__ char smem_raw[];
    constexpr int D = 128;
    constexpr int TILE_U4 = BLOCK_SIZE * (D / 8);
    uint4* Q_smem = reinterpret_cast<uint4*>(smem_raw);
    uint4* K_smem = Q_smem + TILE_U4;
    uint4* V_smem = K_smem + TILE_U4;

    int bh = blockIdx.x;
    int bi_cur = (n_blocks - 1) - blockIdx.y;
    int tid = threadIdx.x;
    int warp_id = tid >> 5;
    int lane = tid & 31;
    int gid = lane >> 2;
    int tid_g = lane & 3;
    int r0 = warp_id * 16 + gid;
    int r1 = r0 + 8;

    const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D;
    const __nv_bfloat16* K_head = K + int64_t(bh) * S * D;
    const __nv_bfloat16* V_head = V + int64_t(bh) * S * D;
    __nv_bfloat16* O_head = O + int64_t(bh) * S * D;

    load_tile_swz_async<D, BLOCK_SIZE>(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 128);
    if (bi_cur < TOP_N) {
        load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, 0, S, tid, 128);
        cp_async_commit();
        load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, 0, S, tid, 128);
        cp_async_commit();
        cp_async_wait<1>();
        __syncthreads();

        float o[D / 8][4];
        #pragma unroll
        for (int di = 0; di < D / 8; ++di) { o[di][0]=0; o[di][1]=0; o[di][2]=0; o[di][3]=0; }
        float m0 = -1e30f, l0 = 0.0f; bool has0 = false;
        float m1 = -1e30f, l1 = 0.0f; bool has1 = false;
        const float scale_log2e = scale * 1.4426950408889634f;

        for (int bi = 0; bi <= bi_cur; ++bi) {
            float s[8][4];
            warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
            uint32_t p_lo[8], p_hi[8];
            if (bi < bi_cur) {
                online_softmax_prep<D, false>(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, false, true, true, scale_log2e, lane);
            } else {
                online_softmax_prep<D, true>(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, true, true, r0, r1, true, true, true, scale_log2e, lane);
            }
            cp_async_wait<0>();
            __syncthreads();
            if (bi + 1 <= bi_cur) { load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, (bi + 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); }
            warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);
            if (bi + 1 <= bi_cur) { cp_async_wait<0>(); __syncthreads(); load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, (bi + 1) * BLOCK_SIZE, S, tid, 128); cp_async_commit(); }
        }
        __syncthreads();
        store_O_bf16_via_smem<D, true>(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane);
        return;
    }

    int64_t kh_stride = int64_t(n_blocks) * D;
    const __nv_bfloat16* kh_base = K_h + int64_t(bh) * 3 * kh_stride;
    const uint4* kh1_u4 = reinterpret_cast<const uint4*>(kh_base);
    const uint4* kh2_u4 = reinterpret_cast<const uint4*>(kh_base + kh_stride);
    const uint4* kh3_u4 = reinterpret_cast<const uint4*>(kh_base + 2 * kh_stride);
    constexpr int HALF16_U4 = 16 * (D / 8);
    #pragma unroll
    for (int idx = tid; idx < HALF16_U4; idx += 128) {
        int r = idx / (D / 8), c = idx % (D / 8);
        bool valid = (r < bi_cur);
        int off = valid ? idx : 0;
        cp_async_16_pred(&K_smem[swz_chunk_idx<D>(r, c)],      &kh1_u4[off], valid);
        cp_async_16_pred(&K_smem[swz_chunk_idx<D>(16 + r, c)], &kh2_u4[off], valid);
        cp_async_16_pred(&K_smem[swz_chunk_idx<D>(32 + r, c)], &kh3_u4[off], valid);
    }
    cp_async_commit();
    cp_async_wait<0>();
    __syncthreads();

    int q_row = warp_id * 16 + (lane & 15);
    int q_k_sub = (lane >> 4);
    int k_row_sub = ((lane >> 4) << 3) + (lane & 7);
    int k_k_sub = (lane >> 3) & 1;

    float sc00=0,sc01=0,sc02=0,sc03=0, sc10=0,sc11=0,sc12=0,sc13=0;
    #pragma unroll
    for (int ks = 0; ks < D / 16; ++ks) {
        int k_chunk = ks * 2;
        uint32_t a0,a1,a2,a3, b0,b1,b2,b3;
        ldmatrix_x4(a0,a1,a2,a3, &Q_smem[swz_chunk_idx<D>(q_row, k_chunk + q_k_sub)]);
        ldmatrix_x4(b0,b1,b2,b3, &K_smem[swz_chunk_idx<D>(k_row_sub, k_chunk + k_k_sub)]);
        mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1);
        mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3);
        ldmatrix_x4(b0,b1,b2,b3, &K_smem[swz_chunk_idx<D>(16+k_row_sub, k_chunk + k_k_sub)]);
        mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1);
        mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3);
        ldmatrix_x4(b0,b1,b2,b3, &K_smem[swz_chunk_idx<D>(32+k_row_sub, k_chunk + k_k_sub)]);
        mma_m16n8k16_bf16(sc00,sc01,sc02,sc03, a0,a1,a2,a3, b0,b1);
        mma_m16n8k16_bf16(sc10,sc11,sc12,sc13, a0,a1,a2,a3, b2,b3);
    }

    int b_base = tid_g * 2;
    uint64_t a0 = (b_base+0 < bi_cur) ? make_score_key(sc00*scale, b_base+0) : 0ULL;
    uint64_t a1 = (b_base+1 < bi_cur) ? make_score_key(sc01*scale, b_base+1) : 0ULL;
    uint64_t a2 = (b_base+8 < bi_cur) ? make_score_key(sc10*scale, b_base+8) : 0ULL;
    uint64_t a3 = (b_base+9 < bi_cur) ? make_score_key(sc11*scale, b_base+9) : 0ULL;
    uint64_t g0 = (b_base+0 < bi_cur) ? make_score_key(sc02*scale, b_base+0) : 0ULL;
    uint64_t g1 = (b_base+1 < bi_cur) ? make_score_key(sc03*scale, b_base+1) : 0ULL;
    uint64_t g2 = (b_base+8 < bi_cur) ? make_score_key(sc12*scale, b_base+8) : 0ULL;
    uint64_t g3 = (b_base+9 < bi_cur) ? make_score_key(sc13*scale, b_base+9) : 0ULL;

    const float* imp_head = imp_diag + int64_t(bh) * S;
    int t0 = bi_cur * BLOCK_SIZE + r0, t1 = bi_cur * BLOCK_SIZE + r1;
    bool v0 = (t0 < S), v1 = (t1 < S);
    uint64_t d0 = (tid_g == 0 && v0) ? make_score_key(imp_head[t0], bi_cur) : 0ULL;
    uint64_t d1 = (tid_g == 0 && v1) ? make_score_key(imp_head[t1], bi_cur) : 0ULL;

    uint32_t msk0 = v0 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u;
    uint32_t msk1 = v1 ? ((1u << bi_cur) | (1u << (bi_cur - 1))) : 0u;
    bool fp0 = false, fp1 = false;
    #pragma unroll
    for (int u = 0; u < TOP_N; ++u) {
        uint64_t w0 = pop_max4_5(a0, a1, a2, a3, d0);
        if (v0 && w0 != 0ULL) { int b = int(w0 & 0xFFFFFFFFu); msk0 |= (1u << b); if (b == bi_cur - 1) fp0 = true; }
        uint64_t w1 = pop_max4_5(g0, g1, g2, g3, d1);
        if (v1 && w1 != 0ULL) { int b = int(w1 & 0xFFFFFFFFu); msk1 |= (1u << b); if (b == bi_cur - 1) fp1 = true; }
    }
    uint32_t my_u = msk0 | msk1;
    #pragma unroll
    for (int off = 16; off > 0; off >>= 1) my_u |= __shfl_xor_sync(0xffffffff, my_u, off);
    __shared__ uint32_t s_um[4];
    if (lane == 0) s_um[warp_id] = my_u;
    __syncthreads();
    uint32_t rem_mask = s_um[0] | s_um[1] | s_um[2] | s_um[3];

    int bi = __ffs(rem_mask) - 1;
    rem_mask &= (rem_mask - 1u);
    load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, bi * BLOCK_SIZE, S, tid, 128);
    cp_async_commit();
    load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, bi * BLOCK_SIZE, S, tid, 128);
    cp_async_commit();
    cp_async_wait<1>();
    __syncthreads();

    float o[D / 8][4];
    #pragma unroll
    for (int di = 0; di < D / 8; ++di) { o[di][0]=0; o[di][1]=0; o[di][2]=0; o[di][3]=0; }
    float m0 = -1e30f, l0 = 0.0f; bool has0 = false;
    float m1 = -1e30f, l1 = 0.0f; bool has1 = false;
    const float scale_log2e = scale * 1.4426950408889634f;

    while (true) {
        float s[8][4];
        warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
        uint32_t p_lo[8], p_hi[8];
        bool keep0 = ((msk0 >> bi) & 1u) != 0u;
        bool keep1 = ((msk1 >> bi) & 1u) != 0u;
        if (bi < bi_cur - 1) {
            online_softmax_prep<D, false>(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, keep0, keep1, r0, r1, false, false, false, scale_log2e, lane);
        } else {
            online_softmax_prep<D, true>(o, p_lo, p_hi, m0, l0, has0, m1, l1, has1, s, keep0, keep1, r0, r1, (bi == bi_cur), fp0, fp1, scale_log2e, lane);
        }
        cp_async_wait<0>();
        __syncthreads();
        bool has_next = (rem_mask != 0u);
        int bi_next = 0;
        if (has_next) {
            bi_next = __ffs(rem_mask) - 1;
            rem_mask &= (rem_mask - 1u);
            load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, bi_next * BLOCK_SIZE, S, tid, 128);
            cp_async_commit();
        }
        warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);
        if (!has_next) break;
        cp_async_wait<0>();
        __syncthreads();
        load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, bi_next * BLOCK_SIZE, S, tid, 128);
        cp_async_commit();
        bi = bi_next;
    }
    __syncthreads();
    store_O_bf16_via_smem<D, true>(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane);
}

// =============================================================================
// PATH B: 4-Kernel Lock-Free Inverted-Index Block-Centric Sparse FlashAttention
// =============================================================================

struct WorkChunk {
    uint16_t g;
    uint16_t bi;
    uint32_t start;
    uint32_t count;
};

// B-Kernel 2 (FUSED!): Exact 3xBF16 Top-8 + Lock-Free Direct q_list Scatter + Chunk Emission!
template <int D>
__device__ __forceinline__ void score_topk_and_scatter_body(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K_h,
    const float* __restrict__ imp_diag,
    uint8_t* __restrict__ remote_count,
    uint8_t* __restrict__ full_prev,
    uint32_t* __restrict__ bucket_count,
    uint32_t* __restrict__ q_list,
    WorkChunk* __restrict__ chunk_table,
    uint32_t* __restrict__ d_num_chunks,
    int bh_start,
    int S,
    int n_blocks,
    int q_stride,
    int chunk_q,
    float scale
) {
    extern __shared__ char smem_raw[];
    constexpr int Q_U4 = BLOCK_SIZE * (D / 8);
    uint4* Q_smem = reinterpret_cast<uint4*>(smem_raw);
    uint4* K_dbl_smem = Q_smem + Q_U4;

    int g = blockIdx.x;
    int bi_cur = (n_blocks - 1) - blockIdx.y;
    int bh = bh_start + g;
    int tid = threadIdx.x;
    int warp_id = tid >> 5;
    int lane = tid & 31;
    int gid = lane >> 2;
    int tid_g = lane & 3;
    int r0 = warp_id * 16 + gid;
    int r1 = r0 + 8;

    const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D;
    const __nv_bfloat16* kh_head = K_h + int64_t(bh) * 3 * n_blocks * D;

    // Issue Q_smem cp.async WITHOUT waiting: score_3bf16_to_reg_heaps commits Q_smem + K_h1[0]
    // in one batch and overlaps K_h2[0] behind their arrival!
    load_tile_swz_async<D, BLOCK_SIZE>(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 128);

    uint64_t h0, h1, h2, h3, h4, h5, h6, h7;
    uint64_t g0, g1, g2, g3, g4, g5, g6, g7;

    score_3bf16_to_reg_heaps<D>(
        h0, h1, h2, h3, h4, h5, h6, h7,
        g0, g1, g2, g3, g4, g5, g6, g7,
        K_dbl_smem, Q_smem, kh_head, n_blocks,
        bi_cur, scale, tid, warp_id, lane
    );

    const float* imp_head = imp_diag + int64_t(bh) * S;

    constexpr int MAX_SMEM_HIST = (D == 64) ? 3500 : 7500;
    bool use_smem_hist = (bi_cur <= MAX_SMEM_HIST);
    uint32_t* smem_rblp = reinterpret_cast<uint32_t*>(smem_raw);
    uint8_t*  smem_krem = reinterpret_cast<uint8_t*>(smem_rblp + BLOCK_SIZE * TOP_N);
    uint32_t* smem_hist = reinterpret_cast<uint32_t*>(smem_krem + 64);

    __syncthreads();
    if (use_smem_hist) {
        for (int b = tid; b < bi_cur; b += 128) smem_hist[b] = 0u;
        __syncthreads();
    }

    int t0 = bi_cur * BLOCK_SIZE + r0;
    int t1 = bi_cur * BLOCK_SIZE + r1;
    bool v0 = (t0 < S);
    bool v1 = (t1 < S);
    float d_mult = (D == 64) ? 8.0f : 1.0f;
    uint64_t d0 = (tid_g == 0 && v0) ? make_score_key(imp_head[t0] * d_mult, bi_cur) : 0ULL;
    uint64_t d1 = (tid_g == 0 && v1) ? make_score_key(imp_head[t1] * d_mult, bi_cur) : 0ULL;

    uint32_t* bc_head = bucket_count + int64_t(g) * n_blocks;
    uint32_t* ql_head = q_list + int64_t(g) * n_blocks * q_stride;
    uint32_t cq = static_cast<uint32_t>(chunk_q);

    auto emit_row_and_hist = [&](
        uint64_t& k0, uint64_t& k1, uint64_t& k2, uint64_t& k3,
        uint64_t& k4, uint64_t& k5, uint64_t& k6, uint64_t& k7,
        uint64_t& kd, int r_loc, int t, bool valid_t
    ) {
        uint8_t fp = 0;
        uint8_t k_rem = 0;
        #pragma unroll
        for (int u = 0; u < TOP_N; ++u) {
            uint64_t w = pop_max4_9(k0, k1, k2, k3, k4, k5, k6, k7, kd);
            if (tid_g == 0 && valid_t && w != 0ULL) {
                int b = static_cast<int>(w & 0xFFFFFFFFu);
                if (b == bi_cur - 1) {
                    fp = 1;
                } else if (b < bi_cur - 1) {
                    if (use_smem_hist) {
                        uint32_t lp = atomicAdd(&smem_hist[b], 1u);
                        smem_rblp[r_loc * TOP_N + k_rem] = (static_cast<uint32_t>(b) << 8) | lp;
                    } else {
                        uint32_t pos = atomicAdd(&bc_head[b], 1u);
                        ql_head[int64_t(b) * q_stride + pos] = (static_cast<uint32_t>(t) << 3) | k_rem;
                        uint32_t nc = pos + 1u;
                        if ((nc % cq) == 0u) {
                            uint32_t cid = atomicAdd(d_num_chunks, 1u);
                            chunk_table[cid] = WorkChunk{
                                static_cast<uint16_t>(g), static_cast<uint16_t>(b), nc - cq, cq
                            };
                        }
                    }
                    k_rem++;
                }
            }
        }
        if (tid_g == 0) {
            if (valid_t) {
                int64_t idx_gt = int64_t(g) * S + t;
                full_prev[idx_gt] = fp;
                remote_count[idx_gt] = k_rem;
            }
            if (use_smem_hist) {
                smem_krem[r_loc] = k_rem;
            }
        }
    };

    emit_row_and_hist(h0, h1, h2, h3, h4, h5, h6, h7, d0, r0, t0, v0);
    emit_row_and_hist(g0, g1, g2, g3, g4, g5, g6, g7, d1, r1, t1, v1);

    if (use_smem_hist) {
        __syncthreads();
        for (int b = tid; b < bi_cur; b += 128) {
            uint32_t c = smem_hist[b];
            if (c > 0u) {
                uint32_t old_c = atomicAdd(&bc_head[b], c);
                smem_hist[b] = old_c;
                uint32_t new_c = old_c + c;
                uint32_t k_first = (old_c / cq) + 1u;
                uint32_t k_last  = new_c / cq;
                for (uint32_t k = k_first; k <= k_last; ++k) {
                    uint32_t cid = atomicAdd(d_num_chunks, 1u);
                    chunk_table[cid] = WorkChunk{
                        static_cast<uint16_t>(g), static_cast<uint16_t>(b), (k - 1u) * cq, cq
                    };
                }
            }
        }
        __syncthreads();

        int base_t = bi_cur * BLOCK_SIZE;
        #pragma unroll
        for (int idx = tid; idx < BLOCK_SIZE * TOP_N; idx += 128) {
            int r = idx >> 3;
            int s = idx & 7;
            if (s < smem_krem[r]) {
                uint32_t rblp = smem_rblp[idx];
                uint32_t b  = rblp >> 8;
                uint32_t lp = rblp & 0xFFu;
                uint32_t pos = smem_hist[b] + lp;
                uint32_t t = static_cast<uint32_t>(base_t + r);
                ql_head[int64_t(b) * q_stride + pos] = (t << 3) | static_cast<uint32_t>(s);
            }
        }
    }
}

__global__ __launch_bounds__(128, 6) void score_topk_and_scatter_64(
    const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K_h,
    const float* __restrict__ imp_diag, uint8_t* __restrict__ remote_count,
    uint8_t* __restrict__ full_prev, uint32_t* __restrict__ bucket_count,
    uint32_t* __restrict__ q_list, WorkChunk* __restrict__ chunk_table,
    uint32_t* __restrict__ d_num_chunks,
    int bh_start, int S, int n_blocks, int q_stride, int chunk_q, float scale
) {
    score_topk_and_scatter_body<64>(
        Q, K_h, imp_diag, remote_count, full_prev,
        bucket_count, q_list, chunk_table, d_num_chunks,
        bh_start, S, n_blocks, q_stride, chunk_q, scale
    );
}

__global__ __launch_bounds__(128, 3) void score_topk_and_scatter_128(
    const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K_h,
    const float* __restrict__ imp_diag, uint8_t* __restrict__ remote_count,
    uint8_t* __restrict__ full_prev, uint32_t* __restrict__ bucket_count,
    uint32_t* __restrict__ q_list, WorkChunk* __restrict__ chunk_table,
    uint32_t* __restrict__ d_num_chunks,
    int bh_start, int S, int n_blocks, int q_stride, int chunk_q, float scale
) {
    score_topk_and_scatter_body<128>(
        Q, K_h, imp_diag, remote_count, full_prev,
        bucket_count, q_list, chunk_table, d_num_chunks,
        bh_start, S, n_blocks, q_stride, chunk_q, scale
    );
}

// B-Kernel 3: MMA-Native Zero-SMEM-Epilogue + Register-Preloaded ql_b + Overlapped Q Prefetch!
template <int D>
__global__ __launch_bounds__(128, (D == 64) ? 4 : 2) void remote_block_attn_kernel(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    const uint32_t* __restrict__ q_list,
    const WorkChunk* __restrict__ chunk_table,
    uint32_t* __restrict__ bucket_count,
    uint32_t* __restrict__ d_counters,
    int8_t* __restrict__ o_partial,
    uint2* __restrict__ mls_partial,
    int bh_start,
    int H_grp,
    int S,
    int n_blocks,
    int q_stride,
    int chunk_q,
    float scale
) {
    extern __shared__ char smem_raw[];
    constexpr int TILE_U4 = BLOCK_SIZE * (D / 8);
    uint4* K_smem = reinterpret_cast<uint4*>(smem_raw);
    uint4* V_smem = K_smem + TILE_U4;
    uint4* Q_smem = V_smem + TILE_U4;
    WorkChunk* s_wc = reinterpret_cast<WorkChunk*>(Q_smem + TILE_U4);

    int tid = threadIdx.x;
    int warp_id = tid >> 5;
    int lane = tid & 31;
    int gid = lane >> 2;
    int tid_g = lane & 3;
    int r0 = warp_id * 16 + gid;
    int r1 = r0 + 8;

    uint32_t num_chunks = d_counters[0];
    uint32_t rem_nb = (n_blocks > 2) ? static_cast<uint32_t>(n_blocks - 2) : 0u;
    uint32_t total_rem_buckets = static_cast<uint32_t>(H_grp) * rem_nb;
    uint32_t cq = static_cast<uint32_t>(chunk_q);
    const float scale_log2e = scale * 1.4426950408889634f;
    constexpr int CHUNKS_PER_ROW = D / 8;
    constexpr int WARP_Q_CHUNKS = 16 * CHUNKS_PER_ROW;
    constexpr int I8_U4_PER_ROW = D / 16;

    bool in_phase2 = false;

    while (true) {
        if (tid == 0) {
            WorkChunk wc_tmp{0, 0, 0, 0};
            if (!in_phase2) {
                uint32_t cid = atomicAdd(&d_counters[1], 1u);
                if (cid < num_chunks) {
                    wc_tmp = chunk_table[cid];
                } else {
                    in_phase2 = true;
                }
            }
            if (in_phase2) {
                while (true) {
                    uint32_t seq_idx = atomicAdd(&d_counters[2], 1u);
                    if (seq_idx >= total_rem_buckets) {
                        wc_tmp.count = 0u;
                        break;
                    }
                    uint32_t bi_t = seq_idx / static_cast<uint32_t>(H_grp);
                    uint32_t g_t  = seq_idx % static_cast<uint32_t>(H_grp);
                    uint32_t b_idx = g_t * static_cast<uint32_t>(n_blocks) + bi_t;
                    uint32_t c = bucket_count[b_idx];
                    if (c > 0u) {
                        bucket_count[b_idx] = 0u;
                        uint32_t rem = c % cq;
                        if (rem > 0u) {
                            wc_tmp.g     = static_cast<uint16_t>(g_t);
                            wc_tmp.bi    = static_cast<uint16_t>(bi_t);
                            wc_tmp.start = c - rem;
                            wc_tmp.count = rem;
                            break;
                        }
                    }
                }
            }
            *s_wc = wc_tmp;
        }
        __syncthreads();

        WorkChunk wc = *s_wc;
        if (wc.count == 0u) break;

        int g  = wc.g;
        int bh = bh_start + g;
        int bi = wc.bi;

        const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D;
        const __nv_bfloat16* K_head = K + int64_t(bh) * S * D;
        const __nv_bfloat16* V_head = V + int64_t(bh) * S * D;
        const uint4* Q_u4 = reinterpret_cast<const uint4*>(Q_head);
        const uint32_t* ql_b = q_list + (int64_t(g) * n_blocks + bi) * q_stride;
        uint4* o_dst_u4 = reinterpret_cast<uint4*>(o_partial + int64_t(g) * S * TOP_N * D);
        uint2* mls_part_head = mls_partial + int64_t(g) * S * TOP_N;

        auto load_warp_q_from_reg = [&](uint32_t pk_lane, int m_val_load) {
            #pragma unroll
            for (int idx = lane; idx < WARP_Q_CHUNKS; idx += 32) {
                int rl = idx / CHUNKS_PER_ROW;
                int c  = idx % CHUNKS_PER_ROW;
                int r  = warp_id * 16 + rl;
                uint32_t pk_r = __shfl_sync(0xffffffff, pk_lane, rl);
                bool valid = (r < m_val_load);
                int t = valid ? static_cast<int>(pk_r >> 3) : 0;
                cp_async_16_pred(&Q_smem[swz_chunk_idx<D>(r, c)], Q_u4 + int64_t(t) * CHUNKS_PER_ROW + c, valid);
            }
        };

        int m_valid0 = min(BLOCK_SIZE, static_cast<int>(wc.count));
        uint32_t pk_cur = 0u;
        if (lane < 16) {
            int r = warp_id * 16 + lane;
            if (r < m_valid0) pk_cur = __ldcs(&ql_b[wc.start + r]);
        }
        load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, bi * BLOCK_SIZE, S, tid, 128);
        if (warp_id * 16 < m_valid0) {
            load_warp_q_from_reg(pk_cur, m_valid0);
        }
        cp_async_commit();

        load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, bi * BLOCK_SIZE, S, tid, 128);
        cp_async_commit();

        uint32_t pk0 = __shfl_sync(0xffffffff, pk_cur, gid);
        uint32_t pk1 = __shfl_sync(0xffffffff, pk_cur, gid + 8);

        cp_async_wait<1>();
        __syncthreads();

        for (uint32_t q_off = 0; q_off < wc.count; q_off += BLOCK_SIZE) {
            int m_valid = min(BLOCK_SIZE, static_cast<int>(wc.count - q_off));
            bool active_warp = (warp_id * 16 < m_valid);
            bool is_first = (q_off == 0u);

            if (!is_first) {
                if (!active_warp) break;
                cp_async_wait<0>();
                __syncwarp();
            }

            bool has_next_q = (q_off + BLOCK_SIZE < wc.count);
            int m_next = has_next_q ? min(BLOCK_SIZE, static_cast<int>(wc.count - (q_off + BLOCK_SIZE))) : 0;
            bool active_next = has_next_q && (warp_id * 16 < m_next);

            uint32_t p_lo[8], p_hi[8];
            float m0 = 0.0f, l0 = 1.0f, m1 = 0.0f, l1 = 1.0f;
            bool did_prefetch = false;
            uint32_t pk_next = 0u;

            if (active_warp) {
                // Issue non-blocking ql_b load for NEXT 64 queries BEFORE QK MMA -> hides L2 latency!
                if (active_next && lane < 16) {
                    int r_n = warp_id * 16 + lane;
                    if (r_n < m_next) pk_next = __ldcs(&ql_b[wc.start + q_off + BLOCK_SIZE + r_n]);
                }

                float s[8][4];
                warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
                __syncwarp();

                if (active_next) {
                    load_warp_q_from_reg(pk_next, m_next);
                    cp_async_commit();
                    did_prefetch = true;
                }

                float mb0 = -1e30f, mb1 = -1e30f;
                #pragma unroll
                for (int ni = 0; ni < 8; ++ni) {
                    mb0 = fmaxf(mb0, fmaxf(s[ni][0], s[ni][1]));
                    mb1 = fmaxf(mb1, fmaxf(s[ni][2], s[ni][3]));
                }
                mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 1));
                mb0 = fmaxf(mb0, __shfl_xor_sync(0xffffffff, mb0, 2));
                mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 1));
                mb1 = fmaxf(mb1, __shfl_xor_sync(0xffffffff, mb1, 2));
                m0 = mb0 * scale_log2e;
                m1 = mb1 * scale_log2e;
                float neg_m0 = -m0;
                float neg_m1 = -m1;

                float lb0 = 0.0f, lb1 = 0.0f;
                #pragma unroll
                for (int ni = 0; ni < 8; ++ni) {
                    float p0 = fast_exp2f(fmaf(s[ni][0], scale_log2e, neg_m0));
                    float p1 = fast_exp2f(fmaf(s[ni][1], scale_log2e, neg_m0));
                    float p2 = fast_exp2f(fmaf(s[ni][2], scale_log2e, neg_m1));
                    float p3 = fast_exp2f(fmaf(s[ni][3], scale_log2e, neg_m1));
                    lb0 += p0 + p1;
                    lb1 += p2 + p3;
                    p_lo[ni] = pack_bf16x2(p0, p1);
                    p_hi[ni] = pack_bf16x2(p2, p3);
                }
                lb0 += __shfl_xor_sync(0xffffffff, lb0, 1);
                lb0 += __shfl_xor_sync(0xffffffff, lb0, 2);
                lb1 += __shfl_xor_sync(0xffffffff, lb1, 1);
                lb1 += __shfl_xor_sync(0xffffffff, lb1, 2);
                l0 = lb0; l1 = lb1;
            }

            if (is_first) {
                if (did_prefetch) {
                    cp_async_wait<1>();
                } else {
                    cp_async_wait<0>();
                }
                __syncthreads();
            }

            if (active_warp) {
                float o[D / 8][4];
                #pragma unroll
                for (int di = 0; di < D / 8; ++di) {
                    o[di][0] = 0.0f; o[di][1] = 0.0f; o[di][2] = 0.0f; o[di][3] = 0.0f;
                }
                warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);

                float amax0 = 0.0f, amax1 = 0.0f;
                #pragma unroll
                for (int di = 0; di < D / 8; ++di) {
                    amax0 = fmaxf(amax0, fmaxf(fabsf(o[di][0]), fabsf(o[di][1])));
                    amax1 = fmaxf(amax1, fmaxf(fabsf(o[di][2]), fabsf(o[di][3])));
                }
                amax0 = fmaxf(amax0, __shfl_xor_sync(0xffffffff, amax0, 1));
                amax0 = fmaxf(amax0, __shfl_xor_sync(0xffffffff, amax0, 2));
                amax1 = fmaxf(amax1, __shfl_xor_sync(0xffffffff, amax1, 1));
                amax1 = fmaxf(amax1, __shfl_xor_sync(0xffffffff, amax1, 2));

                float inv_s0 = (amax0 > 0.0f) ? __fdividef(127.0f, amax0) : 0.0f;
                float inv_s1 = (amax1 > 0.0f) ? __fdividef(127.0f, amax1) : 0.0f;
                float raw_s0 = amax0 * (1.0f / 127.0f);
                float raw_s1 = amax1 * (1.0f / 127.0f);

                if (r0 < m_valid) {
                    int64_t row0 = static_cast<int64_t>(pk0);
                    if (tid_g == 0) mls_part_head[row0] = pack_mls(m0, l0, raw_s0);
                    uint4 u0;
                    u0.x = pack_i8x4(o[0][0] * inv_s0, o[0][1] * inv_s0, o[1][0] * inv_s0, o[1][1] * inv_s0);
                    u0.y = pack_i8x4(o[2][0] * inv_s0, o[2][1] * inv_s0, o[3][0] * inv_s0, o[3][1] * inv_s0);
                    u0.z = pack_i8x4(o[4][0] * inv_s0, o[4][1] * inv_s0, o[5][0] * inv_s0, o[5][1] * inv_s0);
                    u0.w = pack_i8x4(o[6][0] * inv_s0, o[6][1] * inv_s0, o[7][0] * inv_s0, o[7][1] * inv_s0);
                    o_dst_u4[row0 * I8_U4_PER_ROW + tid_g] = u0;
                    if constexpr (D == 128) {
                        uint4 u0h;
                        u0h.x = pack_i8x4(o[8][0]  * inv_s0, o[8][1]  * inv_s0, o[9][0]  * inv_s0, o[9][1]  * inv_s0);
                        u0h.y = pack_i8x4(o[10][0] * inv_s0, o[10][1] * inv_s0, o[11][0] * inv_s0, o[11][1] * inv_s0);
                        u0h.z = pack_i8x4(o[12][0] * inv_s0, o[12][1] * inv_s0, o[13][0] * inv_s0, o[13][1] * inv_s0);
                        u0h.w = pack_i8x4(o[14][0] * inv_s0, o[14][1] * inv_s0, o[15][0] * inv_s0, o[15][1] * inv_s0);
                        o_dst_u4[row0 * I8_U4_PER_ROW + 4 + tid_g] = u0h;
                    }
                }
                if (r1 < m_valid) {
                    int64_t row1 = static_cast<int64_t>(pk1);
                    if (tid_g == 0) mls_part_head[row1] = pack_mls(m1, l1, raw_s1);
                    uint4 u1;
                    u1.x = pack_i8x4(o[0][2] * inv_s1, o[0][3] * inv_s1, o[1][2] * inv_s1, o[1][3] * inv_s1);
                    u1.y = pack_i8x4(o[2][2] * inv_s1, o[2][3] * inv_s1, o[3][2] * inv_s1, o[3][3] * inv_s1);
                    u1.z = pack_i8x4(o[4][2] * inv_s1, o[4][3] * inv_s1, o[5][2] * inv_s1, o[5][3] * inv_s1);
                    u1.w = pack_i8x4(o[6][2] * inv_s1, o[6][3] * inv_s1, o[7][2] * inv_s1, o[7][3] * inv_s1);
                    o_dst_u4[row1 * I8_U4_PER_ROW + tid_g] = u1;
                    if constexpr (D == 128) {
                        uint4 u1h;
                        u1h.x = pack_i8x4(o[8][2]  * inv_s1, o[8][3]  * inv_s1, o[9][2]  * inv_s1, o[9][3]  * inv_s1);
                        u1h.y = pack_i8x4(o[10][2] * inv_s1, o[10][3] * inv_s1, o[11][2] * inv_s1, o[11][3] * inv_s1);
                        u1h.z = pack_i8x4(o[12][2] * inv_s1, o[12][3] * inv_s1, o[13][2] * inv_s1, o[13][3] * inv_s1);
                        u1h.w = pack_i8x4(o[14][2] * inv_s1, o[14][3] * inv_s1, o[15][2] * inv_s1, o[15][3] * inv_s1);
                        o_dst_u4[row1 * I8_U4_PER_ROW + 4 + tid_g] = u1h;
                    }
                }
                if (active_next) {
                    pk0 = __shfl_sync(0xffffffff, pk_next, gid);
                    pk1 = __shfl_sync(0xffffffff, pk_next, gid + 8);
                }
            }
        }

        __syncthreads();
    }
}

// B-Kernel 4: ZERO-SPILL LPT Local Window + Register-Direct MMA-Native Merge!
// On-the-fly alA/alB broadcast frees 6 registers -> 0 spills on BOTH D=64 (<=128 regs, 4 CTAs/SM) AND D=128 (2 CTAs/SM)!
template <int D>
__device__ __forceinline__ void local_window_and_merge_body(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    const uint8_t* __restrict__ remote_count,
    const uint8_t* __restrict__ full_prev,
    const int8_t* __restrict__ o_partial,
    const uint2* __restrict__ mls_partial,
    uint32_t* __restrict__ d_counters,
    __nv_bfloat16* __restrict__ O,
    int bh_start,
    int S,
    int n_blocks,
    float scale
) {
    extern __shared__ char smem_raw[];
    constexpr int TILE_U4 = BLOCK_SIZE * (D / 8);
    uint4* Q_smem = reinterpret_cast<uint4*>(smem_raw);
    uint4* K_smem = Q_smem + TILE_U4;
    uint4* V_smem = K_smem + TILE_U4;

    int g = blockIdx.x;
    int by = blockIdx.y;
    int bi_cur;
    if (n_blocks <= TOP_N) {
        bi_cur = (n_blocks - 1) - by;
    } else if (by < 4) {
        bi_cur = 7 - by;             // 7, 6, 5, 4 (heaviest 8..5 blocks first!)
    } else if (by < n_blocks - 4) {
        bi_cur = 8 + (by - 4);       // 8 .. n_blocks - 1
    } else {
        bi_cur = (n_blocks - 1) - by; // 3, 2, 1, 0 (lightest tail last!)
    }

    int bh = bh_start + g;
    int tid = threadIdx.x;
    int warp_id = tid >> 5;
    int lane = tid & 31;
    int gid = lane >> 2;
    int tid_g = lane & 3;
    int base_lane = gid << 2;
    int r0 = warp_id * 16 + gid;
    int r1 = r0 + 8;

    if (blockIdx.x == 0 && blockIdx.y == 0 && tid < 3) {
        d_counters[tid] = 0u;
    }

    const __nv_bfloat16* Q_head = Q + int64_t(bh) * S * D;
    const __nv_bfloat16* K_head = K + int64_t(bh) * S * D;
    const __nv_bfloat16* V_head = V + int64_t(bh) * S * D;
    __nv_bfloat16* O_head = O + int64_t(bh) * S * D;

    load_tile_swz_async<D, BLOCK_SIZE>(Q_smem, Q_head, bi_cur * BLOCK_SIZE, S, tid, 128);
    cp_async_commit();

    float o[D / 8][4];
    #pragma unroll
    for (int di = 0; di < D / 8; ++di) {
        o[di][0] = 0.0f; o[di][1] = 0.0f; o[di][2] = 0.0f; o[di][3] = 0.0f;
    }
    float m0 = -1e30f, l0 = 0.0f; bool has0 = false;
    float m1 = -1e30f, l1 = 0.0f; bool has1 = false;
    const float scale_log2e = scale * 1.4426950408889634f;

    if (bi_cur < TOP_N) {
        load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, 0, S, tid, 128);
        cp_async_commit();
        load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, 0, S, tid, 128);
        cp_async_commit();
        cp_async_wait<1>();
        __syncthreads();

        for (int bi = 0; bi <= bi_cur; ++bi) {
            float s[8][4];
            warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
            bool has_next = (bi + 1 <= bi_cur);
            if (has_next) {
                __syncthreads();
                load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, (bi + 1) * BLOCK_SIZE, S, tid, 128);
                cp_async_commit();
            }
            uint32_t p_lo[8], p_hi[8];
            if (bi < bi_cur) {
                online_softmax_prep<D, false>(
                    o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
                    s, true, true, r0, r1, false, true, true, scale_log2e, lane
                );
            } else {
                online_softmax_prep<D, true>(
                    o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
                    s, true, true, r0, r1, true, true, true, scale_log2e, lane
                );
            }
            if (has_next) {
                cp_async_wait<1>();
            } else {
                cp_async_wait<0>();
            }
            __syncthreads();
            warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);
            if (has_next) {
                cp_async_wait<0>();
                __syncthreads();
                load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, (bi + 1) * BLOCK_SIZE, S, tid, 128);
                cp_async_commit();
            }
        }
        __syncthreads();
        store_O_bf16_via_smem<D, true>(O_head, Q_smem, o, l0, l1, bi_cur, S, warp_id, lane);
        return;
    }

    int t0 = bi_cur * BLOCK_SIZE + r0;
    int t1 = bi_cur * BLOCK_SIZE + r1;
    bool fp0 = (t0 < S) ? (full_prev[int64_t(g) * S + t0] != 0) : false;
    bool fp1 = (t1 < S) ? (full_prev[int64_t(g) * S + t1] != 0) : false;

    load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, bi_cur * BLOCK_SIZE, S, tid, 128);
    cp_async_commit();
    load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, bi_cur * BLOCK_SIZE, S, tid, 128);
    cp_async_commit();
    cp_async_wait<1>();
    __syncthreads();

    {
        float s[8][4];
        warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
        __syncthreads();
        load_tile_swz_async<D, BLOCK_SIZE>(K_smem, K_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 128);
        cp_async_commit();

        uint32_t p_lo[8], p_hi[8];
        online_softmax_prep<D, true>(
            o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
            s, true, true, r0, r1, true, false, false, scale_log2e, lane
        );
        cp_async_wait<1>();
        __syncthreads();

        warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);
        cp_async_wait<0>();
        __syncthreads();
    }

    if constexpr (D == 128) {
        const uint4* op_u4_head = reinterpret_cast<const uint4*>(o_partial + int64_t(g) * S * TOP_N * D);
        const uint2* mls_part_head = mls_partial + int64_t(g) * S * TOP_N;
        const uint8_t* rc_head = remote_count + int64_t(g) * S;
        constexpr int I8_U4_PER_ROW = D / 16;

        bool valid_t0 = (t0 < S);
        bool valid_t1 = (t1 < S);
        int k_rem0 = valid_t0 ? rc_head[t0] : 0;
        int k_rem1 = valid_t1 ? rc_head[t1] : 0;
        int sA = tid_g;
        int sB = tid_g + 4;
        uint2 pkA0 = make_uint2(0u, 0u), pkB0 = make_uint2(0u, 0u);
        uint2 pkA1 = make_uint2(0u, 0u), pkB1 = make_uint2(0u, 0u);

        {
            load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 128);
            cp_async_commit();

            float s[8][4];
            warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
            uint32_t p_lo[8], p_hi[8];
            online_softmax_prep<D, true>(
                o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
                s, true, true, r0, r1, false, fp0, fp1, scale_log2e, lane
            );

            if (valid_t0 && sA < k_rem0) pkA0 = __ldcs(&mls_part_head[int64_t(t0) * TOP_N + sA]);
            if (valid_t0 && sB < k_rem0) pkB0 = __ldcs(&mls_part_head[int64_t(t0) * TOP_N + sB]);
            if (valid_t1 && sA < k_rem1) pkA1 = __ldcs(&mls_part_head[int64_t(t1) * TOP_N + sA]);
            if (valid_t1 && sB < k_rem1) pkB1 = __ldcs(&mls_part_head[int64_t(t1) * TOP_N + sB]);

            cp_async_wait<0>();
            __syncthreads();
            warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);
        }

        auto merge_reg_row_128 = [&](int t, bool valid_t, int k_rem, float m_loc, float l_loc, uint2 pkA, uint2 pkB, int col_off) {
            float mA = -1e30f, lA = 0.0f, rsA = 0.0f;
            float mB = -1e30f, lB = 0.0f, rsB = 0.0f;
            if (valid_t && sA < k_rem) unpack_mls(pkA, mA, lA, rsA);
            if (valid_t && sB < k_rem) unpack_mls(pkB, mB, lB, rsB);

            float m_fin = fmaxf(mA, mB);
            if (tid_g == 0) m_fin = fmaxf(m_fin, m_loc);
            m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 1));
            m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 2));

            float w_loc_scale = fast_exp2f(m_loc - m_fin);
            float w_loc = l_loc * w_loc_scale;
            float expA = (valid_t && sA < k_rem) ? fast_exp2f(mA - m_fin) : 0.0f;
            float expB = (valid_t && sB < k_rem) ? fast_exp2f(mB - m_fin) : 0.0f;

            float w_rem = lA * expA + lB * expB;
            w_rem += __shfl_xor_sync(0xffffffff, w_rem, 1);
            w_rem += __shfl_xor_sync(0xffffffff, w_rem, 2);

            float inv_l = __frcp_rn(w_loc + w_rem);
            float alpha_loc = w_loc_scale * inv_l;
            float alA = (expA > 1e-8f) ? (expA * inv_l * rsA) : 0.0f;
            float alB = (expB > 1e-8f) ? (expB * inv_l * rsB) : 0.0f;

            if (valid_t) {
                #pragma unroll
                for (int di = 0; di < D / 8; ++di) {
                    o[di][col_off]     *= alpha_loc;
                    o[di][col_off + 1] *= alpha_loc;
                }
            }

            const uint4* op_row = op_u4_head + int64_t(t) * TOP_N * I8_U4_PER_ROW;
            auto accum_slot = [&](bool ok, float w, uint4 rlo, uint4 rhi) {
                if (!ok) return;
                i8x4 c0 = *reinterpret_cast<const i8x4*>(&rlo.x);
                i8x4 c1 = *reinterpret_cast<const i8x4*>(&rlo.y);
                i8x4 c2 = *reinterpret_cast<const i8x4*>(&rlo.z);
                i8x4 c3 = *reinterpret_cast<const i8x4*>(&rlo.w);
                o[0][col_off] = fmaf(w, float(c0.x), o[0][col_off]); o[0][col_off+1] = fmaf(w, float(c0.y), o[0][col_off+1]);
                o[1][col_off] = fmaf(w, float(c0.z), o[1][col_off]); o[1][col_off+1] = fmaf(w, float(c0.w), o[1][col_off+1]);
                o[2][col_off] = fmaf(w, float(c1.x), o[2][col_off]); o[2][col_off+1] = fmaf(w, float(c1.y), o[2][col_off+1]);
                o[3][col_off] = fmaf(w, float(c1.z), o[3][col_off]); o[3][col_off+1] = fmaf(w, float(c1.w), o[3][col_off+1]);
                o[4][col_off] = fmaf(w, float(c2.x), o[4][col_off]); o[4][col_off+1] = fmaf(w, float(c2.y), o[4][col_off+1]);
                o[5][col_off] = fmaf(w, float(c2.z), o[5][col_off]); o[5][col_off+1] = fmaf(w, float(c2.w), o[5][col_off+1]);
                o[6][col_off] = fmaf(w, float(c3.x), o[6][col_off]); o[6][col_off+1] = fmaf(w, float(c3.y), o[6][col_off+1]);
                o[7][col_off] = fmaf(w, float(c3.z), o[7][col_off]); o[7][col_off+1] = fmaf(w, float(c3.w), o[7][col_off+1]);
                i8x4 h0 = *reinterpret_cast<const i8x4*>(&rhi.x);
                i8x4 h1 = *reinterpret_cast<const i8x4*>(&rhi.y);
                i8x4 h2 = *reinterpret_cast<const i8x4*>(&rhi.z);
                i8x4 h3 = *reinterpret_cast<const i8x4*>(&rhi.w);
                o[8][col_off]  = fmaf(w, float(h0.x), o[8][col_off]);  o[8][col_off+1]  = fmaf(w, float(h0.y), o[8][col_off+1]);
                o[9][col_off]  = fmaf(w, float(h0.z), o[9][col_off]);  o[9][col_off+1]  = fmaf(w, float(h0.w), o[9][col_off+1]);
                o[10][col_off] = fmaf(w, float(h1.x), o[10][col_off]); o[10][col_off+1] = fmaf(w, float(h1.y), o[10][col_off+1]);
                o[11][col_off] = fmaf(w, float(h1.z), o[11][col_off]); o[11][col_off+1] = fmaf(w, float(h1.w), o[11][col_off+1]);
                o[12][col_off] = fmaf(w, float(h2.x), o[12][col_off]); o[12][col_off+1] = fmaf(w, float(h2.y), o[12][col_off+1]);
                o[13][col_off] = fmaf(w, float(h2.z), o[13][col_off]); o[13][col_off+1] = fmaf(w, float(h2.w), o[13][col_off+1]);
                o[14][col_off] = fmaf(w, float(h3.x), o[14][col_off]); o[14][col_off+1] = fmaf(w, float(h3.y), o[14][col_off+1]);
                o[15][col_off] = fmaf(w, float(h3.z), o[15][col_off]); o[15][col_off+1] = fmaf(w, float(h3.w), o[15][col_off+1]);
            };

            #pragma unroll
            for (int base_s = 0; base_s < TOP_N; base_s += 4) {
                float src_al = (base_s == 0) ? alA : alB;
                float w0 = __shfl_sync(0xffffffff, src_al, base_lane + 0);
                float w1 = __shfl_sync(0xffffffff, src_al, base_lane + 1);
                float w2 = __shfl_sync(0xffffffff, src_al, base_lane + 2);
                float w3 = __shfl_sync(0xffffffff, src_al, base_lane + 3);
                if (valid_t) {
                    bool ok0 = (base_s + 0 < k_rem) && (w0 != 0.0f);
                    bool ok1 = (base_s + 1 < k_rem) && (w1 != 0.0f);
                    bool ok2 = (base_s + 2 < k_rem) && (w2 != 0.0f);
                    bool ok3 = (base_s + 3 < k_rem) && (w3 != 0.0f);
                    uint4 rlo0, rlo1, rlo2, rlo3, rhi0, rhi1, rhi2, rhi3;
                    if (ok0) { rlo0 = __ldcs(&op_row[(base_s + 0) * I8_U4_PER_ROW + tid_g]); rhi0 = __ldcs(&op_row[(base_s + 0) * I8_U4_PER_ROW + 4 + tid_g]); }
                    if (ok1) { rlo1 = __ldcs(&op_row[(base_s + 1) * I8_U4_PER_ROW + tid_g]); rhi1 = __ldcs(&op_row[(base_s + 1) * I8_U4_PER_ROW + 4 + tid_g]); }
                    if (ok2) { rlo2 = __ldcs(&op_row[(base_s + 2) * I8_U4_PER_ROW + tid_g]); rhi2 = __ldcs(&op_row[(base_s + 2) * I8_U4_PER_ROW + 4 + tid_g]); }
                    if (ok3) { rlo3 = __ldcs(&op_row[(base_s + 3) * I8_U4_PER_ROW + tid_g]); rhi3 = __ldcs(&op_row[(base_s + 3) * I8_U4_PER_ROW + 4 + tid_g]); }
                    accum_slot(ok0, w0, rlo0, rhi0);
                    accum_slot(ok1, w1, rlo1, rhi1);
                    accum_slot(ok2, w2, rlo2, rhi2);
                    accum_slot(ok3, w3, rlo3, rhi3);
                }
            }
        };

        merge_reg_row_128(t0, valid_t0, k_rem0, m0, l0, pkA0, pkB0, 0);
        merge_reg_row_128(t1, valid_t1, k_rem1, m1, l1, pkA1, pkB1, 2);
    } else {
        {
            load_tile_swz_async<D, BLOCK_SIZE>(V_smem, V_head, (bi_cur - 1) * BLOCK_SIZE, S, tid, 128);
            cp_async_commit();

            float s[8][4];
            warp_gemm_qk_16x64<D>(s, Q_smem, K_smem, warp_id, lane);
            uint32_t p_lo[8], p_hi[8];
            online_softmax_prep<D, true>(
                o, p_lo, p_hi, m0, l0, has0, m1, l1, has1,
                s, true, true, r0, r1, false, fp0, fp1, scale_log2e, lane
            );

            cp_async_wait<0>();
            __syncthreads();
            warp_gemm_pv_16xD<D>(o, p_lo, p_hi, V_smem, lane);
        }

        const uint4* op_u4_head = reinterpret_cast<const uint4*>(o_partial + int64_t(g) * S * TOP_N * D);
        const uint2* mls_part_head = mls_partial + int64_t(g) * S * TOP_N;
        const uint8_t* rc_head = remote_count + int64_t(g) * S;
        constexpr int I8_U4_PER_ROW = D / 16;

        auto merge_reg_row_64 = [&](int t, float m_loc, float l_loc, int col_off) {
            bool valid_t = (t < S);
            int k_rem = valid_t ? rc_head[t] : 0;

            int sA = tid_g;
            int sB = tid_g + 4;
            float mA = -1e30f, lA = 0.0f, rsA = 0.0f;
            float mB = -1e30f, lB = 0.0f, rsB = 0.0f;
            if (valid_t && sA < k_rem) unpack_mls(__ldcs(&mls_part_head[int64_t(t) * TOP_N + sA]), mA, lA, rsA);
            if (valid_t && sB < k_rem) unpack_mls(__ldcs(&mls_part_head[int64_t(t) * TOP_N + sB]), mB, lB, rsB);

            float m_fin = fmaxf(mA, mB);
            if (tid_g == 0) m_fin = fmaxf(m_fin, m_loc);
            m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 1));
            m_fin = fmaxf(m_fin, __shfl_xor_sync(0xffffffff, m_fin, 2));

            float w_loc_scale = fast_exp2f(m_loc - m_fin);
            float w_loc = l_loc * w_loc_scale;
            float expA = (valid_t && sA < k_rem) ? fast_exp2f(mA - m_fin) : 0.0f;
            float expB = (valid_t && sB < k_rem) ? fast_exp2f(mB - m_fin) : 0.0f;

            float w_rem = lA * expA + lB * expB;
            w_rem += __shfl_xor_sync(0xffffffff, w_rem, 1);
            w_rem += __shfl_xor_sync(0xffffffff, w_rem, 2);

            float inv_l = __frcp_rn(w_loc + w_rem);
            float alpha_loc = w_loc_scale * inv_l;
            float alA = (expA > 1e-8f) ? (expA * inv_l * rsA) : 0.0f;
            float alB = (expB > 1e-8f) ? (expB * inv_l * rsB) : 0.0f;

            if (valid_t) {
                #pragma unroll
                for (int di = 0; di < D / 8; ++di) {
                    o[di][col_off]     *= alpha_loc;
                    o[di][col_off + 1] *= alpha_loc;
                }
            }

            const uint4* op_row = op_u4_head + int64_t(t) * TOP_N * I8_U4_PER_ROW;
            #pragma unroll
            for (int base_s = 0; base_s < TOP_N; base_s += 2) {
                float src_al = (base_s < 4) ? alA : alB;
                float w0 = __shfl_sync(0xffffffff, src_al, base_lane + (base_s & 3));
                float w1 = __shfl_sync(0xffffffff, src_al, base_lane + ((base_s + 1) & 3));
                if (valid_t) {
                    bool ok0 = (base_s     < k_rem) && (w0 != 0.0f);
                    bool ok1 = (base_s + 1 < k_rem) && (w1 != 0.0f);
                    uint4 rlo0, rlo1;
                    if (ok0) rlo0 = __ldcs(&op_row[(base_s) * I8_U4_PER_ROW + tid_g]);
                    if (ok1) rlo1 = __ldcs(&op_row[(base_s + 1) * I8_U4_PER_ROW + tid_g]);
                    auto accum_64 = [&](bool ok, float w, uint4 rlo) {
                        if (!ok) return;
                        i8x4 c0 = *reinterpret_cast<const i8x4*>(&rlo.x);
                        i8x4 c1 = *reinterpret_cast<const i8x4*>(&rlo.y);
                        i8x4 c2 = *reinterpret_cast<const i8x4*>(&rlo.z);
                        i8x4 c3 = *reinterpret_cast<const i8x4*>(&rlo.w);
                        o[0][col_off] = fmaf(w, float(c0.x), o[0][col_off]); o[0][col_off+1] = fmaf(w, float(c0.y), o[0][col_off+1]);
                        o[1][col_off] = fmaf(w, float(c0.z), o[1][col_off]); o[1][col_off+1] = fmaf(w, float(c0.w), o[1][col_off+1]);
                        o[2][col_off] = fmaf(w, float(c1.x), o[2][col_off]); o[2][col_off+1] = fmaf(w, float(c1.y), o[2][col_off+1]);
                        o[3][col_off] = fmaf(w, float(c1.z), o[3][col_off]); o[3][col_off+1] = fmaf(w, float(c1.w), o[3][col_off+1]);
                        o[4][col_off] = fmaf(w, float(c2.x), o[4][col_off]); o[4][col_off+1] = fmaf(w, float(c2.y), o[4][col_off+1]);
                        o[5][col_off] = fmaf(w, float(c2.z), o[5][col_off]); o[5][col_off+1] = fmaf(w, float(c2.w), o[5][col_off+1]);
                        o[6][col_off] = fmaf(w, float(c3.x), o[6][col_off]); o[6][col_off+1] = fmaf(w, float(c3.y), o[6][col_off+1]);
                        o[7][col_off] = fmaf(w, float(c3.z), o[7][col_off]); o[7][col_off+1] = fmaf(w, float(c3.w), o[7][col_off+1]);
                    };
                    accum_64(ok0, w0, rlo0);
                    accum_64(ok1, w1, rlo1);
                }
            }
        };

        merge_reg_row_64(t0, m0, l0, 0);
        merge_reg_row_64(t1, m1, l1, 2);
    }

    store_O_bf16_via_smem<D, false>(O_head, Q_smem, o, 1.0f, 1.0f, bi_cur, S, warp_id, lane);
}

__global__ __launch_bounds__(128, 4) void local_window_and_merge_64(
    const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V,
    const uint8_t* __restrict__ rc, const uint8_t* __restrict__ fp,
    const int8_t* __restrict__ op, const uint2* __restrict__ mlsp,
    uint32_t* __restrict__ ctr, __nv_bfloat16* __restrict__ O,
    int bh_start, int S, int n_blocks, float scale
) {
    local_window_and_merge_body<64>(Q, K, V, rc, fp, op, mlsp, ctr, O, bh_start, S, n_blocks, scale);
}

__global__ void local_window_and_merge_128(
    const __nv_bfloat16* __restrict__ Q, const __nv_bfloat16* __restrict__ K, const __nv_bfloat16* __restrict__ V,
    const uint8_t* __restrict__ rc, const uint8_t* __restrict__ fp,
    const int8_t* __restrict__ op, const uint2* __restrict__ mlsp,
    uint32_t* __restrict__ ctr, __nv_bfloat16* __restrict__ O,
    int bh_start, int S, int n_blocks, float scale
) {
    local_window_and_merge_body<128>(Q, K, V, rc, fp, op, mlsp, ctr, O, bh_start, S, n_blocks, scale);
}

// =============================================================================
// General fallback CUDA kernel (arbitrary D != 64, 128)
// =============================================================================
__global__ void nsa_fallback_kernel(
    const __nv_bfloat16* __restrict__ Q,
    const __nv_bfloat16* __restrict__ K,
    const __nv_bfloat16* __restrict__ V,
    __nv_bfloat16* __restrict__ O,
    int S, int D, float scale
) {
    int bh = blockIdx.y;
    int t = blockIdx.x;
    int tid = threadIdx.x;
    if (t >= S) return;

    int bi_cur = t / BLOCK_SIZE;
    const __nv_bfloat16* q_t = Q + (int64_t(bh) * S + t) * D;
    const __nv_bfloat16* K_h = K + int64_t(bh) * S * D;
    const __nv_bfloat16* V_h = V + int64_t(bh) * S * D;
    __nv_bfloat16* o_t = O + (int64_t(bh) * S + t) * D;

    __shared__ float s_imp[1024];
    __shared__ int s_top[TOP_N];

    for (int bi = tid; bi <= bi_cur; bi += blockDim.x) {
        int s0 = bi * BLOCK_SIZE;
        int s1 = min(s0 + BLOCK_SIZE, t + 1);
        float sum = 0.0f;
        for (int j = s0; j < s1; ++j) {
            const __nv_bfloat16* kj = K_h + int64_t(j) * D;
            float dot = 0.0f;
            for (int d = 0; d < D; ++d) dot += __bfloat162float(q_t[d]) * __bfloat162float(kj[d]);
            sum += dot * scale;
        }
        s_imp[bi] = sum / float(s1 - s0);
    }
    __syncthreads();

    if (tid == 0) {
        uint64_t top_k[TOP_N];
        for (int u = 0; u < TOP_N; ++u) top_k[u] = 0ULL;
        for (int bi = 0; bi <= bi_cur; ++bi) {
            uint64_t k = make_score_key(s_imp[bi], bi);
            int mp = 0;
            for (int u = 1; u < TOP_N; ++u) if (top_k[u] < top_k[mp]) mp = u;
            if (k > top_k[mp]) top_k[mp] = k;
        }
        for (int u = 0; u < TOP_N; ++u) s_top[u] = (top_k[u] != 0ULL) ? static_cast<int>(top_k[u] & 0xFFFFFFFFu) : -1;
    }
    __syncthreads();

    int w0 = max(0, t + 1 - BLOCK_SIZE);
    __shared__ float s_m, s_l;
    if (tid == 0) {
        float m = -1e30f;
        for (int bi = 0; bi <= bi_cur; ++bi) {
            bool sel = false;
            for (int u = 0; u < TOP_N; ++u) if (s_top[u] == bi) sel = true;
            int s0 = bi * BLOCK_SIZE;
            int s1 = min(s0 + BLOCK_SIZE, t + 1);
            if (!sel && s1 <= w0) continue;
            int j0 = sel ? s0 : max(s0, w0);
            for (int j = j0; j < s1; ++j) {
                const __nv_bfloat16* kj = K_h + int64_t(j) * D;
                float dot = 0.0f;
                for (int d = 0; d < D; ++d) dot += __bfloat162float(q_t[d]) * __bfloat162float(kj[d]);
                m = fmaxf(m, dot * scale);
            }
        }
        s_m = m;
        float l = 0.0f;
        for (int bi = 0; bi <= bi_cur; ++bi) {
            bool sel = false;
            for (int u = 0; u < TOP_N; ++u) if (s_top[u] == bi) sel = true;
            int s0 = bi * BLOCK_SIZE;
            int s1 = min(s0 + BLOCK_SIZE, t + 1);
            if (!sel && s1 <= w0) continue;
            int j0 = sel ? s0 : max(s0, w0);
            for (int j = j0; j < s1; ++j) {
                const __nv_bfloat16* kj = K_h + int64_t(j) * D;
                float dot = 0.0f;
                for (int d = 0; d < D; ++d) dot += __bfloat162float(q_t[d]) * __bfloat162float(kj[d]);
                l += expf(dot * scale - m);
            }
        }
        s_l = l;
    }
    __syncthreads();

    float m = s_m, inv_l = 1.0f / s_l;
    for (int d = tid; d < D; d += blockDim.x) {
        float acc = 0.0f;
        for (int bi = 0; bi <= bi_cur; ++bi) {
            bool sel = false;
            for (int u = 0; u < TOP_N; ++u) if (s_top[u] == bi) sel = true;
            int s0 = bi * BLOCK_SIZE;
            int s1 = min(s0 + BLOCK_SIZE, t + 1);
            if (!sel && s1 <= w0) continue;
            int j0 = sel ? s0 : max(s0, w0);
            for (int j = j0; j < s1; ++j) {
                const __nv_bfloat16* kj = K_h + int64_t(j) * D;
                float dot = 0.0f;
                for (int k = 0; k < D; ++k) dot += __bfloat162float(q_t[k]) * __bfloat162float(kj[k]);
                acc += expf(dot * scale - m) * inv_l * __bfloat162float(V_h[int64_t(j) * D + d]);
            }
        }
        o_t[d] = __float2bfloat16_rn(acc);
    }
}

// =============================================================================
// Host Dispatch + Persistent Workspace
// =============================================================================

struct NSAWorkspace {
    torch::Tensor k_h;
    torch::Tensor imp_diag;
    torch::Tensor remote_count, full_prev;
    torch::Tensor bucket_count;
    torch::Tensor chunk_table, q_list;
    torch::Tensor o_partial, mls_partial;
    torch::Tensor d_counters;
    int64_t cap_km = 0;
    int64_t cap_bhs = 0;
    int64_t cap_grp_s = 0;
    int64_t cap_buckets = 0;
    int64_t cap_qlist = 0;
    int64_t cap_chunks = 0;
    int64_t cap_opart = 0;
    bool smem_configured = false;
};

static NSAWorkspace g_ws;

static void configure_smem_once() {
    if (g_ws.smem_configured) return;
    auto set_smem = [](const void* func, int bytes) {
        cudaFuncSetAttribute(func, cudaFuncAttributeMaxDynamicSharedMemorySize, bytes);
    };
    set_smem((const void*)fused_nsa_direct_bn16_splitkv_64, 49152);
    set_smem((const void*)fused_nsa_direct_bn16_128, 57344);
    set_smem((const void*)score_topk_and_scatter_64, 24576);
    set_smem((const void*)score_topk_and_scatter_128, 40960);
    set_smem((const void*)remote_block_attn_kernel<64>, 32768);
    set_smem((const void*)remote_block_attn_kernel<128>, 57344);
    set_smem((const void*)local_window_and_merge_64, 32768);
    set_smem((const void*)local_window_and_merge_128, 57344);
    g_ws.smem_configured = true;
}

template <int D>
static void dispatch_nsa_impl(
    const torch::Tensor& q,
    const torch::Tensor& k,
    const torch::Tensor& v,
    torch::Tensor& out,
    int direct_max_blocks,
    int h_grp_budget_mb,
    int chunk_q_req
) {
    configure_smem_once();
    cudaStream_t stream = at::cuda::getCurrentCUDAStream();

    int B = q.size(0);
    int H = q.size(1);
    int S = q.size(2);
    int BH = B * H;
    int n_blocks = (S + BLOCK_SIZE - 1) / BLOCK_SIZE;
    float scale = 1.0f / sqrtf(float(D));

    auto opts_bf16 = q.options();
    auto opts_f32  = q.options().dtype(torch::kFloat32);

    int64_t need_km = int64_t(BH) * 3 * n_blocks * D;
    if (g_ws.cap_km < need_km) {
        g_ws.k_h = torch::empty({need_km}, opts_bf16);
        g_ws.cap_km = need_km;
    }
    int64_t need_bhs = int64_t(BH) * S;
    if (g_ws.cap_bhs < need_bhs) {
        g_ws.imp_diag = torch::empty({need_bhs}, opts_f32);
        g_ws.cap_bhs = need_bhs;
    }
    if (!g_ws.d_counters.defined()) {
        g_ws.d_counters = torch::zeros({4}, q.options().dtype(torch::kInt32));
    }

    const __nv_bfloat16* q_ptr = reinterpret_cast<const __nv_bfloat16*>(q.data_ptr());
    const __nv_bfloat16* k_ptr = reinterpret_cast<const __nv_bfloat16*>(k.data_ptr());
    const __nv_bfloat16* v_ptr = reinterpret_cast<const __nv_bfloat16*>(v.data_ptr());
    __nv_bfloat16* o_ptr       = reinterpret_cast<__nv_bfloat16*>(out.data_ptr());
    __nv_bfloat16* kh_ptr      = reinterpret_cast<__nv_bfloat16*>(g_ws.k_h.data_ptr());
    float* imp_ptr             = g_ws.imp_diag.data_ptr<float>();
    uint32_t* d_ctr_ptr        = reinterpret_cast<uint32_t*>(g_ws.d_counters.data_ptr());

    compute_k_mean_and_diag_kernel<D><<<dim3(n_blocks, BH), 128, 0, stream>>>(
        q_ptr, k_ptr, kh_ptr, imp_ptr, d_ctr_ptr, S, n_blocks, scale
    );

    if (n_blocks <= direct_max_blocks && n_blocks <= 16) {
        if constexpr (D == 64) {
            size_t smem_a = 5 * BLOCK_SIZE * 64 * sizeof(__nv_bfloat16);
            fused_nsa_direct_bn16_splitkv_64<<<BH * n_blocks, 256, smem_a, stream>>>(
                q_ptr, k_ptr, v_ptr, kh_ptr, imp_ptr, o_ptr,
                BH, S, n_blocks, scale
            );
        } else {
            size_t smem_a = 3 * BLOCK_SIZE * 128 * sizeof(__nv_bfloat16);
            fused_nsa_direct_bn16_128<<<dim3(BH, n_blocks), 128, smem_a, stream>>>(
                q_ptr, k_ptr, v_ptr, kh_ptr, imp_ptr, o_ptr,
                S, n_blocks, scale
            );
        }
        return;
    }

    int q_stride = ((S + 31) & ~31) + 32;
    int64_t bytes_per_head = int64_t(S) * TOP_N * D * sizeof(int8_t);
    int64_t budget_bytes = int64_t(h_grp_budget_mb) * 1024LL * 1024LL;
    int H_grp = static_cast<int>(budget_bytes / std::max<int64_t>(1, bytes_per_head));
    int64_t qlist_bytes_per_head = int64_t(n_blocks) * q_stride * sizeof(uint32_t);
    int H_grp_ql = static_cast<int>((256LL * 1024LL * 1024LL) / std::max<int64_t>(1, qlist_bytes_per_head));
    H_grp = std::max(1, std::min({BH, H_grp, std::max(1, H_grp_ql)}));

    int persistent_ctas = (D == 64) ? (188 * 4) : (188 * 2);
    int chunk_q = (chunk_q_req > 0) ? chunk_q_req
                : ((D == 128) ? 768 : ((n_blocks <= 80) ? 384 : 256));

    int64_t need_grp_s   = int64_t(H_grp) * S;
    int64_t need_buckets = int64_t(H_grp) * n_blocks;
    int64_t need_qlist   = need_buckets * q_stride;
    int64_t need_chunks  = (need_grp_s * TOP_N) / std::max(1, chunk_q) + need_buckets + 16;
    int64_t need_opart   = need_grp_s * TOP_N * D;

    if (g_ws.cap_grp_s < need_grp_s) {
        g_ws.remote_count = torch::empty({need_grp_s}, q.options().dtype(torch::kUInt8));
        g_ws.full_prev    = torch::empty({need_grp_s}, q.options().dtype(torch::kUInt8));
        g_ws.mls_partial  = torch::empty({need_grp_s * TOP_N * 2}, q.options().dtype(torch::kInt32));
        g_ws.cap_grp_s = need_grp_s;
    }
    if (g_ws.cap_buckets < need_buckets) {
        g_ws.bucket_count = torch::zeros({need_buckets}, q.options().dtype(torch::kInt32));
        g_ws.cap_buckets = need_buckets;
    }
    if (g_ws.cap_qlist < need_qlist) {
        g_ws.q_list = torch::empty({need_qlist}, q.options().dtype(torch::kInt32));
        g_ws.cap_qlist = need_qlist;
    }
    if (g_ws.cap_chunks < need_chunks) {
        g_ws.chunk_table = torch::empty({need_chunks * int64_t(sizeof(WorkChunk))}, q.options().dtype(torch::kUInt8));
        g_ws.cap_chunks = need_chunks;
    }
    if (g_ws.cap_opart < need_opart) {
        g_ws.o_partial = torch::empty({need_opart}, q.options().dtype(torch::kInt8));
        g_ws.cap_opart = need_opart;
    }

    uint8_t* rc_ptr  = g_ws.remote_count.data_ptr<uint8_t>();
    uint8_t* fp_ptr  = g_ws.full_prev.data_ptr<uint8_t>();
    uint32_t* bc_ptr = reinterpret_cast<uint32_t*>(g_ws.bucket_count.data_ptr());
    WorkChunk* ct_ptr = reinterpret_cast<WorkChunk*>(g_ws.chunk_table.data_ptr());
    uint32_t* ql_ptr = reinterpret_cast<uint32_t*>(g_ws.q_list.data_ptr());
    int8_t* op_ptr   = g_ws.o_partial.data_ptr<int8_t>();
    uint2* mlsp_ptr  = reinterpret_cast<uint2*>(g_ws.mls_partial.data_ptr());

    size_t smem_score = 2 * BLOCK_SIZE * D * sizeof(__nv_bfloat16);
    size_t smem_rem   = 3 * BLOCK_SIZE * D * sizeof(__nv_bfloat16) + sizeof(WorkChunk) + 16;
    size_t smem_loc   = 3 * BLOCK_SIZE * D * sizeof(__nv_bfloat16);

    for (int bh_start = 0; bh_start < BH; bh_start += H_grp) {
        int cur_grp = std::min(H_grp, BH - bh_start);

        if (n_blocks > TOP_N) {
            if constexpr (D == 64) {
                score_topk_and_scatter_64<<<dim3(cur_grp, n_blocks - TOP_N), 128, smem_score, stream>>>(
                    q_ptr, kh_ptr, imp_ptr,
                    rc_ptr, fp_ptr, bc_ptr, ql_ptr, ct_ptr, d_ctr_ptr,
                    bh_start, S, n_blocks, q_stride, chunk_q, scale
                );
            } else {
                score_topk_and_scatter_128<<<dim3(cur_grp, n_blocks - TOP_N), 128, smem_score, stream>>>(
                    q_ptr, kh_ptr, imp_ptr,
                    rc_ptr, fp_ptr, bc_ptr, ql_ptr, ct_ptr, d_ctr_ptr,
                    bh_start, S, n_blocks, q_stride, chunk_q, scale
                );
            }

            remote_block_attn_kernel<D><<<persistent_ctas, 128, smem_rem, stream>>>(
                q_ptr, k_ptr, v_ptr, ql_ptr, ct_ptr, bc_ptr, d_ctr_ptr,
                op_ptr, mlsp_ptr, bh_start, cur_grp, S, n_blocks, q_stride, chunk_q, scale
            );
        }

        if constexpr (D == 64) {
            local_window_and_merge_64<<<dim3(cur_grp, n_blocks), 128, smem_loc, stream>>>(
                q_ptr, k_ptr, v_ptr, rc_ptr, fp_ptr, op_ptr, mlsp_ptr, d_ctr_ptr, o_ptr,
                bh_start, S, n_blocks, scale
            );
        } else {
            local_window_and_merge_128<<<dim3(cur_grp, n_blocks), 128, smem_loc, stream>>>(
                q_ptr, k_ptr, v_ptr, rc_ptr, fp_ptr, op_ptr, mlsp_ptr, d_ctr_ptr, o_ptr,
                bh_start, S, n_blocks, scale
            );
        }
    }
}

torch::Tensor nsa_forward(
    torch::Tensor q,
    torch::Tensor k,
    torch::Tensor v,
    int direct_max_blocks,
    int h_grp_budget_mb,
    int chunk_q
) {
    TORCH_CHECK(q.is_cuda() && k.is_cuda() && v.is_cuda(), "q, k, v must be CUDA tensors");
    const bool fast_ok = q.is_contiguous() && k.is_contiguous() && v.is_contiguous()
                      && q.scalar_type() == torch::kBFloat16
                      && k.scalar_type() == torch::kBFloat16
                      && v.scalar_type() == torch::kBFloat16;
    torch::Tensor qc = fast_ok ? q : (q.scalar_type() == torch::kBFloat16 ? q.contiguous() : q.to(torch::kBFloat16).contiguous());
    torch::Tensor kc = fast_ok ? k : (k.scalar_type() == torch::kBFloat16 ? k.contiguous() : k.to(torch::kBFloat16).contiguous());
    torch::Tensor vc = fast_ok ? v : (v.scalar_type() == torch::kBFloat16 ? v.contiguous() : v.to(torch::kBFloat16).contiguous());

    auto out = torch::empty_like(qc);
    int D = qc.size(3);
    if (D == 64) {
        dispatch_nsa_impl<64>(qc, kc, vc, out, direct_max_blocks, h_grp_budget_mb, chunk_q);
    } else if (D == 128) {
        dispatch_nsa_impl<128>(qc, kc, vc, out, direct_max_blocks, h_grp_budget_mb, chunk_q);
    } else {
        int B = qc.size(0), H = qc.size(1), S = qc.size(2);
        float scale = 1.0f / sqrtf(float(D));
        cudaStream_t stream = at::cuda::getCurrentCUDAStream();
        nsa_fallback_kernel<<<dim3(S, B * H), 128, 0, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(qc.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(kc.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(vc.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(out.data_ptr()),
            S, D, scale
        );
    }
    return out;
}
"""

CPP_SRC = r"""
torch::Tensor nsa_forward(
    torch::Tensor q,
    torch::Tensor k,
    torch::Tensor v,
    int direct_max_blocks,
    int h_grp_budget_mb,
    int chunk_q
);
"""

_MOD = None


def _get_mod():
    global _MOD
    if _MOD is None:
        _MOD = load_inline(
            name="deepseek_nsa_sm120_final",
            cpp_sources=CPP_SRC,
            cuda_sources=CUDA_SRC,
            functions=["nsa_forward"],
            extra_cuda_cflags=["-O3", "-arch=sm_120a", "--expt-relaxed-constexpr"],
            verbose=False,
        )
    return _MOD


class Model(nn.Module):
    def __init__(self, B: int, H: int, S: int, D: int):
        super().__init__()
        self.B, self.H, self.S, self.D = B, H, S, D
        self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16))
        self._mod = _get_mod()
        self._direct_max_blocks = int(os.environ.get("NSA_DIRECT_MAX_BLOCKS", "16"))
        self._h_grp_budget_mb = int(os.environ.get("NSA_H_GRP_BUDGET_MB", "72"))
        self._chunk_q = int(os.environ.get("NSA_CHUNK_Q", "0"))

    def forward(self, q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
        return self._mod.nsa_forward(
            q, k, v, self._direct_max_blocks, self._h_grp_budget_mb, self._chunk_q
        )

20261118_150515_agy_gemini-4-neon-xhigh_02_deepseek_nsa