KernelBench hard · RTX PRO 6000

Paged Attention DeepSeek V4.1 Flash (1017)

45.9%geomean peak fraction across shapes

manually audited: clean

One fused split-KV flash-decode kernel (load_inline CUDA): pages stream through cp.async with an L2 evict-first hint, one lane group per GQA head, and the last block to finish merges the partials through an atomic ticket. The split count is planned from the init seq_len and seq_lens is read at runtime.

harnessor-fableagent session48mtotal wall49mcheck13sbenchmark5soutput tokens221,253cost$15.19gpu-lock wait0sgpu-lock held7mregimememory

Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth

8×32×8×128×1024×160.046 ms41.0%0.74 TB/s · 41% of 1.8 TB/s HBM · also 3 TFLOPS (1% of compute)
32×32×8×128×2048×160.220 ms67.9%1.22 TB/s · 68% of 1.8 TB/s HBM · also 5 TFLOPS (1% of compute)
4×64×8×128×4096×160.082 ms45.8%0.82 TB/s · 46% of 1.8 TB/s HBM · also 7 TFLOPS (1% of compute)
16×32×8×128×1535×160.104 ms53.7%0.97 TB/s · 54% of 1.8 TB/s HBM · also 4 TFLOPS (1% of compute)
8×16×4×64×2000×160.031 ms29.6%0.53 TB/s · 30% of 1.8 TB/s HBM · also 2 TFLOPS (0% of compute)

compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)

geomean(41.0% · 67.9% · 45.8% · 53.7% · 29.6%) = 45.9%

Kernel source (redacted)
"""Paged-attention decode kernel (single-query, GQA) for RTX PRO 6000 (SM120).

Single fused kernel, split-KV flash decoding:

  grid = (num_splits, num_kv_heads, batch).  One block streams a page-aligned
  chunk of one (batch, kv_head) KV stream through a cp.async multi-stage
  shared-memory pipeline and accumulates (max, sumexp, sum p*V) for the GQA
  query heads it owns (a lane-group per head, so a GQA group shares one read of
  the KV stream).

  The last block to finish a (batch, kv_head) stream -- tracked with an atomic
  ticket, CUDA threadFenceReduction style -- merges that stream's per-split
  partials with a log-sum-exp and writes the bf16 output, so the whole decode
  is a single kernel launch.

The KV cache layout is (num_blocks, page_size, num_kv_heads, 2*head_dim) with
K|V packed on the last dim, so one token of one kv head is 4*head_dim
contiguous bytes.
"""

import math
import os
import shutil
import sys

import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

# torch's JIT build shells out to ninja; make sure the interpreter's own bin
# directory (where ninja usually lives) is on PATH even when the venv was not
# "activated" by the caller.
if shutil.which("ninja") is None:
    _bindir = os.path.dirname(sys.executable)
    if os.path.exists(os.path.join(_bindir, "ninja")):
        os.environ["PATH"] = _bindir + os.pathsep + os.environ.get("PATH", "")

_CUDA_SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <math_constants.h>
#include <cstdint>

#define DEV __device__ __forceinline__



DEV uint32_t sptr_of(const void* p) {
  return static_cast<uint32_t>(__cvta_generic_to_shared(p));
}

DEV void cp_async16(void* dst, const void* src) {
  asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" ::"r"(sptr_of(dst)), "l"(src));
}

DEV uint64_t evict_first_policy() {
  uint64_t p;
  asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(p));
  return p;
}

DEV void cp_async16_pol(void* dst, const void* src, uint64_t pol) {
  asm volatile("cp.async.cg.shared.global.L2::cache_hint [%0], [%1], 16, %2;\n" ::"r"(sptr_of(dst)),
               "l"(src), "l"(pol));
}

DEV void cp_commit() { asm volatile("cp.async.commit_group;\n" ::); }

template <int N>
DEV void cp_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); }

// L1-bypassing loads: partials were written by *other* blocks, so the
// non-coherent L1 must not serve them.
DEV float ldcg_f(const float* p) {
  float v;
  asm volatile("ld.global.cg.f32 %0, [%1];" : "=f"(v) : "l"(p));
  return v;
}

DEV float4 ldcg4(const float* p) {
  float4 v;
  asm volatile("ld.global.cg.v4.f32 {%0,%1,%2,%3}, [%4];"
               : "=f"(v.x), "=f"(v.y), "=f"(v.z), "=f"(v.w) : "l"(p));
  return v;
}

template <int EPL>
DEV void ld_bf16_f32(const void* p, float* dst) {
  if constexpr (EPL == 8) {
    uint4 r = *reinterpret_cast<const uint4*>(p);
    const uint32_t* w = reinterpret_cast<const uint32_t*>(&r);
    #pragma unroll
    for (int i = 0; i < 4; ++i) {
      __nv_bfloat162 h = *reinterpret_cast<const __nv_bfloat162*>(&w[i]);
      float2 f = __bfloat1622float2(h);
      dst[2 * i] = f.x;
      dst[2 * i + 1] = f.y;
    }
  } else if constexpr (EPL == 4) {
    uint2 r = *reinterpret_cast<const uint2*>(p);
    __nv_bfloat162 h0 = *reinterpret_cast<const __nv_bfloat162*>(&r.x);
    __nv_bfloat162 h1 = *reinterpret_cast<const __nv_bfloat162*>(&r.y);
    float2 f0 = __bfloat1622float2(h0);
    float2 f1 = __bfloat1622float2(h1);
    dst[0] = f0.x; dst[1] = f0.y; dst[2] = f1.x; dst[3] = f1.y;
  } else if constexpr (EPL == 2) {
    uint32_t r = *reinterpret_cast<const uint32_t*>(p);
    __nv_bfloat162 h0 = *reinterpret_cast<const __nv_bfloat162*>(&r);
    float2 f0 = __bfloat1622float2(h0);
    dst[0] = f0.x; dst[1] = f0.y;
  } else {
    const __nv_bfloat16* h = reinterpret_cast<const __nv_bfloat16*>(p);
    #pragma unroll
    for (int i = 0; i < EPL; ++i) dst[i] = __bfloat162float(h[i]);
  }
}

// ---------------------------------------------------------------------------
// fused split-KV attention + cross-split combine
//   LPH : lanes per query head (32 -> a head per warp, 16 -> two per warp)
//   EHINT : tag KV loads as L2 evict-first (single-use stream)
// ---------------------------------------------------------------------------
template <int D, int G, int STAGES, int LPH, int EHINT>
__global__ void __launch_bounds__((G * LPH / 32) * 32)
paged_attn_kernel(const __nv_bfloat16* __restrict__ kv,
                  const __nv_bfloat16* __restrict__ q,
                  const int* __restrict__ btab,
                  const int* __restrict__ seqlens,
                  __nv_bfloat16* __restrict__ out,
                  float* __restrict__ pacc,
                  float* __restrict__ pml,
                  int* __restrict__ cnt,
                  int Hkv, int P, int max_pages, int S, int pps, float scale) {
  constexpr int WARPS = G * LPH / 32;
  constexpr int HPW = 32 / LPH;   // heads handled concurrently per warp
  constexpr int EPL = D / LPH;    // elements of a head per lane
  constexpr int TOK_BYTES = 4 * D;
  constexpr int SEGS = D / 4;     // 16B segments per token of one kv head
  constexpr int NTHREADS = WARPS * 32;

  const int tid = threadIdx.x;
  const int warp = tid >> 5;
  const int lane = tid & 31;
  const int group = lane / LPH;   // lane-group inside the warp
  const int sub = lane % LPH;     // lane inside its group
  const int b = blockIdx.z;
  const int kvh = blockIdx.y;
  const int sp = blockIdx.x;

  const int myhead = warp * HPW + group;          // offset inside the GQA group
  const long nh = (long)b * (Hkv * G) + kvh * G + myhead;  // q / out / partial row

  const int L = seqlens[b];
  const int npages = (L + P - 1) / P;
  const int p0 = sp * pps;
  const int p1 = min(npages, p0 + pps);

  extern __shared__ char smem[];
  __shared__ int ticket;

  const int smem_page = P * TOK_BYTES;
  const long page_bytes = (long)P * Hkv * TOK_BYTES;
  const long kvh_off = (long)kvh * TOK_BYTES;
  const char* kvbase = reinterpret_cast<const char*>(kv);
  const int gstride = Hkv * TOK_BYTES;
  const int* bt = btab + (long)b * max_pages;
  const int nseg_page = P * SEGS;
  const uint64_t pol = EHINT ? evict_first_policy() : 0ull;

  const int mlbase = (int)(nh * S + sp) * 2;
  const long acbase = (nh * S + sp) * (long)D;

  if (p0 < p1) {
    auto load_page = [&](int stage, int p) {
      const int pg = bt[p];
      const char* src = kvbase + (long)pg * page_bytes + kvh_off;
      char* dst = smem + stage * smem_page;
      for (int seg = tid; seg < nseg_page; seg += NTHREADS) {
        const int t = seg / SEGS;
        const int o = seg - t * SEGS;
        const char* s = src + (long)t * gstride + o * 16;
        char* d = dst + t * TOK_BYTES + o * 16;
        if constexpr (EHINT) cp_async16_pol(d, s, pol);
        else cp_async16(d, s);
      }
    };

    float qv[EPL];
    ld_bf16_f32<EPL>(q + nh * D + sub * EPL, qv);

    float acc[EPL];
    #pragma unroll
    for (int i = 0; i < EPL; ++i) acc[i] = 0.f;
    float mx = -CUDART_INF_F;
    float lsum = 0.f;

    #pragma unroll
    for (int s = 0; s < STAGES - 1; ++s) {
      if (p0 + s < p1) load_page(s, p0 + s);
      cp_commit();
    }

    for (int p = p0; p < p1; ++p) {
      const int iter = p - p0;
      const int stage = iter % STAGES;
      const int pn = p + STAGES - 1;
      __syncthreads();
      if (pn < p1) load_page((iter + STAGES - 1) % STAGES, pn);
      cp_commit();
      cp_wait<STAGES - 1>();
      __syncthreads();

      const int ntok = min(P, L - p * P);
      const char* pg = smem + stage * smem_page;
      for (int t = 0; t < ntok; ++t) {
        const char* ks = pg + t * TOK_BYTES;
        float kf[EPL], vf[EPL];
        ld_bf16_f32<EPL>(ks + sub * EPL * 2, kf);
        ld_bf16_f32<EPL>(ks + 2 * D + sub * EPL * 2, vf);
        float s = 0.f;
        #pragma unroll
        for (int i = 0; i < EPL; ++i) s += qv[i] * kf[i];
        #pragma unroll
        for (int off = LPH / 2; off > 0; off >>= 1) s += __shfl_xor_sync(0xffffffffu, s, off);
        s *= scale;
        if (s > mx) {                      // warp-uniform in practice
          const float c = __expf(mx - s);
          lsum *= c;
          #pragma unroll
          for (int i = 0; i < EPL; ++i) acc[i] *= c;
          mx = s;
        }
        const float pr = __expf(s - mx);
        lsum += pr;
        #pragma unroll
        for (int i = 0; i < EPL; ++i) acc[i] += pr * vf[i];
      }
    }

    #pragma unroll
    for (int i = 0; i < EPL; ++i) pacc[acbase + sub * EPL + i] = acc[i];
    if (sub == 0) {
      pml[mlbase] = mx;
      pml[mlbase + 1] = lsum;
    }
  } else {
    // no pages for this split: publish a neutral partial so the merge is a
    // fixed shape
    if (myhead < G) {
      #pragma unroll
      for (int i = 0; i < EPL; ++i) pacc[acbase + sub * EPL + i] = 0.f;
      if (sub == 0) {
        pml[mlbase] = -CUDART_INF_F;
        pml[mlbase + 1] = 0.f;
      }
    }
  }

  // ---- cross-split combine: the last block of this stream does it ----------
  __threadfence();
  __syncthreads();
  if (tid == 0) ticket = atomicAdd(&cnt[b * Hkv + kvh], 1);
  __syncthreads();
  if (ticket != S - 1) return;

  if (myhead < G) {
    const float* ml = pml + nh * S * 2;
    const float* ac = pacc + nh * S * (long)D + sub * EPL;
    float m = -CUDART_INF_F;
    for (int s = 0; s < S; ++s) m = fmaxf(m, ldcg_f(ml + 2 * s));
    float a[EPL];
    #pragma unroll
    for (int i = 0; i < EPL; ++i) a[i] = 0.f;
    float ls = 0.f;
    for (int s = 0; s < S; ++s) {
      const float ms = ldcg_f(ml + 2 * s);
      const float c = (ms == -CUDART_INF_F) ? 0.f : __expf(ms - m);
      ls += ldcg_f(ml + 2 * s + 1) * c;
      const float* p = ac + s * (long)D;
      if constexpr (EPL % 4 == 0) {
        #pragma unroll
        for (int i = 0; i < EPL; i += 4) {
          float4 v = ldcg4(p + i);
          a[i] += v.x * c; a[i + 1] += v.y * c; a[i + 2] += v.z * c; a[i + 3] += v.w * c;
        }
      } else {
        #pragma unroll
        for (int i = 0; i < EPL; ++i) a[i] += ldcg_f(p + i) * c;
      }
    }
    const float inv = (ls > 0.f) ? (1.f / ls) : 0.f;
    __nv_bfloat16* o = out + nh * D + sub * EPL;
    #pragma unroll
    for (int i = 0; i < EPL; ++i) o[i] = __float2bfloat16(a[i] * inv);
  }
  if (tid == 0) cnt[b * Hkv + kvh] = 0;   // leave the ticket counter reusable
}

// ---------------------------------------------------------------------------
// host wrapper
// ---------------------------------------------------------------------------
void paged_attn_decode(torch::Tensor q, torch::Tensor kv, torch::Tensor btab,
                       torch::Tensor seqlens, torch::Tensor out,
                       torch::Tensor pacc, torch::Tensor pml, torch::Tensor cnt,
                       int64_t S, int64_t pps, double scale, int64_t lph,
                       int64_t stages) {
  const int B = q.size(0);
  const int H = q.size(1);
  const int D = q.size(2);
  const int Hkv = kv.size(2);
  const int P = kv.size(1);
  const int G = H / Hkv;
  const int max_pages = btab.size(1);

  const __nv_bfloat16* kvp = reinterpret_cast<const __nv_bfloat16*>(kv.data_ptr());
  const __nv_bfloat16* qp = reinterpret_cast<const __nv_bfloat16*>(q.data_ptr());
  const int* btp = btab.data_ptr<int>();
  const int* slp = seqlens.data_ptr<int>();
  float* pac = pacc.data_ptr<float>();
  float* pmlp = pml.data_ptr<float>();
  int* cntp = cnt.data_ptr<int>();
  __nv_bfloat16* op = reinterpret_cast<__nv_bfloat16*>(out.data_ptr());

  dim3 grid((unsigned)S, (unsigned)Hkv, (unsigned)B);
  cudaStream_t stream = at::cuda::getCurrentCUDAStream();

#define LAUNCH(D_, G_, ST_, LPH_, HI_)                                                       \
  do {                                                                                       \
    const int smem = (ST_) * P * 4 * (D_);                                                   \
    paged_attn_kernel<D_, G_, ST_, LPH_, HI_>                                                  \
        <<<grid, ((G_) * (LPH_) / 32) * 32, smem, stream>>>(                                \
        kvp, qp, btp, slp, op, pac, pmlp, cntp, Hkv, P, max_pages, (int)S, (int)pps,          \
        (float)scale);                                                                        \
  } while (0)

#define DISPATCH_ST(D_, G_, LPH_)                                       \
  do {                                                                  \
    if (HINT_ON && STAGES == 3) LAUNCH(D_, G_, 3, LPH_, 1);             \
    else if (STAGES == 2) LAUNCH(D_, G_, 2, LPH_, 0);                   \
    else if (STAGES == 3) LAUNCH(D_, G_, 3, LPH_, 0);                   \
    else LAUNCH(D_, G_, 4, LPH_, 0);                                    \
  } while (0)

#define DISPATCH_G(D_)                                     \
  do {                                                     \
    if (G == 4) {                                          \
      if (lph == 16) DISPATCH_ST(D_, 4, 16);               \
      else DISPATCH_ST(D_, 4, 32);                         \
    } else if (G == 8) {                                   \
      if (lph == 16) DISPATCH_ST(D_, 8, 16);               \
      else DISPATCH_ST(D_, 8, 32);                         \
    } else if (G == 2) {                                   \
      if (lph == 16) DISPATCH_ST(D_, 2, 16);               \
      else DISPATCH_ST(D_, 2, 32);                         \
    } else if (G == 16) {                                  \
      DISPATCH_ST(D_, 16, 32);                              \
    } else if (G == 1) {                                   \
      DISPATCH_ST(D_, 1, 32);                              \
    } else {                                               \
      TORCH_CHECK(false, "unsupported group size ", G);    \
    }                                                      \
  } while (0)

  const int STAGES = (int)stages;
  const bool HINT_ON = HINT != 0;
  if (D == 128) {
    DISPATCH_G(128);
  } else if (D == 64) {
    DISPATCH_G(64);
  } else {
    TORCH_CHECK(false, "unsupported head_dim ", D);
  }
}
"""

_CPP_SRC = r"""
#include <torch/extension.h>
void paged_attn_decode(torch::Tensor q, torch::Tensor kv, torch::Tensor btab,
                       torch::Tensor seqlens, torch::Tensor out,
                       torch::Tensor pacc, torch::Tensor pml, torch::Tensor cnt,
                       int64_t S, int64_t pps, double scale, int64_t lph,
                       int64_t stages);
"""

# --------------------------------------------------------------------------
# Scheduling.  All blocks are resident at once (one "wave"), so the split count
# is a direct trade: more blocks buy DRAM-level parallelism but shorten each
# block's page run, and past ~2.7 blocks/SM the extra concurrent streams start
# costing DRAM row locality.  Measured optimum on this part is ~512 blocks for
# 4-warp blocks; 8-warp blocks carry twice the in-flight bytes each, so fewer
# of them (~350) is better.  Pipeline depth follows the same logic: with twice
# the warps per block, halve the stage count to keep in-flight bytes constant.
# --------------------------------------------------------------------------
_PPS_OVERRIDE = int(os.environ.get("KBH_PPS", "0"))
_LPH_OVERRIDE = int(os.environ.get("KBH_LPH", "0"))
_STAGES_OVERRIDE = int(os.environ.get("KBH_STAGES", "0"))
_EXT = None


def _ext():
    global _EXT
    if _EXT is None:
        _EXT = load_inline(
            name="paged_attn_decode_sm120",
            cpp_sources=_CPP_SRC,
            cuda_sources=_CUDA_SRC,
            functions=["paged_attn_decode"],
            extra_cflags=["-std=c++20", "-O3"],
            extra_cuda_cflags=[
                "-O3",
                "--use_fast_math",
                "-arch=sm_120",
                "-std=c++20",
                "-DHINT=" + os.environ.get("KBH_HINT", "1"),
            ],
            verbose=os.environ.get("KBH_VERBOSE_BUILD") == "1",
        )
    return _EXT


def _warps_per_block(num_heads, num_kv_heads, lph):
    return max(1, (num_heads // num_kv_heads) * lph // 32)


def _plan(batch, num_heads, num_kv_heads, seq_len, page_size, lph, warps):
    pages = (seq_len + page_size - 1) // page_size
    streams = batch * num_kv_heads
    pps = _PPS_OVERRIDE
    if pps <= 0:
        target = 352 if warps >= 8 else 512
        splits = max(1, min(pages, int(round(float(target) / streams))))
        # prefer a split count that divides the pages: page ranges stay balanced,
        # and the slowest block sets the kernel time.
        lo, hi = max(1, -(-4 * splits // 5)), max(1, (5 * splits) // 4)
        divs = [d for d in range(lo, hi + 1) if pages % d == 0]
        if divs:
            splits = min(divs, key=lambda d: (abs(d - splits), d))
        pps = (pages + splits - 1) // splits
    splits = (pages + pps - 1) // pps
    return splits, pps


class Model(nn.Module):
    def __init__(self, batch, num_heads, num_kv_heads, head_dim, seq_len, page_size):
        super().__init__()
        assert num_heads % num_kv_heads == 0
        self.batch = batch
        self.num_heads = num_heads
        self.num_kv_heads = num_kv_heads
        self.head_dim = head_dim
        self.seq_len = seq_len
        self.page_size = page_size
        self.scale = 1.0 / math.sqrt(head_dim)
        self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16), persistent=False)
        self.lph = _LPH_OVERRIDE
        warps = _warps_per_block(num_heads, num_kv_heads, self.lph or 32)
        self.stages = _STAGES_OVERRIDE or (2 if warps >= 8 else 3)
        self.num_splits, self.pages_per_split = _plan(
            batch, num_heads, num_kv_heads, seq_len, page_size, self.lph, warps
        )
        self._fn = None
        self._args = None
        self._pacc = None
        self._pml = None
        self._cnt = None
        self._out = None

    def _launch_args(self, query, kv_cache, block_table, seq_lens):
        """Everything the extension call needs, built once per distinct input set."""
        if self._fn is None:
            self._fn = _ext().paged_attn_decode
            B, H, D = query.shape
            dev = query.device
            S = self.num_splits
            self._pacc = torch.empty(B * H * S * D, dtype=torch.float32, device=dev)
            self._pml = torch.empty(B * H * S * 2, dtype=torch.float32, device=dev)
            self._cnt = torch.zeros(B * self.num_kv_heads, dtype=torch.int32, device=dev)
            self._out = torch.empty((B, H, D), dtype=query.dtype, device=dev)
        self._args = (
            query, kv_cache, block_table, seq_lens, self._out,
            self._pacc, self._pml, self._cnt,
            self.num_splits, self.pages_per_split, self.scale, self.lph, self.stages,
        )

    def forward(self, query, kv_cache, block_table, seq_lens):
        args = self._args
        if (args is None or args[0] is not query or args[1] is not kv_cache
                or args[2] is not block_table or args[3] is not seq_lens):
            self._launch_args(query, kv_cache, block_table, seq_lens)
            args = self._args
        self._fn(*args)
        return self._out


# --- shape knobs (mirrors reference.py; overridden by the harness) --------
BATCH = 8
NUM_HEADS = 32
NUM_KV_HEADS = 8
HEAD_DIM = 128
SEQ_LEN = 1024
PAGE_SIZE = 16


def get_inputs():
    B, H, Hkv, D, L, P = BATCH, NUM_HEADS, NUM_KV_HEADS, HEAD_DIM, SEQ_LEN, PAGE_SIZE
    pages_per_seq = (L + P - 1) // P
    total_pages = max(B * pages_per_seq + 8, 64)
    query = torch.randn(B, H, D, dtype=torch.bfloat16) * 0.1
    kv_cache = torch.randn(total_pages, P, Hkv, 2 * D, dtype=torch.bfloat16) * 0.1
    perm = torch.randperm(total_pages)[: B * pages_per_seq].reshape(B, pages_per_seq).int()
    block_table = perm.contiguous()
    seq_lens = torch.full((B,), L, dtype=torch.int32)
    return [query, kv_cache, block_table, seq_lens]


def get_init_inputs():
    return [BATCH, NUM_HEADS, NUM_KV_HEADS, HEAD_DIM, SEQ_LEN, PAGE_SIZE]

20261017_215307_or-fable_deepseek_deepseek-v4.1-flash-1017_03_paged_attention