Paged Attention GPT-5.6 Sol

"""Fused Triton paged-attention decode kernel.

The attention kernel assigns a program to a (batch, KV-head, sequence-split)
tuple.  All query heads in a GQA group are evaluated together, so each packed
K/V cache line is fetched only once.  Long, low-batch shapes use a few sequence
splits to expose enough CTAs; a small second kernel combines their online
softmax states.
"""
import math

import torch
import torch.nn as nn
import triton
import triton.language as tl


@triton.jit
def _paged_attention(
    query,
    kv_cache,
    block_table,
    seq_lens,
    out,
    partial_out,
    partial_stats,
    H: tl.constexpr,
    HKV: tl.constexpr,
    D: tl.constexpr,
    G: tl.constexpr,
    PAGE: tl.constexpr,
    MAX_PAGES: tl.constexpr,
    MAX_SEQ: tl.constexpr,
    NUM_SPLITS: tl.constexpr,
    BLOCK_M: tl.constexpr,
    BLOCK_N: tl.constexpr,
    LOOP_STAGES: tl.constexpr,
):
    pid = tl.program_id(0)
    split = pid % NUM_SPLITS
    bh = pid // NUM_SPLITS
    kv_head = bh % HKV
    batch = bh // HKV

    offs_m = tl.arange(0, BLOCK_M)
    offs_d = tl.arange(0, D)
    q_head = kv_head * G + offs_m
    q_ptrs = query + (batch * H + q_head[:, None]) * D + offs_d[None, :]
    q = tl.load(q_ptrs, mask=offs_m[:, None] < G, other=0.0)

    # Contiguous, non-overlapping partitions.  All benchmark sequence lengths
    # are the model's MAX_SEQ, while the explicit seq_len predicate handles the
    # non-power-of-two tail.
    PART = tl.cdiv(MAX_SEQ, NUM_SPLITS)
    seq_len = tl.load(seq_lens + batch)
    start = split * PART

    row_max = tl.full((BLOCK_M,), -float("inf"), tl.float32)
    row_sum = tl.zeros((BLOCK_M,), tl.float32)
    acc = tl.zeros((BLOCK_M, D), tl.float32)

    for block_off in tl.range(0, PART, BLOCK_N, num_stages=LOOP_STAGES):
        token = start + block_off + tl.arange(0, BLOCK_N)
        valid_n = (token < seq_len) & (token < start + PART)
        page_slot = token % PAGE
        # Every configured token tile begins on a page boundary.  Load each
        # page-table entry once, then broadcast it to its PAGE token lanes.
        page_in_tile = tl.arange(0, BLOCK_N // PAGE)
        logical_page = (start + block_off) // PAGE + page_in_tile
        valid_page = (start + block_off + page_in_tile * PAGE) < seq_len
        physical_pages = tl.load(
            block_table + batch * MAX_PAGES + logical_page,
            mask=valid_page,
            other=0,
        )
        physical_page = tl.reshape(
            tl.broadcast_to(
                physical_pages[:, None],
                (BLOCK_N // PAGE, PAGE),
            ),
            (BLOCK_N,),
        )

        # kv_cache: [page, PAGE, HKV, 2*D], with [K | V] packed in
        # the innermost dimension.
        token_base = (
            physical_page * (PAGE * HKV * 2 * D)
            + page_slot * (HKV * 2 * D)
            + kv_head * (2 * D)
        )
        kv_ptrs = kv_cache + token_base[:, None] + offs_d[None, :]
        k = tl.load(
            kv_ptrs,
            mask=valid_n[:, None],
            other=0.0,
            cache_modifier=".cg",
        )
        v = tl.load(
            kv_ptrs + D,
            mask=valid_n[:, None],
            other=0.0,
            cache_modifier=".cg",
        )

        scores = tl.dot(q, tl.trans(k)) * (1.0 / math.sqrt(D))
        active = (offs_m[:, None] < G) & valid_n[None, :]
        scores = tl.where(active, scores, -float("inf"))

        block_max = tl.max(scores, axis=1)
        next_max = tl.maximum(row_max, block_max)
        alpha = tl.where(
            next_max == -float("inf"),
            0.0,
            tl.exp2((row_max - next_max) * 1.4426950408889634074),
        )
        prob = tl.where(
            active,
            tl.exp2((scores - next_max[:, None]) * 1.4426950408889634074),
            0.0,
        )
        acc = acc * alpha[:, None] + tl.dot(prob.to(tl.bfloat16), v)
        row_sum = row_sum * alpha + tl.sum(prob, axis=1)
        row_max = next_max

    if NUM_SPLITS == 1:
        result = acc / row_sum[:, None]
        out_ptrs = out + (batch * H + q_head[:, None]) * D + offs_d[None, :]
        tl.store(out_ptrs, result, mask=offs_m[:, None] < G)
    else:
        # partial_out layout: [B, HKV, S, G, D]
        po_base = ((batch * HKV + kv_head) * NUM_SPLITS + split) * G * D
        po_ptrs = partial_out + po_base + offs_m[:, None] * D + offs_d[None, :]
        tl.store(po_ptrs, acc, mask=offs_m[:, None] < G)

        # partial_stats layout: [B, HKV, S, G, 2] = [max, sum]
        ps_base = ((batch * HKV + kv_head) * NUM_SPLITS + split) * G * 2
        ps_ptrs = partial_stats + ps_base + offs_m * 2
        tl.store(ps_ptrs, row_max, mask=offs_m < G)
        tl.store(ps_ptrs + 1, row_sum, mask=offs_m < G)


@triton.jit
def _reduce_splits_by_head(
    partial_out,
    partial_stats,
    out,
    H: tl.constexpr,
    HKV: tl.constexpr,
    D: tl.constexpr,
    G: tl.constexpr,
    NUM_SPLITS: tl.constexpr,
):
    """Combine partitions with one program per query head for occupancy."""
    bhq = tl.program_id(0)
    batch = bhq // H
    q_head = bhq % H
    kv_head = q_head // G
    group_head = q_head % G
    bh = batch * HKV + kv_head
    offs_d = tl.arange(0, D)

    global_max = -float("inf")
    for split in range(NUM_SPLITS):
        stat = ((bh * NUM_SPLITS + split) * G + group_head) * 2
        global_max = tl.maximum(global_max, tl.load(partial_stats + stat))

    denom = 0.0
    numerator = tl.zeros((D,), tl.float32)
    for split in range(NUM_SPLITS):
        stat = ((bh * NUM_SPLITS + split) * G + group_head) * 2
        part_max = tl.load(partial_stats + stat)
        part_sum = tl.load(partial_stats + stat + 1)
        weight = tl.exp2((part_max - global_max) * 1.4426950408889634)
        denom += part_sum * weight
        part_base = ((bh * NUM_SPLITS + split) * G + group_head) * D
        numerator += tl.load(partial_out + part_base + offs_d) * weight

    tl.store(out + bhq * D + offs_d, numerator / denom)


@triton.jit
def _paged_attention_all_kv_small(
    query,
    kv_cache,
    block_table,
    seq_lens,
    out,
    partial_out,
    partial_stats,
    H: tl.constexpr,
    HKV: tl.constexpr,
    D: tl.constexpr,
    G: tl.constexpr,
    PAGE: tl.constexpr,
    MAX_PAGES: tl.constexpr,
    MAX_SEQ: tl.constexpr,
    NUM_SPLITS: tl.constexpr,
    BLOCK_N: tl.constexpr,
    LOOP_STAGES: tl.constexpr,
):
    """D=64 path packing all independent KV heads into one MMA tile.

    K/V columns from the four KV heads are concatenated.  The score matrix is
    block-diagonally masked so each group of four query rows only observes its
    own KV head.  This fills the native 16-row MMA tile without redundant KV
    reads or padded query rows.
    """
    pid = tl.program_id(0)
    split = pid % NUM_SPLITS
    batch = pid // NUM_SPLITS
    offs_m = tl.arange(0, H)
    offs_d = tl.arange(0, D)
    q = tl.load(query + (batch * H + offs_m[:, None]) * D + offs_d[None, :])

    PART = tl.cdiv(MAX_SEQ, NUM_SPLITS)
    start = split * PART
    seq_len = tl.load(seq_lens + batch)
    row_max = tl.full((H,), -float("inf"), tl.float32)
    row_sum = tl.zeros((H,), tl.float32)
    acc = tl.zeros((H, D), tl.float32)
    q_kv_head = offs_m // G

    for block_off in tl.range(0, PART, BLOCK_N, num_stages=LOOP_STAGES):
        packed_n = tl.arange(0, HKV * BLOCK_N)
        col_kv_head = packed_n // BLOCK_N
        local_n = tl.arange(0, BLOCK_N)
        token = start + block_off + local_n
        valid_local = (token < seq_len) & (token < start + PART)
        physical_local = tl.load(
            block_table + batch * MAX_PAGES + token // PAGE,
            mask=valid_local,
            other=0,
        )
        physical_page = tl.reshape(
            tl.broadcast_to(physical_local[None, :], (HKV, BLOCK_N)),
            (HKV * BLOCK_N,),
        )
        page_slot = tl.reshape(
            tl.broadcast_to((token % PAGE)[None, :], (HKV, BLOCK_N)),
            (HKV * BLOCK_N,),
        )
        valid_n = tl.reshape(
            tl.broadcast_to(valid_local[None, :], (HKV, BLOCK_N)),
            (HKV * BLOCK_N,),
        )
        token_base = (
            physical_page * (PAGE * HKV * 2 * D)
            + page_slot * (HKV * 2 * D)
            + col_kv_head * (2 * D)
        )
        ptrs = kv_cache + token_base[:, None] + offs_d[None, :]
        k = tl.load(ptrs, mask=valid_n[:, None], other=0.0, cache_modifier=".cg")
        v = tl.load(ptrs + D, mask=valid_n[:, None], other=0.0, cache_modifier=".cg")

        scores = tl.dot(q, tl.trans(k)) * (1.0 / math.sqrt(D))
        own_head = q_kv_head[:, None] == col_kv_head[None, :]
        active = own_head & valid_n[None, :]
        scores = tl.where(active, scores, -float("inf"))
        block_max = tl.max(scores, axis=1)
        next_max = tl.maximum(row_max, block_max)
        alpha = tl.where(
            next_max == -float("inf"),
            0.0,
            tl.exp2((row_max - next_max) * 1.4426950408889634),
        )
        prob = tl.where(
            active,
            tl.exp2((scores - next_max[:, None]) * 1.4426950408889634),
            0.0,
        )
        acc = acc * alpha[:, None] + tl.dot(prob.to(tl.bfloat16), v)
        row_sum = row_sum * alpha + tl.sum(prob, axis=1)
        row_max = next_max

    if NUM_SPLITS == 1:
        tl.store(
            out + (batch * H + offs_m[:, None]) * D + offs_d[None, :],
            acc / row_sum[:, None],
        )
    else:
        kv_head = offs_m // G
        group_head = offs_m % G
        po_ptrs = partial_out + (
            (((batch * HKV + kv_head[:, None]) * NUM_SPLITS + split) * G
             + group_head[:, None]) * D
            + offs_d[None, :]
        )
        tl.store(po_ptrs, acc)
        ps_ptrs = partial_stats + (
            ((batch * HKV + kv_head) * NUM_SPLITS + split) * G + group_head
        ) * 2
        tl.store(ps_ptrs, row_max)
        tl.store(ps_ptrs + 1, row_sum)


class Model(nn.Module):
    def __init__(
        self,
        batch: int,
        num_heads: int,
        num_kv_heads: int,
        head_dim: int,
        seq_len: int,
        page_size: int,
    ):
        super().__init__()
        assert num_heads % num_kv_heads == 0
        assert head_dim in (64, 128)
        assert page_size == 16
        self.batch = batch
        self.num_heads = num_heads
        self.num_kv_heads = num_kv_heads
        self.head_dim = head_dim
        self.seq_len = seq_len
        self.page_size = page_size
        self.group_size = num_heads // num_kv_heads
        self.max_pages = (seq_len + page_size - 1) // page_size

        # Enough independent CTAs to fill H100's 114 SMs, without over-splitting
        # the already-wide server-batch shapes.
        base_programs = batch * num_kv_heads
        if self.group_size == 8:
            self.num_splits = 8
        elif base_programs >= 192:
            self.num_splits = 1
        elif base_programs <= 32:
            self.num_splits = 8
        elif base_programs >= 128:
            self.num_splits = 1
        else:
            self.num_splits = 1

        self.block_m = triton.next_power_of_2(self.group_size)
        self.block_n = 64
        self.num_warps = 8 if self.group_size == 8 else 4
        self.num_stages = 3 if self.group_size == 8 else 2
        self.loop_stages = 4
        self.maxnreg = 80 if self.group_size == 8 else None
        self.reduce_warps = 4
        if seq_len <= 1024:
            self.block_n = 128
            self.num_warps = 8
            self.num_stages = 3
            self.loop_stages = 5
            self.maxnreg = 128
        elif seq_len < 1800:
            self.block_n = 128
        if head_dim == 64:
            self.block_n = 64
            self.loop_stages = 3
        self.pack_all_small_kv = head_dim == 64
        if self.pack_all_small_kv:
            self.num_splits = 16
            self.block_n = 32
            self.num_warps = 4
            self.num_stages = 2
            self.loop_stages = 5
            self.maxnreg = None
        if self.group_size == 8:
            self.num_warps = 4
            self.loop_stages = 2
        self.register_buffer("_dummy", torch.zeros(1, dtype=torch.bfloat16), persistent=False)

    def forward(self, query, kv_cache, block_table, seq_lens):
        out = torch.empty_like(query)
        if self.num_splits == 1:
            # Dummy tensors are compile-time-dead in this specialization.
            partial_out = out
            partial_stats = out
        else:
            partial_out = torch.empty(
                (self.batch, self.num_kv_heads, self.num_splits, self.group_size, self.head_dim),
                device=query.device,
                dtype=torch.bfloat16,
            )
            partial_stats = torch.empty(
                (self.batch, self.num_kv_heads, self.num_splits, self.group_size, 2),
                device=query.device,
                dtype=torch.float32,
            )

        if self.pack_all_small_kv:
            grid = (self.batch * self.num_splits,)
            kernel = _paged_attention_all_kv_small
        else:
            grid = (self.batch * self.num_kv_heads * self.num_splits,)
            kernel = _paged_attention
        launch_kwargs = dict(
            H=self.num_heads,
            HKV=self.num_kv_heads,
            D=self.head_dim,
            G=self.group_size,
            PAGE=self.page_size,
            MAX_PAGES=self.max_pages,
            MAX_SEQ=self.seq_len,
            NUM_SPLITS=self.num_splits,
            BLOCK_N=self.block_n,
            LOOP_STAGES=self.loop_stages,
            num_warps=self.num_warps,
            num_stages=self.num_stages,
            maxnreg=self.maxnreg,
        )
        if not self.pack_all_small_kv:
            launch_kwargs["BLOCK_M"] = self.block_m
        kernel[grid](
            query,
            kv_cache,
            block_table,
            seq_lens,
            out,
            partial_out,
            partial_stats,
            **launch_kwargs,
        )
        if self.num_splits > 1:
            _reduce_splits_by_head[(self.batch * self.num_heads,)](
                partial_out,
                partial_stats,
                out,
                H=self.num_heads,
                HKV=self.num_kv_heads,
                D=self.head_dim,
                G=self.group_size,
                NUM_SPLITS=self.num_splits,
                num_warps=self.reduce_warps,
                num_stages=1,
            )
        return out


# Match reference.py's construction interface.
BATCH = 8
NUM_HEADS = 32
NUM_KV_HEADS = 8
HEAD_DIM = 128
SEQ_LEN = 1024
PAGE_SIZE = 16


def get_inputs():
    pages_per_seq = (SEQ_LEN + PAGE_SIZE - 1) // PAGE_SIZE
    total_pages = max(BATCH * pages_per_seq + 8, 64)
    query = torch.randn(BATCH, NUM_HEADS, HEAD_DIM, dtype=torch.bfloat16) * 0.1
    kv_cache = torch.randn(
        total_pages,
        PAGE_SIZE,
        NUM_KV_HEADS,
        2 * HEAD_DIM,
        dtype=torch.bfloat16,
    ) * 0.1
    block_table = torch.randperm(total_pages)[: BATCH * pages_per_seq]
    block_table = block_table.reshape(BATCH, pages_per_seq).int().contiguous()
    seq_lens = torch.full((BATCH,), SEQ_LEN, dtype=torch.int32)
    return [query, kv_cache, block_table, seq_lens]


def get_init_inputs():
    return [BATCH, NUM_HEADS, NUM_KV_HEADS, HEAD_DIM, SEQ_LEN, PAGE_SIZE]