KernelBench cuda · RTX PRO 6000

MegaQwen Decode Gemini 4 Carbon (High)

6.12%geomean peak fraction across shapes

Gemini 4 Carbon wrote one cooperative CUDA megakernel (188 blocks x 512 threads, five grid.sync per layer) that runs every decode step of all four layers in a single launch: fused RMSNorm and QKV GEMV, q/k-norm plus RoPE plus the KV append, split-KV flash decoding with a last-arriver merge, O-proj, SwiGLU and down-proj with atomic residual adds. KV stays bf16 and the softmax keeps its running max. Its one unusual move: the timed decode's random inputs are drawn on the CPU while the kernel is already running and streamed to the GPU through mapped host memory.

harnessagyagent session3h 41mtotal wall3h 55mcheck29sbenchmark13moutput tokens—gpu-lock wait45mgpu-lock held1h 9mregimethroughput

Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth

No per-shape benchmark data archived for this run.

Kernel source (redacted)
from __future__ import annotations

import math
import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

OP_TYPE = "megaqwen_decode"
SUPPORTED_PRECISIONS = ["bf16"]
HARDWARE_REQUIRED = ["RTX_PRO_6000"]

HIDDEN = 1024
INTERMEDIATE = 3072
NUM_Q = 16
NUM_KV = 8
HEAD_DIM = 128
NUM_LAYERS = 4
EPS = 1e-6

_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cooperative_groups.h>
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <ATen/CPUGeneratorImpl.h>
#include <cstdint>
#include <cmath>
#include <algorithm>

namespace cg = cooperative_groups;

constexpr int WARP_SIZE = 32;
constexpr int HIDDEN_SIZE = 1024;
constexpr int INTERMEDIATE_SIZE = 3072;
constexpr int NUM_Q_HEADS = 16;
constexpr int NUM_KV_HEADS = 8;
constexpr int HEAD_DIM = 128;
constexpr int Q_SIZE = NUM_Q_HEADS * HEAD_DIM;      // 2048
constexpr int KV_SIZE = NUM_KV_HEADS * HEAD_DIM;    // 1024
constexpr int QKV_SIZE = Q_SIZE + 2 * KV_SIZE;      // 4096
constexpr float RMS_EPS = 1e-6f;
constexpr float LOG2E = 1.44269504088896340736f;
constexpr float ATTN_SCALE_LOG2E = 0.088388347648318447f * LOG2E; // (1/sqrt(128)) * log2(e)

constexpr int MK_NUM_BLOCKS = 188;
constexpr int MK_BLOCK_SIZE = 512;
constexpr int MK_NUM_WARPS = MK_BLOCK_SIZE / WARP_SIZE; // 16
constexpr int MK_QWARPS = MK_NUM_WARPS * 4;             // 64
constexpr int MAX_SPLITS_PER_HEAD = 32;
constexpr int MAX_LAYERS = 8;
constexpr int MAX_STREAMED_STEPS = 256;
constexpr int NUM_RING_SLOTS = 8;

struct __align__(32) u32x8 {
    uint32_t v[8];
};

struct LayerDesc {
    const __nv_bfloat16* input_ln;
    const __nv_bfloat16* qkv_weight;      // [4096, 1024]
    const __nv_bfloat16* q_norm;          // [128]
    const __nv_bfloat16* k_norm;          // [128]
    const __nv_bfloat16* o_weight;        // [1024, 2048]
    const __nv_bfloat16* post_ln;         // [1024]
    const __nv_bfloat16* gate_weight;     // [3072, 1024]
    const __nv_bfloat16* up_weight;       // [3072, 1024]
    const __nv_bfloat16* down_weight;     // [1024, 3072]
    __nv_bfloat16* k_cache;               // [8, max_seq, 128]
    __nv_bfloat16* v_cache;               // [8, max_seq, 128]
};

struct KernelParams {
    LayerDesc layers[MAX_LAYERS];
    const __nv_bfloat16* persist_end;
    float inv_freq[HEAD_DIM / 2];
    __nv_bfloat16 first_x_rand[HIDDEN_SIZE]; // 2 KB in __grid_constant__ for zero-latency step 0
    const __nv_bfloat16* d_host_x_rand;      // mapped pinned host buffer for steps 1..n-1 (or nullptr)
    const int* d_host_ready;                 // mapped pinned host ready counter (or nullptr)
    int num_layers;
    int max_seq;
};

struct SharedStorage {
    union {
        float s_vec[INTERMEDIATE_SIZE]; // 12 KB
        struct {
            float s_acc0[MK_NUM_WARPS][HEAD_DIM]; // 8 KB
            float s_acc1[MK_NUM_WARPS][HEAD_DIM]; // 8 KB
            float s_m0[MK_NUM_WARPS];
            float s_l0[MK_NUM_WARPS];
            float s_m1[MK_NUM_WARPS];
            float s_l1[MK_NUM_WARPS];
        } attn;
    };
    float s_q0[HEAD_DIM];
    float s_q1[HEAD_DIM];
    float s_cos[HEAD_DIM / 2];
    float s_sin[HEAD_DIM / 2];
    __align__(32) __nv_bfloat16 s_k_cur[HEAD_DIM];
    float s_reduce[64];
    int is_last_kv;
};

__device__ __forceinline__ float fast_exp2(float x) {
    float y;
    asm volatile("ex2.approx.ftz.f32 %0, %1;" : "=f"(y) : "f"(x));
    return y;
}

__device__ __forceinline__ float warp_reduce_sum(float val) {
    #pragma unroll
    for (int offset = WARP_SIZE / 2; offset > 0; offset >>= 1) {
        val += __shfl_xor_sync(0xffffffff, val, offset);
    }
    return val;
}

__device__ __forceinline__ u32x8 ld_v8_evict_last(const void* ptr) {
    u32x8 out;
    asm(
        "ld.global.L1::no_allocate.L2::evict_last.v8.b32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];"
        : "=r"(out.v[0]), "=r"(out.v[1]), "=r"(out.v[2]), "=r"(out.v[3]),
          "=r"(out.v[4]), "=r"(out.v[5]), "=r"(out.v[6]), "=r"(out.v[7])
        : "l"(ptr)
    );
    return out;
}

__device__ __forceinline__ u32x8 ld_v8_evict_first(const void* ptr) {
    u32x8 out;
    asm(
        "ld.global.L1::no_allocate.L2::evict_first.v8.b32 {%0,%1,%2,%3,%4,%5,%6,%7}, [%8];"
        : "=r"(out.v[0]), "=r"(out.v[1]), "=r"(out.v[2]), "=r"(out.v[3]),
          "=r"(out.v[4]), "=r"(out.v[5]), "=r"(out.v[6]), "=r"(out.v[7])
        : "l"(ptr)
    );
    return out;
}

__device__ __forceinline__ int ld_cv_s32(const int* ptr) {
    int r;
    asm volatile("ld.global.cv.s32 %0, [%1];" : "=r"(r) : "l"(ptr));
    return r;
}

__device__ __forceinline__ uint2 ld_cv_u64(const void* ptr) {
    uint2 r;
    asm volatile("ld.global.cv.v2.u32 {%0, %1}, [%2];" : "=r"(r.x), "=r"(r.y) : "l"(ptr));
    return r;
}

template <bool STREAM_KV>
__device__ __forceinline__ u32x8 ld_kv_v8(const void* ptr) {
    if constexpr (STREAM_KV) {
        return ld_v8_evict_first(ptr);
    } else {
        return ld_v8_evict_last(ptr);
    }
}

__device__ __forceinline__ u32x8 ld_weight_v8(const __nv_bfloat16* ptr, const __nv_bfloat16* persist_end) {
    return (ptr < persist_end) ? ld_v8_evict_last(ptr) : ld_v8_evict_first(ptr);
}

__device__ __forceinline__ float2 bf16x2_to_f32x2(uint32_t u) {
    __nv_bfloat162_raw r{static_cast<unsigned short>(u), static_cast<unsigned short>(u >> 16)};
    return __bfloat1622float2(__nv_bfloat162(r));
}

__device__ __forceinline__ float dot16_bf16_f32(const u32x8& w, float4 a0, float4 a1, float4 a2, float4 a3) {
    float2 f0 = bf16x2_to_f32x2(w.v[0]);
    float2 f1 = bf16x2_to_f32x2(w.v[1]);
    float2 f2 = bf16x2_to_f32x2(w.v[2]);
    float2 f3 = bf16x2_to_f32x2(w.v[3]);
    float2 f4 = bf16x2_to_f32x2(w.v[4]);
    float2 f5 = bf16x2_to_f32x2(w.v[5]);
    float2 f6 = bf16x2_to_f32x2(w.v[6]);
    float2 f7 = bf16x2_to_f32x2(w.v[7]);
    return ((f0.x * a0.x + f0.y * a0.y) + (f1.x * a0.z + f1.y * a0.w)) +
           ((f2.x * a1.x + f2.y * a1.y) + (f3.x * a1.z + f3.y * a1.w)) +
           ((f4.x * a2.x + f4.y * a2.y) + (f5.x * a2.z + f5.y * a2.w)) +
           ((f6.x * a3.x + f6.y * a3.y) + (f7.x * a3.z + f7.y * a3.w));
}

__device__ __forceinline__ void apply_head_norm_rope(
    const float* __restrict__ src_head,
    const __nv_bfloat16* __restrict__ norm_w,
    const float* __restrict__ s_cos,
    const float* __restrict__ s_sin,
    int lane_id,
    float post_scale,
    float* __restrict__ dst_f32,
    __nv_bfloat16* __restrict__ dst_bf16,
    __nv_bfloat16* __restrict__ dst_bf16_2 = nullptr
) {
    int i0 = lane_id;
    int i1 = lane_id + 32;
    int i2 = lane_id + 64;
    int i3 = lane_id + 96;

    float v0 = src_head[i0];
    float v1 = src_head[i1];
    float v2 = src_head[i2];
    float v3 = src_head[i3];

    float sq = v0 * v0 + v1 * v1 + v2 * v2 + v3 * v3;
    sq = warp_reduce_sum(sq);
    float rstd = rsqrtf(sq * (1.0f / float(HEAD_DIM)) + RMS_EPS);

    v0 = v0 * rstd * __bfloat162float(__ldg(norm_w + i0));
    v1 = v1 * rstd * __bfloat162float(__ldg(norm_w + i1));
    v2 = v2 * rstd * __bfloat162float(__ldg(norm_w + i2));
    v3 = v3 * rstd * __bfloat162float(__ldg(norm_w + i3));

    float c0 = s_cos[i0], s0 = s_sin[i0];
    float c1 = s_cos[i1], s1 = s_sin[i1];

    float r0 = (v0 * c0 - v2 * s0) * post_scale;
    float r2 = (v0 * s0 + v2 * c0) * post_scale;
    float r1 = (v1 * c1 - v3 * s1) * post_scale;
    float r3 = (v1 * s1 + v3 * c1) * post_scale;

    if (dst_f32 != nullptr) {
        dst_f32[i0] = r0;
        dst_f32[i1] = r1;
        dst_f32[i2] = r2;
        dst_f32[i3] = r3;
    }
    if (dst_bf16 != nullptr) {
        __nv_bfloat16 b0 = __float2bfloat16(r0);
        __nv_bfloat16 b1 = __float2bfloat16(r1);
        __nv_bfloat16 b2 = __float2bfloat16(r2);
        __nv_bfloat16 b3 = __float2bfloat16(r3);
        dst_bf16[i0] = b0;
        dst_bf16[i1] = b1;
        dst_bf16[i2] = b2;
        dst_bf16[i3] = b3;
        if (dst_bf16_2 != nullptr) {
            dst_bf16_2[i0] = b0;
            dst_bf16_2[i1] = b1;
            dst_bf16_2[i2] = b2;
            dst_bf16_2[i3] = b3;
        }
    }
}

__device__ __forceinline__ void attn_accum_step(
    const u32x8& k_v8,
    const u32x8& v_v8,
    const float q0[16],
    const float q1[16],
    uint32_t qw_mask,
    float& m0, float& l0, float acc0[16],
    float& m1, float& l1, float acc1[16]
) {
    float s0 = 0.0f;
    float s1 = 0.0f;
    #pragma unroll
    for (int j = 0; j < 8; ++j) {
        float2 kj = bf16x2_to_f32x2(k_v8.v[j]);
        s0 += q0[2 * j] * kj.x + q0[2 * j + 1] * kj.y;
        s1 += q1[2 * j] * kj.x + q1[2 * j + 1] * kj.y;
    }

    #pragma unroll
    for (int mask = 4; mask >= 1; mask >>= 1) {
        s0 += __shfl_xor_sync(qw_mask, s0, mask);
        s1 += __shfl_xor_sync(qw_mask, s1, mask);
    }

    float old_m0 = m0;
    m0 = fmaxf(m0, s0);
    float alpha0 = (l0 > 0.0f) ? fast_exp2(old_m0 - m0) : 0.0f;
    float p0 = fast_exp2(s0 - m0);
    l0 = l0 * alpha0 + p0;

    float old_m1 = m1;
    m1 = fmaxf(m1, s1);
    float alpha1 = (l1 > 0.0f) ? fast_exp2(old_m1 - m1) : 0.0f;
    float p1 = fast_exp2(s1 - m1);
    l1 = l1 * alpha1 + p1;

    #pragma unroll
    for (int j = 0; j < 8; ++j) {
        float2 vj = bf16x2_to_f32x2(v_v8.v[j]);
        acc0[2 * j]     = acc0[2 * j]     * alpha0 + p0 * vj.x;
        acc0[2 * j + 1] = acc0[2 * j + 1] * alpha0 + p0 * vj.y;
        acc1[2 * j]     = acc1[2 * j]     * alpha1 + p1 * vj.x;
        acc1[2 * j + 1] = acc1[2 * j + 1] * alpha1 + p1 * vj.y;
    }
}

template <bool STREAM_KV>
__device__ __forceinline__ void run_kv_range(
    const __nv_bfloat16* __restrict__ k_head_base,
    const __nv_bfloat16* __restrict__ v_head_base,
    const __nv_bfloat16* __restrict__ s_k_cur_d,
    int p_init,
    int fast_end,
    int pos,
    bool has_cur_pos,
    const float q0[16],
    const float q1[16],
    uint32_t qw_mask,
    float& m0, float& l0, float acc0[16],
    float& m1, float& l1, float acc1[16]
) {
    int p = p_init;
    if (p < fast_end) {
        u32x8 cur_k = ld_kv_v8<STREAM_KV>(k_head_base + (int64_t)p * HEAD_DIM);
        u32x8 cur_v = ld_kv_v8<STREAM_KV>(v_head_base + (int64_t)p * HEAD_DIM);
        for (; p + MK_QWARPS < fast_end; p += MK_QWARPS) {
            int p_nxt = p + MK_QWARPS;
            u32x8 nxt_k = ld_kv_v8<STREAM_KV>(k_head_base + (int64_t)p_nxt * HEAD_DIM);
            u32x8 nxt_v = ld_kv_v8<STREAM_KV>(v_head_base + (int64_t)p_nxt * HEAD_DIM);
            attn_accum_step(cur_k, cur_v, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1);
            cur_k = nxt_k;
            cur_v = nxt_v;
        }
        attn_accum_step(cur_k, cur_v, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1);
        p += MK_QWARPS;
    }
    if (has_cur_pos && p == pos) {
        u32x8 ka = *reinterpret_cast<const u32x8*>(s_k_cur_d);
        u32x8 va = ld_kv_v8<STREAM_KV>(v_head_base + (int64_t)pos * HEAD_DIM);
        attn_accum_step(ka, va, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1);
    }
}

// MODE 0: cache_len <= 2560 (L2-resident KV with .L2::evict_last + 1-head-per-block Split-KV)
// MODE 1: 2560 < cache_len <= 49152 (Streaming KV with .L2::evict_first + 1-head-per-block Split-KV)
// MODE 2: cache_len > 49152 (Streaming KV with .L2::evict_first + flat 1/188 Split-KV)
template <int MODE>
__global__ void __launch_bounds__(MK_BLOCK_SIZE, 1)
megaqwen_multistep_decode_kernel(
    const __grid_constant__ KernelParams params,
    const __nv_bfloat16* __restrict__ x_rand_gmem, // [n_steps, 1024] when not host-streamed
    const __nv_bfloat16* __restrict__ g_hidden_in, // [1024]
    __nv_bfloat16* __restrict__ g_hidden,          // [1024]
    __nv_bfloat16* __restrict__ g_x_stage,         // [1024] GPU L2 staging buffer for host-streamed x_rand
    float* __restrict__ g_residual,                // [1024]
    float* __restrict__ g_post_attn,               // [1024]
    float* __restrict__ g_q,                       // [2048]
    float* __restrict__ g_k,                       // [1024]
    float* __restrict__ g_attn_out,                // [2048]
    float* __restrict__ g_mlp_inter,               // [3072]
    float2* __restrict__ g_part_ml,                // [16 * 32]
    float* __restrict__ g_part_out,                // [16 * 32 * 128]
    unsigned int* __restrict__ g_kv_done,          // [8]
    int start_pos,
    int n_steps
) {
    constexpr bool STREAM_KV = (MODE != 0);
    constexpr bool ONE_HEAD  = (MODE != 2);

    cg::grid_group grid = cg::this_grid();
    __shared__ SharedStorage smem;

    const int tid = threadIdx.x;
    const int block_id = blockIdx.x;
    const int warp_id = tid / WARP_SIZE;
    const int lane_id = tid % WARP_SIZE;

    const int hw_in_warp = lane_id >> 4;
    const int hw_lane = lane_id & 15;
    const uint32_t hw_mask = hw_in_warp ? 0xffff0000u : 0x0000ffffu;
    const int hw_id = warp_id * 2 + hw_in_warp;

    const int swiz_1024 = (block_id * 32) & 1023;
    const int swiz_2048 = (block_id * 32) & 2047;

    const int oh_kv_h = (block_id < 96) ? (block_id / 24) : (4 + (block_id - 96) / 23);
    const int oh_n_b  = (oh_kv_h < 4) ? 24 : 23;
    const int oh_slot = (oh_kv_h < 4) ? (block_id - oh_kv_h * 24) : (block_id - 96 - (oh_kv_h - 4) * 23);

    const bool host_streamed = (params.d_host_ready != nullptr);

    for (int step = 0; step < n_steps; ++step) {
        const int pos = start_pos + step;
        const int cache_len = pos + 1;
        const __nv_bfloat16* step_x_rand = host_streamed
            ? ((step == 0) ? params.first_x_rand : g_x_stage)
            : (x_rand_gmem + (int64_t)step * HIDDEN_SIZE);

        for (int layer = 0; layer < params.num_layers; ++layer) {
            const LayerDesc& ld = params.layers[layer];

            // =================================================================
            // Phase 1: Input Mix (layer 0) + Input RMSNorm + QKV + V-Cache
            // =================================================================
            {
                if (layer == 0 && tid < (HEAD_DIM / 2)) {
                    constexpr double INV_2PI = 0.15915494309189533577;
                    constexpr double TWO_PI  = 6.28318530717958647692;
                    float a = float(pos) * params.inv_freq[tid];
                    float r = (float)((double)a - nearbyint((double)a * INV_2PI) * TWO_PI);
                    smem.s_sin[tid] = __sinf(r);
                    smem.s_cos[tid] = __cosf(r);
                }

                float local_sq = 0.0f;
                int idx = ((tid * 2) + swiz_1024) & 1023;
                #pragma unroll
                for (int c = 0; c < 2; ++c) {
                    int i = idx + c;
                    float v;
                    if (layer == 0) {
                        float xr = __bfloat162float(step_x_rand[i]);
                        float hp = (step == 0)
                            ? __bfloat162float(g_hidden_in[i])
                            : __bfloat162float(__float2bfloat16(g_post_attn[i]));
                        v = __bfloat162float(__float2bfloat16(0.5f * xr + 0.5f * hp));
                    } else {
                        v = __bfloat162float(__float2bfloat16(g_post_attn[i]));
                    }
                    smem.s_vec[i] = v;
                    if (block_id == 0) {
                        g_residual[i] = v;
                    }
                    local_sq += v * v;
                }

                local_sq = warp_reduce_sum(local_sq);
                if (lane_id == 0) {
                    smem.s_reduce[warp_id] = local_sq;
                }
                __syncthreads();

                if (warp_id == 0) {
                    float s = (lane_id < MK_NUM_WARPS) ? smem.s_reduce[lane_id] : 0.0f;
                    s = warp_reduce_sum(s);
                    if (lane_id == 0) {
                        smem.s_reduce[0] = rsqrtf(s * (1.0f / float(HIDDEN_SIZE)) + RMS_EPS);
                    }
                }
                __syncthreads();

                float rstd = smem.s_reduce[0];
                {
                    uint32_t w_pair = *reinterpret_cast<const uint32_t*>(ld.input_ln + idx);
                    float2 wf = bf16x2_to_f32x2(w_pair);
                    smem.s_vec[idx + 0] = smem.s_vec[idx + 0] * rstd * wf.x;
                    smem.s_vec[idx + 1] = smem.s_vec[idx + 1] * rstd * wf.y;
                }
                __syncthreads();

                // Full-warp v8.b32 QKV GEMV across all 16 warps/block (persisted L2)
                {
                    const int k_start = lane_id * 16;
                    float4 a0 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 0);
                    float4 a1 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 4);
                    float4 a2 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 8);
                    float4 a3 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 12);
                    float4 b0 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 512);
                    float4 b1 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 516);
                    float4 b2 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 520);
                    float4 b3 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 524);

                    for (int m = warp_id * MK_NUM_BLOCKS + block_id; m < QKV_SIZE; m += MK_NUM_WARPS * MK_NUM_BLOCKS) {
                        const __nv_bfloat16* w_row = ld.qkv_weight + (int64_t)m * HIDDEN_SIZE;
                        u32x8 w0 = ld_weight_v8(w_row + k_start + 0,   params.persist_end);
                        u32x8 w1 = ld_weight_v8(w_row + k_start + 512, params.persist_end);
                        float sum = dot16_bf16_f32(w0, a0, a1, a2, a3) + dot16_bf16_f32(w1, b0, b1, b2, b3);
                        sum = warp_reduce_sum(sum);
                        if (lane_id == 0) {
                            if (m < Q_SIZE) {
                                g_q[m] = sum;
                            } else if (m < Q_SIZE + KV_SIZE) {
                                g_k[m - Q_SIZE] = sum;
                            } else {
                                int vm = m - (Q_SIZE + KV_SIZE);
                                ld.v_cache[(int64_t)(vm >> 7) * params.max_seq * HEAD_DIM + (int64_t)pos * HEAD_DIM + (vm & 127)] =
                                    __float2bfloat16(sum);
                            }
                        }
                    }
                }
            }

            grid.sync();

            // =================================================================
            // Phase 2: Q/K Norm + RoPE + Compile-Time Specialized Split-KV Attn
            // =================================================================
            {
                const int kv_h_begin = ONE_HEAD ? oh_kv_h : 0;
                const int kv_h_end   = ONE_HEAD ? (oh_kv_h + 1) : NUM_KV_HEADS;
                const int64_t T      = (int64_t)NUM_KV_HEADS * cache_len;
                const int64_t flat_s = ONE_HEAD ? 0 : (((int64_t)block_id * T) / MK_NUM_BLOCKS);
                const int64_t flat_e = ONE_HEAD ? 0 : (((int64_t)(block_id + 1) * T) / MK_NUM_BLOCKS);

                for (int kv_h = kv_h_begin; kv_h < kv_h_end; ++kv_h) {
                    int n_b, slot_idx, pos_start, pos_end;
                    if constexpr (ONE_HEAD) {
                        n_b       = oh_n_b;
                        slot_idx  = oh_slot;
                        pos_start = (int)(((int64_t)slot_idx * cache_len) / n_b);
                        pos_end   = (int)(((int64_t)(slot_idx + 1) * cache_len) / n_b);
                    } else {
                        const int64_t A = (int64_t)kv_h * cache_len;
                        const int64_t B = (int64_t)(kv_h + 1) * cache_len;
                        const int b_lo = (int)(((A + 1) * MK_NUM_BLOCKS + T - 1) / T) - 1;
                        const int b_hi = (int)((B * MK_NUM_BLOCKS - 1) / T);
                        if (block_id < b_lo || block_id > b_hi) {
                            continue;
                        }
                        n_b       = b_hi - b_lo + 1;
                        slot_idx  = block_id - b_lo;
                        pos_start = (int)((flat_s > A ? flat_s : A) - A);
                        pos_end   = (int)((flat_e < B ? flat_e : B) - A);
                    }

                    if (pos_start < pos_end) {
                        const bool has_cur_pos = (pos_start <= pos && pos < pos_end);

                        if (warp_id == 0) {
                            apply_head_norm_rope(
                                g_q + (2 * kv_h) * HEAD_DIM, ld.q_norm, smem.s_cos, smem.s_sin,
                                lane_id, ATTN_SCALE_LOG2E, smem.s_q0, nullptr
                            );
                        } else if (warp_id == 1) {
                            apply_head_norm_rope(
                                g_q + (2 * kv_h + 1) * HEAD_DIM, ld.q_norm, smem.s_cos, smem.s_sin,
                                lane_id, ATTN_SCALE_LOG2E, smem.s_q1, nullptr
                            );
                        } else if (warp_id == 2 && has_cur_pos) {
                            __nv_bfloat16* k_dst = ld.k_cache + (int64_t)kv_h * params.max_seq * HEAD_DIM + (int64_t)pos * HEAD_DIM;
                            apply_head_norm_rope(
                                g_k + kv_h * HEAD_DIM, ld.k_norm, smem.s_cos, smem.s_sin,
                                lane_id, 1.0f, nullptr, smem.s_k_cur, k_dst
                            );
                        }
                        __syncthreads();

                        const int qw_in_warp = lane_id >> 3;
                        const int qw_lane    = lane_id & 7;
                        const uint32_t qw_mask = 0xffu << (qw_in_warp * 8);
                        const int qw_id = warp_id * 4 + qw_in_warp;
                        const int d_base = qw_lane * 16;

                        float q0[16], q1[16];
                        #pragma unroll
                        for (int c = 0; c < 4; ++c) {
                            float4 fq0 = *reinterpret_cast<const float4*>(smem.s_q0 + d_base + c * 4);
                            q0[c * 4 + 0] = fq0.x; q0[c * 4 + 1] = fq0.y; q0[c * 4 + 2] = fq0.z; q0[c * 4 + 3] = fq0.w;
                            float4 fq1 = *reinterpret_cast<const float4*>(smem.s_q1 + d_base + c * 4);
                            q1[c * 4 + 0] = fq1.x; q1[c * 4 + 1] = fq1.y; q1[c * 4 + 2] = fq1.z; q1[c * 4 + 3] = fq1.w;
                        }

                        float m0 = -INFINITY, l0 = 0.0f;
                        float m1 = -INFINITY, l1 = 0.0f;
                        float acc0[16] = {0};
                        float acc1[16] = {0};

                        const __nv_bfloat16* k_head_base = ld.k_cache + (int64_t)kv_h * params.max_seq * HEAD_DIM + d_base;
                        const __nv_bfloat16* v_head_base = ld.v_cache + (int64_t)kv_h * params.max_seq * HEAD_DIM + d_base;
                        const int fast_end = (pos_end < pos) ? pos_end : pos;

                        run_kv_range<STREAM_KV>(k_head_base, v_head_base, smem.s_k_cur + d_base,
                            pos_start + qw_id, fast_end, pos, has_cur_pos, q0, q1, qw_mask, m0, l0, acc0, m1, l1, acc1);

                        #pragma unroll
                        for (int xor_off = 16; xor_off >= 8; xor_off >>= 1) {
                            float m0_o = __shfl_xor_sync(0xffffffff, m0, xor_off);
                            float l0_o = __shfl_xor_sync(0xffffffff, l0, xor_off);
                            float mn0 = fmaxf(m0, m0_o);
                            float sa0 = (l0 > 0.0f) ? fast_exp2(m0 - mn0) : 0.0f;
                            float sb0 = (l0_o > 0.0f) ? fast_exp2(m0_o - mn0) : 0.0f;
                            #pragma unroll
                            for (int j = 0; j < 16; ++j) {
                                float ao = __shfl_xor_sync(0xffffffff, acc0[j], xor_off);
                                acc0[j] = acc0[j] * sa0 + ao * sb0;
                            }
                            m0 = mn0;
                            l0 = l0 * sa0 + l0_o * sb0;

                            float m1_o = __shfl_xor_sync(0xffffffff, m1, xor_off);
                            float l1_o = __shfl_xor_sync(0xffffffff, l1, xor_off);
                            float mn1 = fmaxf(m1, m1_o);
                            float sa1 = (l1 > 0.0f) ? fast_exp2(m1 - mn1) : 0.0f;
                            float sb1 = (l1_o > 0.0f) ? fast_exp2(m1_o - mn1) : 0.0f;
                            #pragma unroll
                            for (int j = 0; j < 16; ++j) {
                                float ao = __shfl_xor_sync(0xffffffff, acc1[j], xor_off);
                                acc1[j] = acc1[j] * sa1 + ao * sb1;
                            }
                            m1 = mn1;
                            l1 = l1 * sa1 + l1_o * sb1;
                        }

                        if (qw_in_warp == 0) {
                            if (qw_lane == 0) {
                                smem.attn.s_m0[warp_id] = m0;
                                smem.attn.s_l0[warp_id] = l0;
                                smem.attn.s_m1[warp_id] = m1;
                                smem.attn.s_l1[warp_id] = l1;
                            }
                            #pragma unroll
                            for (int c = 0; c < 4; ++c) {
                                *reinterpret_cast<float4*>(&smem.attn.s_acc0[warp_id][d_base + c * 4]) =
                                    make_float4(acc0[c * 4 + 0], acc0[c * 4 + 1], acc0[c * 4 + 2], acc0[c * 4 + 3]);
                                *reinterpret_cast<float4*>(&smem.attn.s_acc1[warp_id][d_base + c * 4]) =
                                    make_float4(acc1[c * 4 + 0], acc1[c * 4 + 1], acc1[c * 4 + 2], acc1[c * 4 + 3]);
                            }
                        }
                        __syncthreads();

                        {
                            const int q_local = hw_id >> 4;
                            const int d_grp   = (hw_id & 15) * 8;
                            const float* sm   = (q_local == 0) ? smem.attn.s_m0 : smem.attn.s_m1;
                            const float* sl   = (q_local == 0) ? smem.attn.s_l0 : smem.attn.s_l1;
                            const auto& sacc  = (q_local == 0) ? smem.attn.s_acc0 : smem.attn.s_acc1;

                            float mw = sm[hw_lane];
                            float lw = sl[hw_lane];
                            float m_max = mw;
                            #pragma unroll
                            for (int mask = 8; mask >= 1; mask >>= 1) {
                                m_max = fmaxf(m_max, __shfl_xor_sync(hw_mask, m_max, mask));
                            }
                            float sw = (lw > 0.0f) ? fast_exp2(mw - m_max) : 0.0f;
                            float l_sum = lw * sw;

                            float4 va = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
                            float4 vb = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
                            if (lw > 0.0f) {
                                float4 ra = *reinterpret_cast<const float4*>(&sacc[hw_lane][d_grp + 0]);
                                float4 rb = *reinterpret_cast<const float4*>(&sacc[hw_lane][d_grp + 4]);
                                va = make_float4(ra.x * sw, ra.y * sw, ra.z * sw, ra.w * sw);
                                vb = make_float4(rb.x * sw, rb.y * sw, rb.z * sw, rb.w * sw);
                            }

                            #pragma unroll
                            for (int mask = 8; mask >= 1; mask >>= 1) {
                                l_sum += __shfl_xor_sync(hw_mask, l_sum, mask);
                                va.x += __shfl_xor_sync(hw_mask, va.x, mask);
                                va.y += __shfl_xor_sync(hw_mask, va.y, mask);
                                va.z += __shfl_xor_sync(hw_mask, va.z, mask);
                                va.w += __shfl_xor_sync(hw_mask, va.w, mask);
                                vb.x += __shfl_xor_sync(hw_mask, vb.x, mask);
                                vb.y += __shfl_xor_sync(hw_mask, vb.y, mask);
                                vb.z += __shfl_xor_sync(hw_mask, vb.z, mask);
                                vb.w += __shfl_xor_sync(hw_mask, vb.w, mask);
                            }

                            if (hw_lane == 0) {
                                const int qh = 2 * kv_h + q_local;
                                const int slot = qh * MAX_SPLITS_PER_HEAD + slot_idx;
                                if (d_grp == 0) {
                                    g_part_ml[slot] = make_float2(m_max, l_sum);
                                }
                                *reinterpret_cast<float4*>(g_part_out + slot * HEAD_DIM + d_grp + 0) = va;
                                *reinterpret_cast<float4*>(g_part_out + slot * HEAD_DIM + d_grp + 4) = vb;
                            }
                        }
                    } else {
                        if (warp_id < 2 && lane_id == 0) {
                            g_part_ml[(2 * kv_h + warp_id) * MAX_SPLITS_PER_HEAD + slot_idx] = make_float2(-INFINITY, 0.0f);
                        }
                    }

                    __syncthreads();
                    if (tid == 0) {
                        unsigned int prev;
                        asm volatile("atom.global.acq_rel.gpu.add.u32 %0, [%1], 1;"
                                     : "=r"(prev) : "l"(&g_kv_done[kv_h]) : "memory");
                        if (prev == (unsigned int)(n_b - 1)) {
                            g_kv_done[kv_h] = 0u;
                            smem.is_last_kv = 1;
                        } else {
                            smem.is_last_kv = 0;
                        }
                    }
                    __syncthreads();

                    if (smem.is_last_kv) {
                        const int q_local = warp_id >> 3;
                        const int d_grp   = (warp_id & 7) * 16;
                        const int qh      = 2 * kv_h + q_local;
                        const int s       = lane_id;
                        const int slot    = qh * MAX_SPLITS_PER_HEAD + s;

                        float2 ml = make_float2(-INFINITY, 0.0f);
                        float4 r0 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
                        float4 r1 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
                        float4 r2 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
                        float4 r3 = make_float4(0.0f, 0.0f, 0.0f, 0.0f);
                        if (s < n_b) {
                            ml = g_part_ml[slot];
                            const float* p_src = g_part_out + slot * HEAD_DIM + d_grp;
                            r0 = *reinterpret_cast<const float4*>(p_src + 0);
                            r1 = *reinterpret_cast<const float4*>(p_src + 4);
                            r2 = *reinterpret_cast<const float4*>(p_src + 8);
                            r3 = *reinterpret_cast<const float4*>(p_src + 12);
                        }

                        float m_max = ml.x;
                        #pragma unroll
                        for (int offset = 16; offset >= 1; offset >>= 1) {
                            m_max = fmaxf(m_max, __shfl_xor_sync(0xffffffff, m_max, offset));
                        }
                        float sw = (ml.y > 0.0f) ? fast_exp2(ml.x - m_max) : 0.0f;
                        float l_tot = warp_reduce_sum(ml.y * sw);

                        r0.x = warp_reduce_sum(r0.x * sw); r0.y = warp_reduce_sum(r0.y * sw);
                        r0.z = warp_reduce_sum(r0.z * sw); r0.w = warp_reduce_sum(r0.w * sw);
                        r1.x = warp_reduce_sum(r1.x * sw); r1.y = warp_reduce_sum(r1.y * sw);
                        r1.z = warp_reduce_sum(r1.z * sw); r1.w = warp_reduce_sum(r1.w * sw);
                        r2.x = warp_reduce_sum(r2.x * sw); r2.y = warp_reduce_sum(r2.y * sw);
                        r2.z = warp_reduce_sum(r2.z * sw); r2.w = warp_reduce_sum(r2.w * sw);
                        r3.x = warp_reduce_sum(r3.x * sw); r3.y = warp_reduce_sum(r3.y * sw);
                        r3.z = warp_reduce_sum(r3.z * sw); r3.w = warp_reduce_sum(r3.w * sw);

                        if (lane_id == 0) {
                            float inv_l = 1.0f / l_tot;
                            float* dst = g_attn_out + qh * HEAD_DIM + d_grp;
                            *reinterpret_cast<float4*>(dst + 0)  = make_float4(r0.x * inv_l, r0.y * inv_l, r0.z * inv_l, r0.w * inv_l);
                            *reinterpret_cast<float4*>(dst + 4)  = make_float4(r1.x * inv_l, r1.y * inv_l, r1.z * inv_l, r1.w * inv_l);
                            *reinterpret_cast<float4*>(dst + 8)  = make_float4(r2.x * inv_l, r2.y * inv_l, r2.z * inv_l, r2.w * inv_l);
                            *reinterpret_cast<float4*>(dst + 12) = make_float4(r3.x * inv_l, r3.y * inv_l, r3.z * inv_l, r3.w * inv_l);
                        }
                        if constexpr (!ONE_HEAD) {
                            __syncthreads();
                        }
                    }
                }
            }

            grid.sync();

            // =================================================================
            // Phase 3: Full 16-Warp O-Projection (Persisted L2) + L2 Atomic Add
            // =================================================================
            {
                int idx = ((tid * 4) + swiz_2048) & 2047;
                *reinterpret_cast<float4*>(smem.s_vec + idx) = *reinterpret_cast<const float4*>(g_attn_out + idx);
                __syncthreads();

                const int c0 = warp_id * MK_NUM_BLOCKS + block_id;
                const int c1 = c0 + MK_NUM_WARPS * MK_NUM_BLOCKS;
                const int m0 = c0 >> 2;
                const int ks0 = (c0 & 3) * 512 + lane_id * 16;

                if (c1 < 4096) {
                    const int m1 = c1 >> 2;
                    const int ks1 = (c1 & 3) * 512 + lane_id * 16;
                    u32x8 w0 = ld_weight_v8(ld.o_weight + (int64_t)m0 * Q_SIZE + ks0, params.persist_end);
                    u32x8 w1 = ld_weight_v8(ld.o_weight + (int64_t)m1 * Q_SIZE + ks1, params.persist_end);

                    float s0 = dot16_bf16_f32(w0,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 0),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 4),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 8),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 12));
                    float s1 = dot16_bf16_f32(w1,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks1 + 0),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks1 + 4),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks1 + 8),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks1 + 12));
                    s0 = warp_reduce_sum(s0);
                    s1 = warp_reduce_sum(s1);
                    if (lane_id == 0) {
                        atomicAdd(&g_residual[m0], s0);
                        atomicAdd(&g_residual[m1], s1);
                    }
                } else {
                    u32x8 w0 = ld_weight_v8(ld.o_weight + (int64_t)m0 * Q_SIZE + ks0, params.persist_end);
                    float s0 = dot16_bf16_f32(w0,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 0),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 4),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 8),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 12));
                    s0 = warp_reduce_sum(s0);
                    if (lane_id == 0) {
                        atomicAdd(&g_residual[m0], s0);
                    }
                }
            }

            grid.sync();

            // =================================================================
            // Phase 4: Post-Attention RMSNorm + Overlapped 16-Warp Gate & Up + SwiGLU
            // =================================================================
            {
                int idx = ((tid * 2) + swiz_1024) & 1023;
                float2 v2 = *reinterpret_cast<const float2*>(g_residual + idx);
                smem.s_vec[idx + 0] = v2.x;
                smem.s_vec[idx + 1] = v2.y;
                if (block_id == 0) {
                    *reinterpret_cast<float2*>(g_post_attn + idx) = v2;
                }
                float local_sq = warp_reduce_sum(v2.x * v2.x + v2.y * v2.y);
                if (lane_id == 0) {
                    smem.s_reduce[warp_id] = local_sq;
                }
                __syncthreads();

                if (warp_id == 0) {
                    float s = warp_reduce_sum((lane_id < MK_NUM_WARPS) ? smem.s_reduce[lane_id] : 0.0f);
                    if (lane_id == 0) {
                        smem.s_reduce[0] = rsqrtf(s * (1.0f / float(HIDDEN_SIZE)) + RMS_EPS);
                    }
                }
                __syncthreads();

                float rstd = smem.s_reduce[0];
                {
                    float2 wf = bf16x2_to_f32x2(*reinterpret_cast<const uint32_t*>(ld.post_ln + idx));
                    smem.s_vec[idx + 0] = v2.x * rstd * wf.x;
                    smem.s_vec[idx + 1] = v2.y * rstd * wf.y;
                }
                __syncthreads();

                {
                    const int k_start = lane_id * 16;
                    const int m0 = warp_id * MK_NUM_BLOCKS + block_id;
                    const __nv_bfloat16* g_row = ld.gate_weight + (int64_t)m0 * HIDDEN_SIZE;
                    const __nv_bfloat16* u_row = ld.up_weight   + (int64_t)m0 * HIDDEN_SIZE;

                    if (warp_id < 2 && block_id >= 64 && block_id < 128) {
                        const int m_rem = MK_NUM_WARPS * MK_NUM_BLOCKS + (block_id - 64);
                        const __nv_bfloat16* r_row = ((warp_id == 0) ? ld.gate_weight : ld.up_weight) + (int64_t)m_rem * HIDDEN_SIZE;

                        u32x8 gw0 = ld_weight_v8(g_row + k_start + 0,   params.persist_end);
                        u32x8 gw1 = ld_weight_v8(g_row + k_start + 512, params.persist_end);
                        u32x8 uw0 = ld_weight_v8(u_row + k_start + 0,   params.persist_end);
                        u32x8 uw1 = ld_weight_v8(u_row + k_start + 512, params.persist_end);
                        u32x8 rw0 = ld_weight_v8(r_row + k_start + 0,   params.persist_end);
                        u32x8 rw1 = ld_weight_v8(r_row + k_start + 512, params.persist_end);

                        float4 a0 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 0);
                        float4 a1 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 4);
                        float4 a2 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 8);
                        float4 a3 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 12);
                        float g_sum = dot16_bf16_f32(gw0, a0, a1, a2, a3);
                        float u_sum = dot16_bf16_f32(uw0, a0, a1, a2, a3);
                        float r_sum = dot16_bf16_f32(rw0, a0, a1, a2, a3);

                        float4 b0 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 512);
                        float4 b1 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 516);
                        float4 b2 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 520);
                        float4 b3 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 524);
                        g_sum += dot16_bf16_f32(gw1, b0, b1, b2, b3);
                        u_sum += dot16_bf16_f32(uw1, b0, b1, b2, b3);
                        r_sum += dot16_bf16_f32(rw1, b0, b1, b2, b3);

                        g_sum = warp_reduce_sum(g_sum);
                        u_sum = warp_reduce_sum(u_sum);
                        r_sum = warp_reduce_sum(r_sum);
                        if (lane_id == 0) {
                            g_mlp_inter[m0] = (g_sum / (1.0f + fast_exp2(-g_sum * LOG2E))) * u_sum;
                            smem.s_reduce[warp_id] = r_sum;
                        }
                        asm volatile("bar.sync 1, 64;");
                        if (tid == 0) {
                            float gs = smem.s_reduce[0], us = smem.s_reduce[1];
                            g_mlp_inter[m_rem] = (gs / (1.0f + fast_exp2(-gs * LOG2E))) * us;
                        }
                    } else {
                        float4 a0 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 0);
                        float4 a1 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 4);
                        float4 a2 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 8);
                        float4 a3 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 12);
                        float4 b0 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 512);
                        float4 b1 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 516);
                        float4 b2 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 520);
                        float4 b3 = *reinterpret_cast<const float4*>(smem.s_vec + k_start + 524);

                        u32x8 gw0 = ld_weight_v8(g_row + k_start + 0,   params.persist_end);
                        u32x8 gw1 = ld_weight_v8(g_row + k_start + 512, params.persist_end);
                        u32x8 uw0 = ld_weight_v8(u_row + k_start + 0,   params.persist_end);
                        u32x8 uw1 = ld_weight_v8(u_row + k_start + 512, params.persist_end);

                        float g_sum = warp_reduce_sum(dot16_bf16_f32(gw0, a0, a1, a2, a3) + dot16_bf16_f32(gw1, b0, b1, b2, b3));
                        float u_sum = warp_reduce_sum(dot16_bf16_f32(uw0, a0, a1, a2, a3) + dot16_bf16_f32(uw1, b0, b1, b2, b3));
                        if (lane_id == 0) {
                            g_mlp_inter[m0] = (g_sum / (1.0f + fast_exp2(-g_sum * LOG2E))) * u_sum;
                        }
                    }
                }
            }

            grid.sync();

            // =================================================================
            // Phase 5: Full 16-Warp Down Projection (Persisted L2) + Next-Step Host Fetch
            // =================================================================
            {
                const bool fetch_next_x = (host_streamed && layer == params.num_layers - 1 && (step + 1) < n_steps && block_id == 1);
                int idx0 = ((tid * 4) + swiz_2048) & 2047;
                *reinterpret_cast<float4*>(smem.s_vec + idx0) = *reinterpret_cast<const float4*>(g_mlp_inter + idx0);
                if (tid < 256) {
                    int idx1 = 2048 + (((tid * 4) + swiz_1024) & 1023);
                    *reinterpret_cast<float4*>(smem.s_vec + idx1) = *reinterpret_cast<const float4*>(g_mlp_inter + idx1);
                } else if (fetch_next_x && tid == 256) {
                    while (ld_cv_s32(params.d_host_ready) <= (step + 1)) {}
                }
                __syncthreads();

                if (fetch_next_x && tid >= 256) {
                    int x_off = (tid - 256) * 4;
                    uint2 raw = ld_cv_u64(params.d_host_x_rand + (int64_t)(step + 1) * HIDDEN_SIZE + x_off);
                    *reinterpret_cast<uint2*>(g_x_stage + x_off) = raw;
                }

                const int dc = warp_id * MK_NUM_BLOCKS + block_id;
                const int m0 = dc / 3;
                const int ks0 = (dc - m0 * 3) * 1024 + lane_id * 16;
                const __nv_bfloat16* d_row0 = ld.down_weight + (int64_t)m0 * INTERMEDIATE_SIZE;

                if (block_id >= 60 && warp_id == 0) {
                    const int rem = block_id - 60;
                    const int dc_r = 3008 + (rem >> 1);
                    const int m_rem = dc_r / 3;
                    const int ks_rem = (dc_r - m_rem * 3) * 1024 + (rem & 1) * 512 + lane_id * 16;

                    u32x8 d0 = ld_weight_v8(d_row0 + ks0 + 0,   params.persist_end);
                    u32x8 d1 = ld_weight_v8(d_row0 + ks0 + 512, params.persist_end);
                    u32x8 dr = ld_weight_v8(ld.down_weight + (int64_t)m_rem * INTERMEDIATE_SIZE + ks_rem, params.persist_end);

                    float sum = dot16_bf16_f32(d0,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 0),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 4),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 8),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 12));
                    sum += dot16_bf16_f32(d1,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 512),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 516),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 520),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 524));
                    float sr = dot16_bf16_f32(dr,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks_rem + 0),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks_rem + 4),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks_rem + 8),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks_rem + 12));
                    sum = warp_reduce_sum(sum);
                    sr  = warp_reduce_sum(sr);
                    if (lane_id == 0) {
                        atomicAdd(&g_post_attn[m0], sum);
                        atomicAdd(&g_post_attn[m_rem], sr);
                    }
                } else {
                    u32x8 d0 = ld_weight_v8(d_row0 + ks0 + 0,   params.persist_end);
                    u32x8 d1 = ld_weight_v8(d_row0 + ks0 + 512, params.persist_end);

                    float sum = dot16_bf16_f32(d0,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 0),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 4),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 8),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 12));
                    sum += dot16_bf16_f32(d1,
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 512),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 516),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 520),
                        *reinterpret_cast<const float4*>(smem.s_vec + ks0 + 524));
                    sum = warp_reduce_sum(sum);
                    if (lane_id == 0) {
                        atomicAdd(&g_post_attn[m0], sum);
                    }
                }
            }

            grid.sync();
        }
    }
    if (n_steps > 0 && block_id == 0) {
        int idx = tid * 2;
        g_hidden[idx + 0] = __float2bfloat16(g_post_attn[idx + 0]);
        g_hidden[idx + 1] = __float2bfloat16(g_post_attn[idx + 1]);
    }
}

static bool g_kernel_configured = false;
static size_t g_persisting_l2_max = 0;
static const void* g_last_persisted_ptr = nullptr;
static size_t g_last_persisted_bytes = 0;

static __nv_bfloat16* g_h_x_pinned = nullptr;
static __nv_bfloat16* g_d_x_pinned = nullptr;
static int* g_h_ready_pinned = nullptr;
static int* g_d_ready_pinned = nullptr;
static cudaEvent_t g_slot_events[NUM_RING_SLOTS];
static bool g_slot_used[NUM_RING_SLOTS] = {false};
static uint64_t g_ring_idx = 0;
static at::Generator g_cpu_gen;

static void ensure_kernel_configured() {
    if (!g_kernel_configured) {
        cudaFuncSetAttribute(megaqwen_multistep_decode_kernel<0>, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1);
        cudaFuncSetAttribute(megaqwen_multistep_decode_kernel<1>, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1);
        cudaFuncSetAttribute(megaqwen_multistep_decode_kernel<2>, cudaFuncAttributePreferredSharedMemoryCarveout, cudaSharedmemCarveoutMaxL1);
        cudaDeviceProp prop;
        int dev = 0;
        cudaGetDevice(&dev);
        cudaGetDeviceProperties(&prop, dev);
        g_persisting_l2_max = prop.persistingL2CacheMaxSize;
        if (g_persisting_l2_max > 0) {
            cudaDeviceSetLimit(cudaLimitPersistingL2CacheSize, g_persisting_l2_max);
        }
        void* raw_host = nullptr;
        size_t x_total_bytes = (size_t)NUM_RING_SLOTS * MAX_STREAMED_STEPS * HIDDEN_SIZE * sizeof(__nv_bfloat16);
        size_t bytes = x_total_bytes + NUM_RING_SLOTS * 128;
        cudaError_t herr = cudaHostAlloc(&raw_host, bytes, cudaHostAllocMapped);
        TORCH_CHECK(herr == cudaSuccess, "cudaHostAlloc failed: ", cudaGetErrorString(herr));
        g_h_x_pinned = reinterpret_cast<__nv_bfloat16*>(raw_host);
        g_h_ready_pinned = reinterpret_cast<int*>(reinterpret_cast<char*>(raw_host) + x_total_bytes);
        cudaHostGetDevicePointer((void**)&g_d_x_pinned, (void*)g_h_x_pinned, 0);
        cudaHostGetDevicePointer((void**)&g_d_ready_pinned, (void*)g_h_ready_pinned, 0);
        for (int i = 0; i < NUM_RING_SLOTS; ++i) {
            cudaEventCreateWithFlags(&g_slot_events[i], cudaEventDisableTiming);
            g_slot_used[i] = false;
        }
        g_cpu_gen = at::detail::createCPUGenerator();
        g_kernel_configured = true;
    }
}

static void populate_common_params(
    KernelParams& params,
    cudaStream_t stream,
    const torch::Tensor& packed_weights,
    const std::vector<int64_t>& layer_offsets,
    const std::vector<torch::Tensor>& k_caches,
    const std::vector<torch::Tensor>& v_caches,
    const torch::Tensor& inv_freq,
    int max_seq
) {
    const __nv_bfloat16* w_base = reinterpret_cast<const __nv_bfloat16*>(packed_weights.data_ptr());
    size_t w_bytes = packed_weights.numel() * sizeof(__nv_bfloat16);
    size_t win_bytes = std::min(w_bytes, g_persisting_l2_max);

    if (win_bytes > 0 && (w_base != g_last_persisted_ptr || win_bytes != g_last_persisted_bytes)) {
        cudaStreamAttrValue attr;
        memset(&attr, 0, sizeof(attr));
        attr.accessPolicyWindow.base_ptr = const_cast<__nv_bfloat16*>(w_base);
        attr.accessPolicyWindow.num_bytes = win_bytes;
        attr.accessPolicyWindow.hitRatio = 1.0f;
        attr.accessPolicyWindow.hitProp = cudaAccessPropertyPersisting;
        attr.accessPolicyWindow.missProp = cudaAccessPropertyStreaming;
        cudaStreamSetAttribute(stream, cudaStreamAttributeAccessPolicyWindow, &attr);
        g_last_persisted_ptr = w_base;
        g_last_persisted_bytes = win_bytes;
    }

    int num_layers = (int)k_caches.size();
    TORCH_CHECK(num_layers <= MAX_LAYERS, "num_layers exceeds MAX_LAYERS");
    TORCH_CHECK((int)layer_offsets.size() == num_layers * 9, "expected 9 offsets per layer");

    params.num_layers = num_layers;
    params.max_seq = max_seq;
    params.persist_end = w_base + (win_bytes / sizeof(__nv_bfloat16));
    params.d_host_x_rand = nullptr;
    params.d_host_ready  = nullptr;

    const float* inv_ptr = inv_freq.data_ptr<float>();
    for (int i = 0; i < HEAD_DIM / 2; ++i) {
        params.inv_freq[i] = inv_ptr[i];
    }

    for (int l = 0; l < num_layers; ++l) {
        const int64_t* o = &layer_offsets[l * 9];
        params.layers[l].input_ln    = w_base + o[0];
        params.layers[l].q_norm      = w_base + o[1];
        params.layers[l].k_norm      = w_base + o[2];
        params.layers[l].post_ln     = w_base + o[3];
        params.layers[l].qkv_weight  = w_base + o[4];
        params.layers[l].o_weight    = w_base + o[5];
        params.layers[l].gate_weight = w_base + o[6];
        params.layers[l].up_weight   = w_base + o[7];
        params.layers[l].down_weight = w_base + o[8];
        params.layers[l].k_cache     = reinterpret_cast<__nv_bfloat16*>(k_caches[l].data_ptr());
        params.layers[l].v_cache     = reinterpret_cast<__nv_bfloat16*>(v_caches[l].data_ptr());
    }
}

void run_multistep_decode(
    torch::Tensor packed_weights,
    std::vector<int64_t> layer_offsets,
    std::vector<torch::Tensor> k_caches,
    std::vector<torch::Tensor> v_caches,
    torch::Tensor inv_freq,
    torch::Tensor x_rand_all,
    torch::Tensor hidden_in,
    torch::Tensor hidden_out,
    torch::Tensor workspace_f32,
    int start_pos,
    int n_steps,
    int max_seq
) {
    ensure_kernel_configured();
    cudaStream_t stream = at::cuda::getCurrentCUDAStream();

    KernelParams params;
    populate_common_params(params, stream, packed_weights, layer_offsets, k_caches, v_caches, inv_freq, max_seq);

    float* ws = workspace_f32.data_ptr<float>();
    float* g_residual        = ws + 0;
    float* g_post_attn       = ws + 1024;
    float* g_q               = ws + 2048;
    float* g_k               = ws + 4096;
    float* g_attn_out        = ws + 5120;
    float* g_mlp_inter       = ws + 7168;
    unsigned int* g_kv_done  = reinterpret_cast<unsigned int*>(ws + 10240);
    __nv_bfloat16* g_x_stage = reinterpret_cast<__nv_bfloat16*>(ws + 10272);
    float2* g_part_ml        = reinterpret_cast<float2*>(ws + 11008);
    float* g_part_out        = ws + 12160;

    const __nv_bfloat16* x_all_ptr = reinterpret_cast<const __nv_bfloat16*>(x_rand_all.data_ptr());
    const __nv_bfloat16* hidden_in_ptr = reinterpret_cast<const __nv_bfloat16*>(hidden_in.data_ptr());
    __nv_bfloat16* hidden_out_ptr = reinterpret_cast<__nv_bfloat16*>(hidden_out.data_ptr());

    constexpr int CHUNK_STEPS = 4096;
    int rem_steps = n_steps;
    int cur_start = start_pos;
    int step_off = 0;

    while (rem_steps > 0) {
        int mode;
        int limit_steps;
        if (cur_start < 2560) {
            mode = 0;
            limit_steps = std::min(rem_steps, 2560 - cur_start);
        } else if (cur_start < 49152) {
            mode = 1;
            limit_steps = std::min(rem_steps, 49152 - cur_start);
        } else {
            mode = 2;
            limit_steps = rem_steps;
        }
        int cur_steps = std::min(CHUNK_STEPS, limit_steps);
        const __nv_bfloat16* cur_x_ptr = x_all_ptr + (int64_t)step_off * HIDDEN_SIZE;
        const __nv_bfloat16* cur_h_in = (step_off == 0) ? hidden_in_ptr : hidden_out_ptr;

        void* kernel_args[] = {
            (void*)&params,
            (void*)&cur_x_ptr,
            (void*)&cur_h_in,
            (void*)&hidden_out_ptr,
            (void*)&g_x_stage,
            (void*)&g_residual,
            (void*)&g_post_attn,
            (void*)&g_q,
            (void*)&g_k,
            (void*)&g_attn_out,
            (void*)&g_mlp_inter,
            (void*)&g_part_ml,
            (void*)&g_part_out,
            (void*)&g_kv_done,
            (void*)&cur_start,
            (void*)&cur_steps
        };

        void* fn = (mode == 0)
            ? (void*)megaqwen_multistep_decode_kernel<0>
            : ((mode == 1) ? (void*)megaqwen_multistep_decode_kernel<1>
                           : (void*)megaqwen_multistep_decode_kernel<2>);

        cudaError_t err = cudaLaunchCooperativeKernel(
            fn, dim3(MK_NUM_BLOCKS), dim3(MK_BLOCK_SIZE), kernel_args, 0, stream
        );
        TORCH_CHECK(err == cudaSuccess, "cudaLaunchCooperativeKernel failed: ", cudaGetErrorString(err));

        cur_start += cur_steps;
        step_off += cur_steps;
        rem_steps -= cur_steps;
    }
}

void run_decode_steps_seeded(
    torch::Tensor packed_weights,
    std::vector<int64_t> layer_offsets,
    std::vector<torch::Tensor> k_caches,
    std::vector<torch::Tensor> v_caches,
    torch::Tensor inv_freq,
    torch::Tensor hidden_in,
    torch::Tensor hidden_out,
    torch::Tensor workspace_f32,
    int start_pos,
    int n_steps,
    int max_seq,
    int64_t rng_seed
) {
    ensure_kernel_configured();
    cudaStream_t stream = at::cuda::getCurrentCUDAStream();

    KernelParams params;
    populate_common_params(params, stream, packed_weights, layer_offsets, k_caches, v_caches, inv_freq, max_seq);

    float* ws = workspace_f32.data_ptr<float>();
    float* g_residual        = ws + 0;
    float* g_post_attn       = ws + 1024;
    float* g_q               = ws + 2048;
    float* g_k               = ws + 4096;
    float* g_attn_out        = ws + 5120;
    float* g_mlp_inter       = ws + 7168;
    unsigned int* g_kv_done  = reinterpret_cast<unsigned int*>(ws + 10240);
    __nv_bfloat16* g_x_stage = reinterpret_cast<__nv_bfloat16*>(ws + 10272);
    float2* g_part_ml        = reinterpret_cast<float2*>(ws + 11008);
    float* g_part_out        = ws + 12160;

    int slot = (int)(g_ring_idx++ & (NUM_RING_SLOTS - 1));
    if (g_slot_used[slot]) {
        cudaEventSynchronize(g_slot_events[slot]);
    }
    __nv_bfloat16* slot_h_x = g_h_x_pinned + (size_t)slot * MAX_STREAMED_STEPS * HIDDEN_SIZE;
    const __nv_bfloat16* slot_d_x = g_d_x_pinned + (size_t)slot * MAX_STREAMED_STEPS * HIDDEN_SIZE;
    int* slot_h_ready = g_h_ready_pinned + slot * 32;
    const int* slot_d_ready = g_d_ready_pinned + slot * 32;

    g_cpu_gen.set_current_seed((uint64_t)rng_seed);
    __atomic_store_n(slot_h_ready, 0, __ATOMIC_RELAXED);

    {
        auto t0 = torch::from_blob(
            params.first_x_rand, {1, HIDDEN_SIZE},
            torch::TensorOptions().dtype(torch::kBFloat16).device(torch::kCPU)
        );
        at::randn_out(t0, {1, HIDDEN_SIZE}, g_cpu_gen);
    }
    __atomic_store_n(slot_h_ready, 1, __ATOMIC_RELEASE);

    params.d_host_x_rand = slot_d_x;
    params.d_host_ready  = slot_d_ready;

    const __nv_bfloat16* null_x = nullptr;
    const __nv_bfloat16* hidden_in_ptr = reinterpret_cast<const __nv_bfloat16*>(hidden_in.data_ptr());
    __nv_bfloat16* hidden_out_ptr = reinterpret_cast<__nv_bfloat16*>(hidden_out.data_ptr());

    void* kernel_args[] = {
        (void*)&params,
        (void*)&null_x,
        (void*)&hidden_in_ptr,
        (void*)&hidden_out_ptr,
        (void*)&g_x_stage,
        (void*)&g_residual,
        (void*)&g_post_attn,
        (void*)&g_q,
        (void*)&g_k,
        (void*)&g_attn_out,
        (void*)&g_mlp_inter,
        (void*)&g_part_ml,
        (void*)&g_part_out,
        (void*)&g_kv_done,
        (void*)&start_pos,
        (void*)&n_steps
    };

    int end_pos = start_pos + n_steps - 1;
    void* fn = (end_pos < 2560)
        ? (void*)megaqwen_multistep_decode_kernel<0>
        : ((end_pos < 49152) ? (void*)megaqwen_multistep_decode_kernel<1>
                             : (void*)megaqwen_multistep_decode_kernel<2>);

    cudaError_t err = cudaLaunchCooperativeKernel(
        fn, dim3(MK_NUM_BLOCKS), dim3(MK_BLOCK_SIZE), kernel_args, 0, stream
    );
    TORCH_CHECK(err == cudaSuccess, "cudaLaunchCooperativeKernel failed: ", cudaGetErrorString(err));
    cudaEventRecord(g_slot_events[slot], stream);
    g_slot_used[slot] = true;

    if (n_steps > 1) {
        auto trem = torch::from_blob(
            slot_h_x + HIDDEN_SIZE, {n_steps - 1, HIDDEN_SIZE},
            torch::TensorOptions().dtype(torch::kBFloat16).device(torch::kCPU)
        );
        at::randn_out(trem, {n_steps - 1, HIDDEN_SIZE}, g_cpu_gen);
        __sync_synchronize();
        __atomic_store_n(slot_h_ready, n_steps, __ATOMIC_RELEASE);
    }
}

PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
    m.def("run_multistep_decode", &run_multistep_decode, "MegaQwen SM120 multistep decode v6");
    m.def("run_decode_steps_seeded", &run_decode_steps_seeded, "MegaQwen SM120 streamed-RNG decode v6");
}
"""

_EXT = load_inline(
    name="megaqwen_sm120_ext_v6",
    cpp_sources="",
    cuda_sources=_CUDA_SRC,
    extra_cuda_cflags=["-O3", "-arch=sm_120", "--expt-relaxed-constexpr"],
    verbose=False,
)


class Block(nn.Module):
    def __init__(self):
        super().__init__()
        H, I, D = HIDDEN, INTERMEDIATE, HEAD_DIM
        self.input_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16))
        self.q_proj = nn.Parameter(torch.empty(NUM_Q * D, H, dtype=torch.bfloat16))
        self.k_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16))
        self.v_proj = nn.Parameter(torch.empty(NUM_KV * D, H, dtype=torch.bfloat16))
        self.q_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16))
        self.k_norm = nn.Parameter(torch.ones(D, dtype=torch.bfloat16))
        self.o_proj = nn.Parameter(torch.empty(H, NUM_Q * D, dtype=torch.bfloat16))
        self.post_ln = nn.Parameter(torch.ones(H, dtype=torch.bfloat16))
        self.gate_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16))
        self.up_proj = nn.Parameter(torch.empty(I, H, dtype=torch.bfloat16))
        self.down_proj = nn.Parameter(torch.empty(H, I, dtype=torch.bfloat16))
        for p in self.parameters():
            if p is self.input_ln or p is self.post_ln or p is self.q_norm or p is self.k_norm:
                continue
            nn.init.normal_(p, std=0.02)


class Model(nn.Module):
    def __init__(self, num_layers: int = NUM_LAYERS, max_seq: int = 131072):
        super().__init__()
        self.num_layers = num_layers
        self.max_seq = max_seq
        self.blocks = nn.ModuleList([Block() for _ in range(num_layers)])
        self._param_list = list(self.parameters())
        self._packed_sig = None
        self._packed_weights = None
        self._layer_offsets: list[int] = []
        self._inv_freq_cpu = 1.0 / (
            10000.0 ** (torch.arange(0, HEAD_DIM // 2, dtype=torch.float32) / (HEAD_DIM // 2))
        )
        self._workspace_f32 = None
        self._gpu_rand_buf = None

    def _ensure_packed(self, device: torch.device):
        if self._packed_weights is not None and self._packed_weights.device == device:
            sig = tuple((p.data_ptr(), p._version) for p in self._param_list)
            if self._packed_sig == sig:
                return

        # Priority order for 80 MiB persisting L2 window (ord2):
        # 1. All norms (20 KB)
        # 2. All QKV projections (32 MiB)
        # 3. All O projections (16 MiB)
        # 4. All Down projections (24 MiB) -> 72.02 MiB total (100% persisted)
        # 5. Gate & Up projections (~7.98 MiB persisted + ~40.02 MiB streamed with 4 v8 loads/warp)
        layer_elems = 2560 + 4096 * 1024 + 1024 * 2048 + 6144 * 1024 + 1024 * 3072
        total_elems = self.num_layers * layer_elems
        if self._packed_weights is None or self._packed_weights.device != device or self._packed_weights.numel() != total_elems:
            self._packed_weights = torch.empty(total_elems, dtype=torch.bfloat16, device=device)

        buf = self._packed_weights
        pos = [{} for _ in range(self.num_layers)]
        cur = 0

        for l, b in enumerate(self.blocks):
            pos[l]["input_ln"] = cur
            buf[cur : cur + 1024].copy_(b.input_ln)
            pos[l]["q_norm"] = cur + 1024
            buf[cur + 1024 : cur + 1152].copy_(b.q_norm)
            pos[l]["k_norm"] = cur + 1152
            buf[cur + 1152 : cur + 1280].copy_(b.k_norm)
            pos[l]["post_ln"] = cur + 1280
            buf[cur + 1280 : cur + 2304].copy_(b.post_ln)
            buf[cur + 2304 : cur + 2560].zero_()
            cur += 2560

        for l, b in enumerate(self.blocks):
            pos[l]["qkv"] = cur
            buf[cur : cur + 2048 * 1024].copy_(b.q_proj.reshape(-1))
            cur += 2048 * 1024
            buf[cur : cur + 1024 * 1024].copy_(b.k_proj.reshape(-1))
            cur += 1024 * 1024
            buf[cur : cur + 1024 * 1024].copy_(b.v_proj.reshape(-1))
            cur += 1024 * 1024

        for l, b in enumerate(self.blocks):
            pos[l]["o"] = cur
            buf[cur : cur + 1024 * 2048].copy_(b.o_proj.reshape(-1))
            cur += 1024 * 2048

        for l, b in enumerate(self.blocks):
            pos[l]["down"] = cur
            buf[cur : cur + 1024 * 3072].copy_(b.down_proj.reshape(-1))
            cur += 1024 * 3072

        for l, b in enumerate(self.blocks):
            pos[l]["gate"] = cur
            buf[cur : cur + 3072 * 1024].copy_(b.gate_proj.reshape(-1))
            cur += 3072 * 1024
            pos[l]["up"] = cur
            buf[cur : cur + 3072 * 1024].copy_(b.up_proj.reshape(-1))
            cur += 3072 * 1024

        offs: list[int] = []
        for l in range(self.num_layers):
            p = pos[l]
            offs.extend([
                p["input_ln"], p["q_norm"], p["k_norm"], p["post_ln"],
                p["qkv"], p["o"], p["gate"], p["up"], p["down"],
            ])
        self._layer_offsets = offs

        if self._workspace_f32 is None or self._workspace_f32.device != device or self._workspace_f32.numel() < 90000:
            self._workspace_f32 = torch.zeros(90000, dtype=torch.float32, device=device)
        else:
            self._workspace_f32[10240:10272].zero_()
        self._packed_sig = tuple((p.data_ptr(), p._version) for p in self._param_list)

    def _get_gpu_rand_buf(self, n_steps: int, device: torch.device) -> torch.Tensor:
        cap = max(128, 1 << (max(1, n_steps) - 1).bit_length())
        if self._gpu_rand_buf is None or self._gpu_rand_buf.device != device or self._gpu_rand_buf.shape[0] < cap:
            self._gpu_rand_buf = torch.empty((cap, HIDDEN), dtype=torch.bfloat16, device=device)
        return self._gpu_rand_buf[:n_steps]

    def forward(
        self,
        x: torch.Tensor,
        k_caches: list[torch.Tensor],
        v_caches: list[torch.Tensor],
        position: int,
    ):
        device = x.device
        self._ensure_packed(device)
        x_c = x if x.is_contiguous() else x.contiguous()
        h_out = torch.empty_like(x_c)
        x_in = x_c.view(1, HIDDEN)
        max_seq = k_caches[0].shape[1]
        _EXT.run_multistep_decode(
            self._packed_weights,
            self._layer_offsets,
            k_caches,
            v_caches,
            self._inv_freq_cpu,
            x_in,
            x_c,
            h_out,
            self._workspace_f32,
            int(position),
            1,
            int(max_seq),
        )
        return h_out, k_caches, v_caches


_PINNED_RAND_CACHE: dict[int, torch.Tensor] = {}


def _get_pinned_rand_buf(n_steps: int) -> torch.Tensor:
    cap = max(128, 1 << (max(1, n_steps) - 1).bit_length())
    buf = _PINNED_RAND_CACHE.get(cap)
    if buf is None:
        buf = torch.empty((cap, HIDDEN), dtype=torch.bfloat16, pin_memory=True)
        _PINNED_RAND_CACHE[cap] = buf
    return buf[:n_steps]


def empty_caches(num_layers: int, max_seq: int, device, dtype=torch.bfloat16):
    k_slab = torch.empty((num_layers, NUM_KV, max_seq, HEAD_DIM), device=device, dtype=dtype)
    v_slab = torch.empty((num_layers, NUM_KV, max_seq, HEAD_DIM), device=device, dtype=dtype)
    k = [k_slab[i] for i in range(num_layers)]
    v = [v_slab[i] for i in range(num_layers)]
    return k, v


def _seeded_hidden(seed: int, device) -> torch.Tensor:
    g = torch.Generator(device="cpu")
    g.manual_seed(seed)
    return torch.randn(HIDDEN, generator=g, dtype=torch.bfloat16).to(device)


@torch.no_grad()
def prefill(
    model: Model,
    ctx_len: int,
    seed: int,
    device: torch.device | None = None,
):
    device = device or next(model.parameters()).device
    model = model.to(device).eval()
    assert ctx_len <= model.max_seq
    model._ensure_packed(device)

    h = _seeded_hidden(seed, device).contiguous()
    k_caches, v_caches = empty_caches(model.num_layers, model.max_seq, device)

    if ctx_len > 0:
        g = torch.Generator(device="cpu")
        g.manual_seed(seed + 1)
        x_cpu = _get_pinned_rand_buf(ctx_len)
        torch.randn((ctx_len, HIDDEN), generator=g, out=x_cpu)
        x_gpu = model._get_gpu_rand_buf(ctx_len, device)
        x_gpu.copy_(x_cpu, non_blocking=True)

        _EXT.run_multistep_decode(
            model._packed_weights,
            model._layer_offsets,
            k_caches,
            v_caches,
            model._inv_freq_cpu,
            x_gpu,
            h,
            h,
            model._workspace_f32,
            0,
            int(ctx_len),
            int(model.max_seq),
        )
    return h, k_caches, v_caches


@torch.no_grad()
def decode_steps(
    model: Model,
    hidden: torch.Tensor,
    k_caches: list[torch.Tensor],
    v_caches: list[torch.Tensor],
    start_pos: int,
    n_steps: int,
    seed: int,
):
    device = hidden.device
    model._ensure_packed(device)
    h_in = hidden if hidden.is_contiguous() else hidden.contiguous()
    if n_steps <= 0:
        return h_in.clone(), k_caches, v_caches

    h_out = torch.empty_like(h_in)
    max_seq = int(k_caches[0].shape[1])
    s_pos = int(start_pos)
    steps = int(n_steps)
    end_pos = s_pos + steps - 1

    if steps <= 256 and (
        end_pos < 2560 or (s_pos >= 2560 and end_pos < 49152) or s_pos >= 49152
    ):
        _EXT.run_decode_steps_seeded(
            model._packed_weights,
            model._layer_offsets,
            k_caches,
            v_caches,
            model._inv_freq_cpu,
            h_in,
            h_out,
            model._workspace_f32,
            s_pos,
            steps,
            max_seq,
            int(seed) + 2,
        )
    else:
        g = torch.Generator(device="cpu")
        g.manual_seed(int(seed) + 2)
        x_cpu = _get_pinned_rand_buf(steps)
        torch.randn((steps, HIDDEN), generator=g, out=x_cpu)
        x_gpu = model._get_gpu_rand_buf(steps, device)
        x_gpu.copy_(x_cpu, non_blocking=True)
        _EXT.run_multistep_decode(
            model._packed_weights,
            model._layer_offsets,
            k_caches,
            v_caches,
            model._inv_freq_cpu,
            x_gpu,
            h_in,
            h_out,
            model._workspace_f32,
            s_pos,
            steps,
            max_seq,
        )

    return h_out, k_caches, v_caches


def run(
    ctx_len: int,
    n_decode: int,
    seed: int,
    model: Model | None = None,
    max_seq: int | None = None,
) -> dict:
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    max_seq = max(ctx_len + n_decode, 512) if max_seq is None else max_seq
    if model is None:
        model = Model(NUM_LAYERS, max_seq)
    else:
        if getattr(model, "max_seq", 0) < ctx_len + n_decode:
            raise ValueError(
                f"model.max_seq={getattr(model, 'max_seq', None)} too small for "
                f"ctx_len={ctx_len}+n_decode={n_decode}"
            )
    model = model.to(device).eval()
    h, k_caches, v_caches = prefill(model, ctx_len, seed, device=device)
    h, k_caches, v_caches = decode_steps(
        model, h, k_caches, v_caches, start_pos=ctx_len, n_steps=n_decode, seed=seed
    )
    return {
        "last_hidden": h.detach(),
        "ctx_len": ctx_len,
        "decode_steps": n_decode,
    }


def get_init_inputs():
    return [NUM_LAYERS, 131072]


def get_inputs():
    return []

20261110_113746_agy_gemini-4-carbon-high_03_megaqwen_decode