KernelBench cuda · RTX PRO 6000

GLM-5.3 Fused MoE Gemini 4 Neon (XHigh)

cleandid not score

manually audited: clean

Gemini 4 Neon (XHigh) wrote the GLM-5.2 MoE layer in hand-written SM120 CUDA C++ with inline PTX (cp.async.cg, ldmatrix.x4, bf16 mma.sync m16n8k16, red.relaxed.gpu.global.add.v4.f32) and a 2-kernel streaming GEMV path for small batches (T <= 4). Notably, 0.1010 is its uniterated V1 first draft: after passing an initial GPU smoke test, a sibling cell's deadlock held the shared cuda GPU lock for 2h42m, so this session exited cleanly at 5:29 pm without ever timing or optimizing on GPU.

harnessagyagent session2h 25mtotal wall3h 7mcheck5mbenchmark4moutput tokens—gpu-lock wait32mgpu-lock held13mregimecompute

Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth

shape 09.805 ms37.9%1.33 TB/s · 74% of 1.8 TB/s HBM · also 189 TFLOPS (38% of compute)
shape 110.252 ms36.5%1.27 TB/s · 70% of 1.8 TB/s HBM · also 182 TFLOPS (36% of compute)
shape 20.302 ms0.3%42.82 TB/s · 100% of 1.8 TB/s HBM · also 2 TFLOPS (0% of compute)
shape 317.748 ms41.8%209 TFLOPS · 42% of 500 TF bf16 peak · also 0.74 TB/s (41% of HBM)
shape 48.172 ms5.7%1.58 TB/s · 88% of 1.8 TB/s HBM · also 28 TFLOPS (6% of compute)
shape 58.398 ms10.8%1.54 TB/s · 86% of 1.8 TB/s HBM · also 54 TFLOPS (11% of compute)

compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)

geomean(37.9% · 36.5% · 0.3% · 41.8% · 5.7% · 10.8%) = 10.1%

Kernel source (redacted)
"""Fused GLM-5.2 MoE layer in CUDA C++/PTX for RTX PRO 6000 (SM120 Blackwell).

Architecture:
  - Small-batch decode path (T <= 4):
      2-kernel fused GEMV covering all active routed + shared experts in one shot:
        1) moe_decode_gemv1_silu_mul: vectorized 128-bit streaming gate+up+SiLU*mul -> h_buf (fp32)
        2) moe_decode_gemv2_reduce:   vectorized 128-bit down-proj + weighted sum -> out (bf16)
  - Grouped prefill/batch path (T > 4):
      0) Zero-sync GPU routing histogram + prefix sum + scatter + M-tile lookup table
      1) Fused Grouped GEMM1 (SM120 PTX mma.sync.aligned.m16n8k16 + cp.async.cg + 128B XOR swizzle
         + ldmatrix.x4): gathers x[token_id] on-the-fly from L2, computes gate & up simultaneously,
         applies SiLU(gate)*up in FP32 registers, writes packed h (bf16) via SMEM coalescing.
      2) Fused Grouped GEMM2: computes h @ w2^T, scales by routing weight in FP32, and scatters
         directly into out_fp32 via 128-bit vectorized red.relaxed.gpu.global.add.v4.f32.
      3) Fused Shared Expert GEMM1 + GEMM2: computes shared expert(s), fusing the final addition
         with out_fp32 and FP32->BF16 conversion into out (bf16).
"""
from __future__ import annotations

import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

_CUDA_SRC = r"""
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
#include <torch/extension.h>
#include <c10/cuda/CUDAStream.h>
#include <c10/cuda/CUDAException.h>

// ============================================================================
// PTX & Low-Level Device Primitives (SM120 Blackwell)
// ============================================================================

__device__ __forceinline__ uint32_t smem_u32(const void* ptr) {
    return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}

__device__ __forceinline__ void cp_async_cg_16(uint32_t dst_smem, const void* src_gmem) {
    asm volatile(
        "cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\n"
        :: "r"(dst_smem), "l"(src_gmem)
    );
}

__device__ __forceinline__ void cp_async_cg_16_zfill(uint32_t dst_smem, const void* src_gmem, int src_bytes) {
    asm volatile(
        "cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n"
        :: "r"(dst_smem), "l"(src_gmem), "r"(src_bytes)
    );
}

__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n" ::: "memory");
}

template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N) : "memory");
}

__device__ __forceinline__ void ldmatrix_x4(
    uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t smem_addr
) {
    asm volatile(
        "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
        : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
        : "r"(smem_addr)
    );
}

__device__ __forceinline__ void mma_m16n8k16_bf16(
    float& d0, float& d1, float& d2, float& d3,
    uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
    uint32_t b0, uint32_t b1
) {
    asm volatile(
        "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
        "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
        : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
        : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)
    );
}

__device__ __forceinline__ void red_global_add_v4_f32(float* addr, float4 v) {
    asm volatile(
        "red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n"
        :: "l"(addr), "f"(v.x), "f"(v.y), "f"(v.z), "f"(v.w)
    );
}

__device__ __forceinline__ float silu_f32(float x) {
    return x / (1.0f + expf(-x));
}

__device__ __forceinline__ float warp_reduce_sum(float val) {
    #pragma unroll
    for (int offset = 16; offset > 0; offset >>= 1) {
        val += __shfl_down_sync(0xffffffff, val, offset);
    }
    return val;
}

__device__ __forceinline__ float dot8_bf16_f32(uint4 a, uint4 b) {
    const __nv_bfloat162* a2 = reinterpret_cast<const __nv_bfloat162*>(&a);
    const __nv_bfloat162* b2 = reinterpret_cast<const __nv_bfloat162*>(&b);
    float acc = 0.0f;
    #pragma unroll
    for (int i = 0; i < 4; ++i) {
        float2 fa = __bfloat1622float2(a2[i]);
        float2 fb = __bfloat1622float2(b2[i]);
        acc = fmaf(fa.x, fb.x, acc);
        acc = fmaf(fa.y, fb.y, acc);
    }
    return acc;
}

// ============================================================================
// REGIME A: Ultra-Fast Fused Decode GEMV (T <= 4, e.g. T = 1)
// ============================================================================

// Kernel D1: Computes h_buf[t, slot, col] = silu(dot(x[t], gate)) * dot(x[t], up)
// Grid: x = I / WARPS_PER_BLOCK, y = total_slots (top_k + n_shared), z = T
// Block: WARPS_PER_BLOCK * 32 threads (default 4 warps = 128 threads)
template <int WARPS_PER_BLOCK>
__global__ void moe_decode_gemv1_silu_mul_kernel(
    const __nv_bfloat16* __restrict__ x,          // (T, H)
    const int64_t* __restrict__ expert_ids,       // (T, top_k)
    const __nv_bfloat16* __restrict__ w1_routed,  // (E, 2*I, H)
    const __nv_bfloat16* __restrict__ w1_shared,  // (n_shared, 2*I, H)
    __nv_bfloat16* __restrict__ h_buf,            // (T, top_k + n_shared, I)
    int T, int E, int top_k, int n_shared, int H, int I
) {
    extern __shared__ __nv_bfloat16 s_x[]; // H bf16 (8 KB for H=4096)

    const int t = blockIdx.z;
    const int slot = blockIdx.y;
    const int warp_id = threadIdx.x >> 5;
    const int lane = threadIdx.x & 31;
    const int col = blockIdx.x * WARPS_PER_BLOCK + warp_id;

    // Cooperatively load x[t, 0..H-1] into SMEM using 128-bit loads
    const int H_u4 = H >> 3;
    const uint4* x_u4 = reinterpret_cast<const uint4*>(x + (size_t)t * H);
    uint4* sx_u4 = reinterpret_cast<uint4*>(s_x);
    for (int i = threadIdx.x; i < H_u4; i += WARPS_PER_BLOCK * 32) {
        sx_u4[i] = x_u4[i];
    }
    __syncthreads();

    if (col >= I) return;

    const __nv_bfloat16* w1_base;
    if (slot < top_k) {
        int64_t e = expert_ids[(size_t)t * top_k + slot];
        w1_base = w1_routed + (size_t)e * (size_t)(2 * I) * (size_t)H;
    } else {
        int s = slot - top_k;
        w1_base = w1_shared + (size_t)s * (size_t)(2 * I) * (size_t)H;
    }

    const uint4* wg_u4 = reinterpret_cast<const uint4*>(w1_base + (size_t)col * H);
    const uint4* wu_u4 = reinterpret_cast<const uint4*>(w1_base + (size_t)(I + col) * H);

    float acc_g0 = 0.0f, acc_g1 = 0.0f;
    float acc_u0 = 0.0f, acc_u1 = 0.0f;

    // 2x unrolled streaming 128-bit loads along H
    int i = lane;
    for (; i + 32 < H_u4; i += 64) {
        uint4 x0 = sx_u4[i];
        uint4 x1 = sx_u4[i + 32];
        uint4 g0 = __ldcs(&wg_u4[i]);
        uint4 u0 = __ldcs(&wu_u4[i]);
        uint4 g1 = __ldcs(&wg_u4[i + 32]);
        uint4 u1 = __ldcs(&wu_u4[i + 32]);
        acc_g0 += dot8_bf16_f32(x0, g0);
        acc_u0 += dot8_bf16_f32(x0, u0);
        acc_g1 += dot8_bf16_f32(x1, g1);
        acc_u1 += dot8_bf16_f32(x1, u1);
    }
    for (; i < H_u4; i += 32) {
        uint4 x0 = sx_u4[i];
        uint4 g0 = __ldcs(&wg_u4[i]);
        uint4 u0 = __ldcs(&wu_u4[i]);
        acc_g0 += dot8_bf16_f32(x0, g0);
        acc_u0 += dot8_bf16_f32(x0, u0);
    }

    float g_sum = warp_reduce_sum(acc_g0 + acc_g1);
    float u_sum = warp_reduce_sum(acc_u0 + acc_u1);

    if (lane == 0) {
        int total_slots = top_k + n_shared;
        float h = silu_f32(g_sum) * u_sum;
        h_buf[((size_t)t * total_slots + slot) * I + col] = __float2bfloat16(h);
    }
}

// Kernel D2: Computes out[t, col] = sum_slot w_slot * dot(h_buf[t, slot, :], w2_slot[col, :])
// Grid: x = H / WARPS_PER_BLOCK, z = T
// Block: WARPS_PER_BLOCK * 32 threads (2 warps = 64 threads -> 2048 CTAs for H=4096 -> 10.9 waves on 188 SMs!)
template <int WARPS_PER_BLOCK>
__global__ void moe_decode_gemv2_reduce_kernel(
    const __nv_bfloat16* __restrict__ h_buf,          // (T, total_slots, I)
    const int64_t* __restrict__ expert_ids,           // (T, top_k)
    const __nv_bfloat16* __restrict__ expert_weights, // (T, top_k)
    const __nv_bfloat16* __restrict__ w2_routed,      // (E, H, I)
    const __nv_bfloat16* __restrict__ w2_shared,      // (n_shared, H, I)
    __nv_bfloat16* __restrict__ out,                  // (T, H)
    int T, int E, int top_k, int n_shared, int H, int I
) {
    const int t = blockIdx.z;
    const int warp_id = threadIdx.x >> 5;
    const int lane = threadIdx.x & 31;
    const int col = blockIdx.x * WARPS_PER_BLOCK + warp_id;
    if (col >= H) return;

    const int total_slots = top_k + n_shared;
    const int I_u4 = I >> 3;

    float total_acc = 0.0f;

    for (int slot = 0; slot < total_slots; ++slot) {
        const __nv_bfloat16* w2_row;
        float weight;
        if (slot < top_k) {
            int64_t e = expert_ids[(size_t)t * top_k + slot];
            weight = __bfloat162float(expert_weights[(size_t)t * top_k + slot]);
            w2_row = w2_routed + ((size_t)e * (size_t)H + (size_t)col) * (size_t)I;
        } else {
            int s = slot - top_k;
            weight = 1.0f;
            w2_row = w2_shared + ((size_t)s * (size_t)H + (size_t)col) * (size_t)I;
        }

        const uint4* h_u4 = reinterpret_cast<const uint4*>(
            h_buf + ((size_t)t * total_slots + slot) * I
        );
        const uint4* w_u4 = reinterpret_cast<const uint4*>(w2_row);

        float acc0 = 0.0f, acc1 = 0.0f;
        int i = lane;
        #pragma unroll 4
        for (; i + 32 < I_u4; i += 64) {
            uint4 hv0 = __ldg(&h_u4[i]);
            uint4 wv0 = __ldcs(&w_u4[i]);
            uint4 hv1 = __ldg(&h_u4[i + 32]);
            uint4 wv1 = __ldcs(&w_u4[i + 32]);
            acc0 += dot8_bf16_f32(hv0, wv0);
            acc1 += dot8_bf16_f32(hv1, wv1);
        }
        for (; i < I_u4; i += 32) {
            uint4 hv0 = __ldg(&h_u4[i]);
            uint4 wv0 = __ldcs(&w_u4[i]);
            acc0 += dot8_bf16_f32(hv0, wv0);
        }
        total_acc += weight * (acc0 + acc1);
    }

    float final_sum = warp_reduce_sum(total_acc);
    if (lane == 0) {
        out[(size_t)t * H + col] = __float2bfloat16(final_sum);
    }
}

// ============================================================================
// REGIME B: Zero-Sync Routing Sort + M-Tile Table Construction
// ============================================================================

__global__ void moe_count_experts_kernel(
    const int64_t* __restrict__ expert_ids,
    int32_t* __restrict__ expert_counts,
    int M_total, int E
) {
    extern __shared__ int32_t s_counts[];
    for (int i = threadIdx.x; i < E; i += blockDim.x) {
        s_counts[i] = 0;
    }
    __syncthreads();

    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    int stride = gridDim.x * blockDim.x;
    for (; idx < M_total; idx += stride) {
        int e = static_cast<int>(expert_ids[idx]);
        atomicAdd(&s_counts[e], 1);
    }
    __syncthreads();

    for (int i = threadIdx.x; i < E; i += blockDim.x) {
        int c = s_counts[i];
        if (c > 0) {
            atomicAdd(&expert_counts[i], c);
        }
    }
}

// Single-block kernel (up to E <= 1024) computing prefix sums & tile table
__global__ void moe_build_offsets_and_tiles_kernel(
    const int32_t* __restrict__ expert_counts,
    int32_t* __restrict__ expert_offsets,       // (E + 1)
    int32_t* __restrict__ write_cursors,        // (E)
    int32_t* __restrict__ expert_tile_prefix,   // (E + 1)
    int32_t* __restrict__ m_tile_to_expert,     // (max_m_tiles)
    int E, int BM
) {
    // Simple warp/thread scan (E=256 takes < 2 microseconds)
    if (threadIdx.x == 0) {
        int off = 0;
        int tile_off = 0;
        for (int e = 0; e < E; ++e) {
            int c = expert_counts[e];
            expert_offsets[e] = off;
            write_cursors[e] = off;
            expert_tile_prefix[e] = tile_off;
            int mt = (c + BM - 1) / BM;
            for (int m = 0; m < mt; ++m) {
                m_tile_to_expert[tile_off + m] = e;
            }
            off += c;
            tile_off += mt;
        }
        expert_offsets[E] = off;
        expert_tile_prefix[E] = tile_off;
    }
}

__global__ void moe_scatter_tokens_kernel(
    const int64_t* __restrict__ expert_ids,
    const __nv_bfloat16* __restrict__ expert_weights,
    int32_t* __restrict__ write_cursors,
    int32_t* __restrict__ sorted_token_ids,
    float* __restrict__ sorted_weights,
    int M_total, int top_k
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    int stride = gridDim.x * blockDim.x;
    for (; idx < M_total; idx += stride) {
        int e = static_cast<int>(expert_ids[idx]);
        int pos = atomicAdd(&write_cursors[e], 1);
        sorted_token_ids[pos] = idx / top_k;
        sorted_weights[pos] = __bfloat162float(expert_weights[idx]);
    }
}

// ============================================================================
// REGIME B: Fused Grouped GEMM1 (Gather X + Gate + Up + SiLU*Mul -> h)
// ============================================================================
// Tile: BM x BN_OUT (BN_OUT = 64 -> 64 gate + 64 up = 128 weight rows), BK = 64
// 8 warps (256 threads):
//   - BM = 32:  WARPS_M = 2 (WM = 16, 1 tile), WARPS_N = 4 (WN = 16, 1 tile of 16)
//   - BM = 64:  WARPS_M = 2 (WM = 32, 2 tiles), WARPS_N = 4 (WN = 16, 1 tile of 16)
//   - BM = 128: WARPS_M = 4 (WM = 32, 2 tiles), WARPS_N = 2 (WN = 32, 2 tiles of 16)

template <int BM, int BN_OUT, int BK, int STAGES, int WARPS_M, int WARPS_N, bool IS_SHARED>
__global__ __launch_bounds__(WARPS_M * WARPS_N * 32, 1)
void moe_gemm1_silu_mul_kernel(
    const __nv_bfloat16* __restrict__ x,               // (T, H)
    const __nv_bfloat16* __restrict__ w1,              // (E or n_shared, 2*I, H)
    __nv_bfloat16* __restrict__ h_out,                 // (M_total or T, I)
    const int32_t* __restrict__ expert_offsets,        // (E + 1) [routed]
    const int32_t* __restrict__ expert_tile_prefix,    // (E + 1) [routed]
    const int32_t* __restrict__ m_tile_to_expert,      // (max_m_tiles) [routed]
    const int32_t* __restrict__ sorted_token_ids,      // (M_total) [routed]
    int T, int E, int H, int I, int shared_idx
) {
    static_assert(BK == 64, "BK must be 64 for 128B swizzle");
    static_assert(BN_OUT == 64, "BN_OUT must be 64");

    constexpr int NWARPS = WARPS_M * WARPS_N;
    constexpr int NTHREADS = NWARPS * 32;
    constexpr int WM = BM / WARPS_M;
    constexpr int WN = BN_OUT / WARPS_N;
    constexpr int M_TILES = WM / 16;
    constexpr int N_TILES = WN / 16; // each N_TILE is 16 cols = two m16n8k16 MMAs

    const int tid = threadIdx.x;
    const int warp_id = tid >> 5;
    const int lane = tid & 31;
    const int warp_m = warp_id / WARPS_N;
    const int warp_n = warp_id % WARPS_N;

    const int n_tiles_total = I / BN_OUT; // 32 for I=2048

    int e, row_base, m_valid, n_blk;
    if constexpr (IS_SHARED) {
        const int work_id = blockIdx.x;
        const int m_tiles_shared = (T + BM - 1) / BM;
        if (work_id >= m_tiles_shared * n_tiles_total) return;
        e = shared_idx;
        n_blk = work_id / m_tiles_shared;
        int m_blk = work_id - n_blk * m_tiles_shared;
        row_base = m_blk * BM;
        m_valid = min(BM, T - row_base);
    } else {
        const int work_id = blockIdx.x;
        const int total_m_tiles = expert_tile_prefix[E];
        if (work_id >= total_m_tiles * n_tiles_total) return;
        int m_tile_approx = work_id / n_tiles_total;
        e = m_tile_to_expert[m_tile_approx];
        int p0 = expert_tile_prefix[e];
        int p1 = expert_tile_prefix[e + 1];
        int m_tiles_e = p1 - p0;
        int local = work_id - p0 * n_tiles_total;
        n_blk = local / m_tiles_e;
        int m_blk = local - n_blk * m_tiles_e;
        int off0 = expert_offsets[e];
        int off1 = expert_offsets[e + 1];
        row_base = off0 + m_blk * BM;
        m_valid = min(BM, off1 - row_base);
    }

    const int col_base = n_blk * BN_OUT;

    // Shared memory layout:
    //   int32 s_tok[BM] (at the end or separate? Put s_tok in small static smem!)
    //   Dynamic SMEM (16B aligned):
    //     For each stage s in [0, STAGES):
    //       A[s]:      BM * 64 bf16     = BM * 128 B
    //       B_gate[s]: BN_OUT * 64 bf16 = 8192 B
    //       B_up[s]:   BN_OUT * 64 bf16 = 8192 B
    //   Epilogue reuses dynamic SMEM: s_h[BM][BN_OUT + 8] bf16
    __shared__ int32_t s_tok[BM];
    if constexpr (!IS_SHARED) {
        if (tid < BM) {
            s_tok[tid] = (tid < m_valid) ? sorted_token_ids[row_base + tid] : 0;
        }
        __syncthreads();
    }

    extern __shared__ char smem_raw[];
    constexpr uint32_t A_STAGE_BYTES = BM * 128;
    constexpr uint32_t B_STAGE_BYTES = BN_OUT * 128; // 8192 B
    constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_STAGE_BYTES;

    const uint32_t smem_base = smem_u32(smem_raw);

    const __nv_bfloat16* w1_e = w1 + (size_t)e * (size_t)(2 * I) * (size_t)H;
    const __nv_bfloat16* wg_base = w1_e + (size_t)col_base * (size_t)H;
    const __nv_bfloat16* wu_base = w1_e + (size_t)(I + col_base) * (size_t)H;

    // Accumulators in FP32
    float acc_g[M_TILES][N_TILES * 2][4];
    float acc_u[M_TILES][N_TILES * 2][4];
    #pragma unroll
    for (int mi = 0; mi < M_TILES; ++mi) {
        #pragma unroll
        for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
            acc_g[mi][n8][0] = 0.0f; acc_g[mi][n8][1] = 0.0f;
            acc_g[mi][n8][2] = 0.0f; acc_g[mi][n8][3] = 0.0f;
            acc_u[mi][n8][0] = 0.0f; acc_u[mi][n8][1] = 0.0f;
            acc_u[mi][n8][2] = 0.0f; acc_u[mi][n8][3] = 0.0f;
        }
    }

    // Helper lambda to issue async loads for a K-tile `kt` into `stage`
    auto load_stage = [&](int kt, int stage) {
        const int k_off = kt * BK;
        const uint32_t st_base = smem_base + stage * STAGE_BYTES;
        const uint32_t a_base = st_base;
        const uint32_t bg_base = st_base + A_STAGE_BYTES;
        const uint32_t bu_base = bg_base + B_STAGE_BYTES;

        // Load A: BM * 8 chunks of 16B
        constexpr int A_CHUNKS = BM * 8;
        #pragma unroll
        for (int idx = tid; idx < A_CHUNKS; idx += NTHREADS) {
            int r = idx >> 3;
            int c8 = idx & 7;
            int swiz = c8 ^ (r & 7);
            uint32_t dst = a_base + (r * 8 + swiz) * 16;
            int valid_bytes = (r < m_valid) ? 16 : 0;
            const __nv_bfloat16* src;
            if constexpr (IS_SHARED) {
                int r_clamped = (r < m_valid) ? (row_base + r) : row_base;
                src = x + (size_t)r_clamped * H + k_off + c8 * 8;
            } else {
                int tok = s_tok[r];
                src = x + (size_t)tok * H + k_off + c8 * 8;
            }
            cp_async_cg_16_zfill(dst, src, valid_bytes);
        }

        // Load B_gate & B_up: BN_OUT * 8 = 512 chunks each
        constexpr int B_CHUNKS = BN_OUT * 8;
        #pragma unroll
        for (int idx = tid; idx < B_CHUNKS; idx += NTHREADS) {
            int r = idx >> 3;
            int c8 = idx & 7;
            int swiz = c8 ^ (r & 7);
            uint32_t off = (r * 8 + swiz) * 16;
            cp_async_cg_16(bg_base + off, wg_base + (size_t)r * H + k_off + c8 * 8);
            cp_async_cg_16(bu_base + off, wu_base + (size_t)r * H + k_off + c8 * 8);
        }
    };

    const int num_k_tiles = H / BK;

    // Prologue: fill STAGES - 1 buffers
    #pragma unroll
    for (int s = 0; s < STAGES - 1; ++s) {
        if (s < num_k_tiles) {
            load_stage(s, s);
        }
        cp_async_commit();
    }

    const int lane_r16 = lane & 15;
    const int lane_r7 = lane & 7;
    const int lane_c_bit = lane >> 4; // 0 or 1

    const int warp_m_base = warp_m * WM;
    const int warp_n_base = warp_n * WN;

    // Main K-loop
    for (int kt = 0; kt < num_k_tiles; ++kt) {
        const int read_stage = kt % STAGES;
        const int write_stage = (kt + STAGES - 1) % STAGES;
        const int next_kt = kt + STAGES - 1;

        cp_async_wait<STAGES - 2>();
        __syncthreads();

        // Issue next async stage immediately
        if (next_kt < num_k_tiles) {
            load_stage(next_kt, write_stage);
        }
        cp_async_commit();

        const uint32_t st_base = smem_base + read_stage * STAGE_BYTES;
        const uint32_t a_base = st_base;
        const uint32_t bg_base = st_base + A_STAGE_BYTES;
        const uint32_t bu_base = bg_base + B_STAGE_BYTES;

        // Compute 4 K-steps of 16 with register double-buffering on A
        uint32_t ra[2][M_TILES][4];

        // Prefetch A for ks = 0
        #pragma unroll
        for (int mi = 0; mi < M_TILES; ++mi) {
            int r = warp_m_base + mi * 16 + lane_r16;
            int c8 = lane_c_bit;
            uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
            ldmatrix_x4(ra[0][mi][0], ra[0][mi][1], ra[0][mi][2], ra[0][mi][3], addr);
        }

        #pragma unroll
        for (int ks = 0; ks < 4; ++ks) {
            const int cur = ks & 1;
            const int nxt = cur ^ 1;

            // Prefetch A for ks + 1
            if (ks < 3) {
                #pragma unroll
                for (int mi = 0; mi < M_TILES; ++mi) {
                    int r = warp_m_base + mi * 16 + lane_r16;
                    int c8 = (ks + 1) * 2 + lane_c_bit;
                    uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
                    ldmatrix_x4(ra[nxt][mi][0], ra[nxt][mi][1], ra[nxt][mi][2], ra[nxt][mi][3], addr);
                }
            }

            // Stream B_gate and B_up per N-tile and run MMA
            #pragma unroll
            for (int ni = 0; ni < N_TILES; ++ni) {
                int nb = warp_n_base + ni * 16 + lane_r16;
                int c8 = ks * 2 + lane_c_bit;
                uint32_t off = (nb * 8 + (c8 ^ lane_r7)) * 16;
                uint32_t bg0, bg1, bg2, bg3;
                uint32_t bu0, bu1, bu2, bu3;
                ldmatrix_x4(bg0, bg1, bg2, bg3, bg_base + off);
                ldmatrix_x4(bu0, bu1, bu2, bu3, bu_base + off);

                #pragma unroll
                for (int mi = 0; mi < M_TILES; ++mi) {
                    mma_m16n8k16_bf16(
                        acc_g[mi][2 * ni + 0][0], acc_g[mi][2 * ni + 0][1],
                        acc_g[mi][2 * ni + 0][2], acc_g[mi][2 * ni + 0][3],
                        ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
                        bg0, bg2
                    );
                    mma_m16n8k16_bf16(
                        acc_g[mi][2 * ni + 1][0], acc_g[mi][2 * ni + 1][1],
                        acc_g[mi][2 * ni + 1][2], acc_g[mi][2 * ni + 1][3],
                        ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
                        bg1, bg3
                    );
                    mma_m16n8k16_bf16(
                        acc_u[mi][2 * ni + 0][0], acc_u[mi][2 * ni + 0][1],
                        acc_u[mi][2 * ni + 0][2], acc_u[mi][2 * ni + 0][3],
                        ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
                        bu0, bu2
                    );
                    mma_m16n8k16_bf16(
                        acc_u[mi][2 * ni + 1][0], acc_u[mi][2 * ni + 1][1],
                        acc_u[mi][2 * ni + 1][2], acc_u[mi][2 * ni + 1][3],
                        ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
                        bu1, bu3
                    );
                }
            }
        }
    }

    // Drain any remaining async ops before reusing SMEM for epilogue
    cp_async_wait<0>();
    __syncthreads();

    // Epilogue: compute SiLU(gate) * up in FP32, stage bf16 to SMEM with +8 padding
    constexpr int S_H_STRIDE = BN_OUT + 8; // 72 bf16 = 144B (16B aligned, bank-conflict-free!)
    __nv_bfloat16* s_h = reinterpret_cast<__nv_bfloat16*>(smem_raw);

    const int g = lane >> 2;
    const int t = lane & 3;

    #pragma unroll
    for (int mi = 0; mi < M_TILES; ++mi) {
        int r0 = warp_m_base + mi * 16 + g;
        int r1 = r0 + 8;
        #pragma unroll
        for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
            int c0 = warp_n_base + n8 * 8 + t * 2;
            float h0 = silu_f32(acc_g[mi][n8][0]) * acc_u[mi][n8][0];
            float h1 = silu_f32(acc_g[mi][n8][1]) * acc_u[mi][n8][1];
            float h2 = silu_f32(acc_g[mi][n8][2]) * acc_u[mi][n8][2];
            float h3 = silu_f32(acc_g[mi][n8][3]) * acc_u[mi][n8][3];
            *reinterpret_cast<__nv_bfloat162*>(&s_h[r0 * S_H_STRIDE + c0]) = __floats2bfloat162_rn(h0, h1);
            *reinterpret_cast<__nv_bfloat162*>(&s_h[r1 * S_H_STRIDE + c0]) = __floats2bfloat162_rn(h2, h3);
        }
    }
    __syncthreads();

    // Coalesced 128-bit stores to global h_out[row_base + r, col_base + c]
    constexpr int H_CHUNKS = BM * (BN_OUT / 8); // BM * 8
    #pragma unroll
    for (int idx = tid; idx < H_CHUNKS; idx += NTHREADS) {
        int r = idx >> 3;
        int c8 = idx & 7;
        if (r < m_valid) {
            uint4 v = reinterpret_cast<const uint4*>(&s_h[r * S_H_STRIDE])[c8];
            reinterpret_cast<uint4*>(h_out + (size_t)(row_base + r) * I + col_base)[c8] = v;
        }
    }
}

// ============================================================================
// REGIME B: Fused Grouped GEMM2 (Down Proj + Weight Scale + v4.f32 Scatter-Add)
//           Also supports Shared Expert (FUSE_ADD_CAST -> direct bf16 output!)
// ============================================================================
// Tile: BM x BN (BN = 128), BK = 64
// 8 warps (256 threads):
//   - BM = 32:  WARPS_M = 2 (WM = 16, 1 tile), WARPS_N = 4 (WN = 32, 2 tiles of 16)
//   - BM = 64:  WARPS_M = 2 (WM = 32, 2 tiles), WARPS_N = 4 (WN = 32, 2 tiles of 16)
//   - BM = 128: WARPS_M = 4 (WM = 32, 2 tiles), WARPS_N = 2 (WN = 64, 4 tiles of 16)

// EPILOGUE_MODE:
//   0 = Routed scatter-add: scale by sorted_weights[r], red.global.add.v4.f32 -> out_fp32[tok]
//   1 = Shared add to out_fp32: red.global.add.v4.f32 -> out_fp32[row] (for multi-shared s < n_shared-1)
//   2 = Shared final fused add+cast: out_bf16[row] = bf16(acc + out_fp32[row]) (for last shared expert!)
template <int BM, int BN, int BK, int STAGES, int WARPS_M, int WARPS_N, int EPILOGUE_MODE>
__global__ __launch_bounds__(WARPS_M * WARPS_N * 32, 1)
void moe_gemm2_kernel(
    const __nv_bfloat16* __restrict__ h_in,            // (M_total or T, I)
    const __nv_bfloat16* __restrict__ w2,              // (E or n_shared, H, I)
    float* __restrict__ out_fp32,                      // (T, H)
    __nv_bfloat16* __restrict__ out_bf16,              // (T, H) [mode 2]
    const int32_t* __restrict__ expert_offsets,        // (E + 1) [mode 0]
    const int32_t* __restrict__ expert_tile_prefix,    // (E + 1) [mode 0]
    const int32_t* __restrict__ m_tile_to_expert,      // (max_m_tiles) [mode 0]
    const int32_t* __restrict__ sorted_token_ids,      // (M_total) [mode 0]
    const float* __restrict__ sorted_weights,          // (M_total) [mode 0]
    int T, int E, int H, int I, int shared_idx
) {
    static_assert(BK == 64, "BK must be 64");
    static_assert(BN == 128, "BN must be 128");

    constexpr bool IS_SHARED = (EPILOGUE_MODE != 0);
    constexpr int NWARPS = WARPS_M * WARPS_N;
    constexpr int NTHREADS = NWARPS * 32;
    constexpr int WM = BM / WARPS_M;
    constexpr int WN = BN / WARPS_N;
    constexpr int M_TILES = WM / 16;
    constexpr int N_TILES = WN / 16;

    const int tid = threadIdx.x;
    const int warp_id = tid >> 5;
    const int lane = tid & 31;
    const int warp_m = warp_id / WARPS_N;
    const int warp_n = warp_id % WARPS_N;

    const int n_tiles_total = H / BN; // 32 for H=4096

    int e, row_base, m_valid, n_blk;
    if constexpr (IS_SHARED) {
        const int work_id = blockIdx.x;
        const int m_tiles_shared = (T + BM - 1) / BM;
        if (work_id >= m_tiles_shared * n_tiles_total) return;
        e = shared_idx;
        n_blk = work_id / m_tiles_shared;
        int m_blk = work_id - n_blk * m_tiles_shared;
        row_base = m_blk * BM;
        m_valid = min(BM, T - row_base);
    } else {
        const int work_id = blockIdx.x;
        const int total_m_tiles = expert_tile_prefix[E];
        if (work_id >= total_m_tiles * n_tiles_total) return;
        int m_tile_approx = work_id / n_tiles_total;
        e = m_tile_to_expert[m_tile_approx];
        int p0 = expert_tile_prefix[e];
        int p1 = expert_tile_prefix[e + 1];
        int m_tiles_e = p1 - p0;
        int local = work_id - p0 * n_tiles_total;
        n_blk = local / m_tiles_e;
        int m_blk = local - n_blk * m_tiles_e;
        int off0 = expert_offsets[e];
        int off1 = expert_offsets[e + 1];
        row_base = off0 + m_blk * BM;
        m_valid = min(BM, off1 - row_base);
    }

    const int col_base = n_blk * BN;

    __shared__ int32_t s_tok[BM];
    __shared__ float s_wt[BM];
    if constexpr (!IS_SHARED) {
        if (tid < BM) {
            if (tid < m_valid) {
                s_tok[tid] = sorted_token_ids[row_base + tid];
                s_wt[tid] = sorted_weights[row_base + tid];
            } else {
                s_tok[tid] = 0;
                s_wt[tid] = 0.0f;
            }
        }
        __syncthreads();
    }

    extern __shared__ char smem_raw[];
    constexpr uint32_t A_STAGE_BYTES = BM * 128;
    constexpr uint32_t B_STAGE_BYTES = BN * 128; // 16384 B
    constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES;

    const uint32_t smem_base = smem_u32(smem_raw);
    const __nv_bfloat16* w2_base = w2 + ((size_t)e * (size_t)H + (size_t)col_base) * (size_t)I;

    float acc[M_TILES][N_TILES * 2][4];
    #pragma unroll
    for (int mi = 0; mi < M_TILES; ++mi) {
        #pragma unroll
        for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
            acc[mi][n8][0] = 0.0f; acc[mi][n8][1] = 0.0f;
            acc[mi][n8][2] = 0.0f; acc[mi][n8][3] = 0.0f;
        }
    }

    auto load_stage = [&](int kt, int stage) {
        const int k_off = kt * BK;
        const uint32_t st_base = smem_base + stage * STAGE_BYTES;
        const uint32_t a_base = st_base;
        const uint32_t b_base = st_base + A_STAGE_BYTES;

        // Load A (h_in is contiguous!)
        constexpr int A_CHUNKS = BM * 8;
        #pragma unroll
        for (int idx = tid; idx < A_CHUNKS; idx += NTHREADS) {
            int r = idx >> 3;
            int c8 = idx & 7;
            int swiz = c8 ^ (r & 7);
            uint32_t dst = a_base + (r * 8 + swiz) * 16;
            int valid_bytes = (r < m_valid) ? 16 : 0;
            int r_clamped = (r < m_valid) ? (row_base + r) : row_base;
            const __nv_bfloat16* src = h_in + (size_t)r_clamped * I + k_off + c8 * 8;
            cp_async_cg_16_zfill(dst, src, valid_bytes);
        }

        // Load B (w2)
        constexpr int B_CHUNKS = BN * 8; // 1024 chunks
        #pragma unroll
        for (int idx = tid; idx < B_CHUNKS; idx += NTHREADS) {
            int r = idx >> 3;
            int c8 = idx & 7;
            int swiz = c8 ^ (r & 7);
            uint32_t dst = b_base + (r * 8 + swiz) * 16;
            cp_async_cg_16(dst, w2_base + (size_t)r * I + k_off + c8 * 8);
        }
    };

    const int num_k_tiles = I / BK;

    #pragma unroll
    for (int s = 0; s < STAGES - 1; ++s) {
        if (s < num_k_tiles) {
            load_stage(s, s);
        }
        cp_async_commit();
    }

    const int lane_r16 = lane & 15;
    const int lane_r7 = lane & 7;
    const int lane_c_bit = lane >> 4;

    const int warp_m_base = warp_m * WM;
    const int warp_n_base = warp_n * WN;

    for (int kt = 0; kt < num_k_tiles; ++kt) {
        const int read_stage = kt % STAGES;
        const int write_stage = (kt + STAGES - 1) % STAGES;
        const int next_kt = kt + STAGES - 1;

        cp_async_wait<STAGES - 2>();
        __syncthreads();

        if (next_kt < num_k_tiles) {
            load_stage(next_kt, write_stage);
        }
        cp_async_commit();

        const uint32_t st_base = smem_base + read_stage * STAGE_BYTES;
        const uint32_t a_base = st_base;
        const uint32_t b_base = st_base + A_STAGE_BYTES;

        uint32_t ra[2][M_TILES][4];

        #pragma unroll
        for (int mi = 0; mi < M_TILES; ++mi) {
            int r = warp_m_base + mi * 16 + lane_r16;
            int c8 = lane_c_bit;
            uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
            ldmatrix_x4(ra[0][mi][0], ra[0][mi][1], ra[0][mi][2], ra[0][mi][3], addr);
        }

        #pragma unroll
        for (int ks = 0; ks < 4; ++ks) {
            const int cur = ks & 1;
            const int nxt = cur ^ 1;

            if (ks < 3) {
                #pragma unroll
                for (int mi = 0; mi < M_TILES; ++mi) {
                    int r = warp_m_base + mi * 16 + lane_r16;
                    int c8 = (ks + 1) * 2 + lane_c_bit;
                    uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
                    ldmatrix_x4(ra[nxt][mi][0], ra[nxt][mi][1], ra[nxt][mi][2], ra[nxt][mi][3], addr);
                }
            }

            #pragma unroll
            for (int ni = 0; ni < N_TILES; ++ni) {
                int nb = warp_n_base + ni * 16 + lane_r16;
                int c8 = ks * 2 + lane_c_bit;
                uint32_t b0, b1, b2, b3;
                ldmatrix_x4(b0, b1, b2, b3, b_base + (nb * 8 + (c8 ^ lane_r7)) * 16);

                #pragma unroll
                for (int mi = 0; mi < M_TILES; ++mi) {
                    mma_m16n8k16_bf16(
                        acc[mi][2 * ni + 0][0], acc[mi][2 * ni + 0][1],
                        acc[mi][2 * ni + 0][2], acc[mi][2 * ni + 0][3],
                        ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
                        b0, b2
                    );
                    mma_m16n8k16_bf16(
                        acc[mi][2 * ni + 1][0], acc[mi][2 * ni + 1][1],
                        acc[mi][2 * ni + 1][2], acc[mi][2 * ni + 1][3],
                        ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
                        b1, b3
                    );
                }
            }
        }
    }

    cp_async_wait<0>();
    __syncthreads();

    // Epilogue: stage FP32 results to SMEM (padded stride 136 floats = 544B, 16B aligned)
    // Max size: BM=128 -> 128 * 136 * 4 = 69,632 B (fits in 96KB SMEM!)
    //           BM=64  ->  64 * 136 * 4 = 34,816 B (fits in 96KB SMEM!)
    //           BM=32  ->  32 * 136 * 4 = 17,408 B (fits in 80KB SMEM!)
    constexpr int S_OUT_STRIDE = BN + 8; // 136 floats
    float* s_out = reinterpret_cast<float*>(smem_raw);

    const int g = lane >> 2;
    const int t = lane & 3;

    #pragma unroll
    for (int mi = 0; mi < M_TILES; ++mi) {
        int r0 = warp_m_base + mi * 16 + g;
        int r1 = r0 + 8;
        float w0 = IS_SHARED ? 1.0f : s_wt[r0];
        float w1 = IS_SHARED ? 1.0f : s_wt[r1];
        #pragma unroll
        for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
            int c0 = warp_n_base + n8 * 8 + t * 2;
            float2 v0 = make_float2(acc[mi][n8][0] * w0, acc[mi][n8][1] * w0);
            float2 v1 = make_float2(acc[mi][n8][2] * w1, acc[mi][n8][3] * w1);
            *reinterpret_cast<float2*>(&s_out[r0 * S_OUT_STRIDE + c0]) = v0;
            *reinterpret_cast<float2*>(&s_out[r1 * S_OUT_STRIDE + c0]) = v1;
        }
    }
    __syncthreads();

    // 1 warp writes 1 full row of 128 floats (32 float4s = 512B) in a single coalesced step!
    for (int r = warp_id; r < m_valid; r += NWARPS) {
        float4 v = reinterpret_cast<const float4*>(&s_out[r * S_OUT_STRIDE])[lane];
        if constexpr (EPILOGUE_MODE == 0) {
            int tok = s_tok[r];
            float* dst = out_fp32 + (size_t)tok * H + col_base + lane * 4;
            red_global_add_v4_f32(dst, v);
        } else if constexpr (EPILOGUE_MODE == 1) {
            int row = row_base + r;
            float* dst = out_fp32 + (size_t)row * H + col_base + lane * 4;
            red_global_add_v4_f32(dst, v);
        } else {
            int row = row_base + r;
            const float4 prev = reinterpret_cast<const float4*>(
                out_fp32 + (size_t)row * H + col_base
            )[lane];
            __nv_bfloat162 b0 = __floats2bfloat162_rn(v.x + prev.x, v.y + prev.y);
            __nv_bfloat162 b1 = __floats2bfloat162_rn(v.z + prev.z, v.w + prev.w);
            uint2 packed;
            packed.x = *reinterpret_cast<const uint32_t*>(&b0);
            packed.y = *reinterpret_cast<const uint32_t*>(&b1);
            reinterpret_cast<uint2*>(out_bf16 + (size_t)row * H + col_base)[lane] = packed;
        }
    }
}

__global__ void cast_f32_to_bf16_kernel(
    const float* __restrict__ src,
    __nv_bfloat16* __restrict__ dst,
    size_t n_vec4
) {
    size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
    size_t stride = (size_t)gridDim.x * blockDim.x;
    const float4* src4 = reinterpret_cast<const float4*>(src);
    uint2* dst2 = reinterpret_cast<uint2*>(dst);
    for (; idx < n_vec4; idx += stride) {
        float4 v = src4[idx];
        __nv_bfloat162 b0 = __floats2bfloat162_rn(v.x, v.y);
        __nv_bfloat162 b1 = __floats2bfloat162_rn(v.z, v.w);
        uint2 packed;
        packed.x = *reinterpret_cast<const uint32_t*>(&b0);
        packed.y = *reinterpret_cast<const uint32_t*>(&b1);
        dst2[idx] = packed;
    }
}

// ============================================================================
// C++ Launch Wrappers
// ============================================================================

template <typename KernelT>
static inline void ensure_smem_once(bool& configured, KernelT kernel, size_t smem_bytes) {
    if (!configured && smem_bytes > 48 * 1024) {
        C10_CUDA_CHECK(cudaFuncSetAttribute(
            kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(smem_bytes)
        ));
        configured = true;
    }
}

void launch_decode_moe(
    torch::Tensor x,
    torch::Tensor expert_ids,
    torch::Tensor expert_weights,
    torch::Tensor w1_routed,
    torch::Tensor w2_routed,
    torch::Tensor w1_shared,
    torch::Tensor w2_shared,
    torch::Tensor h_buf,
    torch::Tensor out
) {
    const int T = static_cast<int>(x.size(0));
    const int H = static_cast<int>(x.size(1));
    const int E = static_cast<int>(w1_routed.size(0));
    const int I = static_cast<int>(w2_routed.size(2));
    const int top_k = static_cast<int>(expert_ids.size(1));
    const int n_shared = static_cast<int>(w1_shared.size(0));
    const int total_slots = top_k + n_shared;

    cudaStream_t stream = c10::cuda::getCurrentCUDAStream();

    // Kernel D1: 4 warps/block -> 4608 blocks for T=1, 9 slots, I=2048
    {
        constexpr int WPB = 4;
        dim3 grid((I + WPB - 1) / WPB, total_slots, T);
        dim3 block(WPB * 32);
        size_t smem = (size_t)H * sizeof(__nv_bfloat16);
        moe_decode_gemv1_silu_mul_kernel<WPB><<<grid, block, smem, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            expert_ids.data_ptr<int64_t>(),
            reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(h_buf.data_ptr()),
            T, E, top_k, n_shared, H, I
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }

    // Kernel D2: 2 warps/block -> 2048 blocks for T=1, H=4096 (10.9 waves on 188 SMs)
    {
        constexpr int WPB = 2;
        dim3 grid((H + WPB - 1) / WPB, 1, T);
        dim3 block(WPB * 32);
        moe_decode_gemv2_reduce_kernel<WPB><<<grid, block, 0, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(h_buf.data_ptr()),
            expert_ids.data_ptr<int64_t>(),
            reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(out.data_ptr()),
            T, E, top_k, n_shared, H, I
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }
}

template <int BM, int STAGES, int WARPS_M, int WARPS_N>
static void run_routed_grouped_impl(
    const __nv_bfloat16* x_ptr,
    const __nv_bfloat16* w1_ptr,
    const __nv_bfloat16* w2_ptr,
    __nv_bfloat16* h_ptr,
    float* out_fp32_ptr,
    const int32_t* offsets_ptr,
    const int32_t* tile_prefix_ptr,
    const int32_t* m_tile_to_expert_ptr,
    const int32_t* sorted_tok_ptr,
    const float* sorted_wt_ptr,
    int T, int E, int H, int I, int max_m_tiles,
    cudaStream_t stream
) {
    constexpr int BN1_OUT = 64;
    constexpr int BN2 = 128;
    constexpr int BK = 64;
    constexpr int NTHREADS = WARPS_M * WARPS_N * 32;

    // GEMM1
    {
        static bool cfg1 = false;
        constexpr size_t smem1 = STAGES * (BM * 128 + 2 * BN1_OUT * 128);
        auto k1 = moe_gemm1_silu_mul_kernel<BM, BN1_OUT, BK, STAGES, WARPS_M, WARPS_N, false>;
        ensure_smem_once(cfg1, k1, smem1);
        int grid1 = max_m_tiles * (I / BN1_OUT);
        k1<<<grid1, NTHREADS, smem1, stream>>>(
            x_ptr, w1_ptr, h_ptr,
            offsets_ptr, tile_prefix_ptr, m_tile_to_expert_ptr, sorted_tok_ptr,
            T, E, H, I, 0
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }

    // GEMM2
    {
        static bool cfg2 = false;
        constexpr size_t smem2_pipe = STAGES * (BM * 128 + BN2 * 128);
        constexpr size_t smem2_epi = BM * (BN2 + 8) * sizeof(float);
        constexpr size_t smem2 = smem2_pipe > smem2_epi ? smem2_pipe : smem2_epi;
        auto k2 = moe_gemm2_kernel<BM, BN2, BK, STAGES, WARPS_M, WARPS_N, 0>;
        ensure_smem_once(cfg2, k2, smem2);
        int grid2 = max_m_tiles * (H / BN2);
        k2<<<grid2, NTHREADS, smem2, stream>>>(
            h_ptr, w2_ptr, out_fp32_ptr, nullptr,
            offsets_ptr, tile_prefix_ptr, m_tile_to_expert_ptr, sorted_tok_ptr, sorted_wt_ptr,
            T, E, H, I, 0
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }
}

template <int BM, int STAGES, int WARPS_M, int WARPS_N>
static void run_shared_impl(
    const __nv_bfloat16* x_ptr,
    const __nv_bfloat16* w1_s_ptr,
    const __nv_bfloat16* w2_s_ptr,
    __nv_bfloat16* h_s_ptr,
    float* out_fp32_ptr,
    __nv_bfloat16* out_bf16_ptr,
    int T, int H, int I, int s_idx, bool is_last,
    cudaStream_t stream
) {
    constexpr int BN1_OUT = 64;
    constexpr int BN2 = 128;
    constexpr int BK = 64;
    constexpr int NTHREADS = WARPS_M * WARPS_N * 32;
    const int m_tiles = (T + BM - 1) / BM;

    // Shared GEMM1
    {
        static bool cfg_s1 = false;
        constexpr size_t smem1 = STAGES * (BM * 128 + 2 * BN1_OUT * 128);
        auto k1 = moe_gemm1_silu_mul_kernel<BM, BN1_OUT, BK, STAGES, WARPS_M, WARPS_N, true>;
        ensure_smem_once(cfg_s1, k1, smem1);
        int grid1 = m_tiles * (I / BN1_OUT);
        k1<<<grid1, NTHREADS, smem1, stream>>>(
            x_ptr, w1_s_ptr, h_s_ptr,
            nullptr, nullptr, nullptr, nullptr,
            T, 1, H, I, s_idx
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }

    // Shared GEMM2
    {
        static bool cfg_s2_last = false;
        static bool cfg_s2_mid = false;
        constexpr size_t smem2_pipe = STAGES * (BM * 128 + BN2 * 128);
        constexpr size_t smem2_epi = BM * (BN2 + 8) * sizeof(float);
        constexpr size_t smem2 = smem2_pipe > smem2_epi ? smem2_pipe : smem2_epi;
        int grid2 = m_tiles * (H / BN2);
        if (is_last) {
            auto k2 = moe_gemm2_kernel<BM, BN2, BK, STAGES, WARPS_M, WARPS_N, 2>;
            ensure_smem_once(cfg_s2_last, k2, smem2);
            k2<<<grid2, NTHREADS, smem2, stream>>>(
                h_s_ptr, w2_s_ptr, out_fp32_ptr, out_bf16_ptr,
                nullptr, nullptr, nullptr, nullptr, nullptr,
                T, 1, H, I, s_idx
            );
        } else {
            auto k2 = moe_gemm2_kernel<BM, BN2, BK, STAGES, WARPS_M, WARPS_N, 1>;
            ensure_smem_once(cfg_s2_mid, k2, smem2);
            k2<<<grid2, NTHREADS, smem2, stream>>>(
                h_s_ptr, w2_s_ptr, out_fp32_ptr, nullptr,
                nullptr, nullptr, nullptr, nullptr, nullptr,
                T, 1, H, I, s_idx
            );
        }
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }
}

void launch_grouped_moe(
    torch::Tensor x,
    torch::Tensor expert_ids,
    torch::Tensor expert_weights,
    torch::Tensor w1_routed,
    torch::Tensor w2_routed,
    torch::Tensor w1_shared,
    torch::Tensor w2_shared,
    torch::Tensor h_routed,
    torch::Tensor h_shared,
    torch::Tensor out_fp32,
    torch::Tensor out_bf16,
    torch::Tensor expert_counts,
    torch::Tensor expert_offsets,
    torch::Tensor write_cursors,
    torch::Tensor expert_tile_prefix,
    torch::Tensor m_tile_to_expert,
    torch::Tensor sorted_token_ids,
    torch::Tensor sorted_weights,
    int bm_routed
) {
    const int T = static_cast<int>(x.size(0));
    const int H = static_cast<int>(x.size(1));
    const int E = static_cast<int>(w1_routed.size(0));
    const int I = static_cast<int>(w2_routed.size(2));
    const int top_k = static_cast<int>(expert_ids.size(1));
    const int n_shared = static_cast<int>(w1_shared.size(0));
    const int M_total = T * top_k;
    const int max_m_tiles = (M_total + bm_routed - 1) / bm_routed + E;

    cudaStream_t stream = c10::cuda::getCurrentCUDAStream();

    // 1. Zero expert_counts and out_fp32
    C10_CUDA_CHECK(cudaMemsetAsync(expert_counts.data_ptr(), 0, (size_t)E * sizeof(int32_t), stream));
    C10_CUDA_CHECK(cudaMemsetAsync(out_fp32.data_ptr(), 0, (size_t)T * (size_t)H * sizeof(float), stream));

    // 2. Histogram expert counts
    {
        int blocks = min(256, (M_total + 255) / 256);
        moe_count_experts_kernel<<<blocks, 256, (size_t)E * sizeof(int32_t), stream>>>(
            expert_ids.data_ptr<int64_t>(),
            expert_counts.data_ptr<int32_t>(),
            M_total, E
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }

    // 3. Prefix sum + tile lookup table
    moe_build_offsets_and_tiles_kernel<<<1, 1, 0, stream>>>(
        expert_counts.data_ptr<int32_t>(),
        expert_offsets.data_ptr<int32_t>(),
        write_cursors.data_ptr<int32_t>(),
        expert_tile_prefix.data_ptr<int32_t>(),
        m_tile_to_expert.data_ptr<int32_t>(),
        E, bm_routed
    );
    C10_CUDA_KERNEL_LAUNCH_CHECK();

    // 4. Scatter sorted token_ids and weights
    {
        int blocks = min(256, (M_total + 255) / 256);
        moe_scatter_tokens_kernel<<<blocks, 256, 0, stream>>>(
            expert_ids.data_ptr<int64_t>(),
            reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
            write_cursors.data_ptr<int32_t>(),
            sorted_token_ids.data_ptr<int32_t>(),
            sorted_weights.data_ptr<float>(),
            M_total, top_k
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }

    const __nv_bfloat16* x_ptr = reinterpret_cast<const __nv_bfloat16*>(x.data_ptr());
    const __nv_bfloat16* w1_r_ptr = reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr());
    const __nv_bfloat16* w2_r_ptr = reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr());
    __nv_bfloat16* h_r_ptr = reinterpret_cast<__nv_bfloat16*>(h_routed.data_ptr());
    float* out_f32_ptr = out_fp32.data_ptr<float>();
    __nv_bfloat16* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr());

    // 5. Routed Experts (GEMM1 fused SiLU*mul -> GEMM2 fused weight-scale + v4.f32 scatter-add)
    if (bm_routed == 32) {
        run_routed_grouped_impl<32, 4, 2, 4>(
            x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr,
            expert_offsets.data_ptr<int32_t>(),
            expert_tile_prefix.data_ptr<int32_t>(),
            m_tile_to_expert.data_ptr<int32_t>(),
            sorted_token_ids.data_ptr<int32_t>(),
            sorted_weights.data_ptr<float>(),
            T, E, H, I, max_m_tiles, stream
        );
    } else if (bm_routed == 64) {
        run_routed_grouped_impl<64, 4, 2, 4>(
            x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr,
            expert_offsets.data_ptr<int32_t>(),
            expert_tile_prefix.data_ptr<int32_t>(),
            m_tile_to_expert.data_ptr<int32_t>(),
            sorted_token_ids.data_ptr<int32_t>(),
            sorted_weights.data_ptr<float>(),
            T, E, H, I, max_m_tiles, stream
        );
    } else {
        run_routed_grouped_impl<128, 3, 4, 2>(
            x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr,
            expert_offsets.data_ptr<int32_t>(),
            expert_tile_prefix.data_ptr<int32_t>(),
            m_tile_to_expert.data_ptr<int32_t>(),
            sorted_token_ids.data_ptr<int32_t>(),
            sorted_weights.data_ptr<float>(),
            T, E, H, I, max_m_tiles, stream
        );
    }

    // 6. Shared Expert(s) — last shared expert fuses add(out_fp32) + cast(bf16) -> out_bf16!
    if (n_shared > 0) {
        const __nv_bfloat16* w1_s_ptr = reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr());
        const __nv_bfloat16* w2_s_ptr = reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr());
        __nv_bfloat16* h_s_ptr = reinterpret_cast<__nv_bfloat16*>(h_shared.data_ptr());
        for (int s = 0; s < n_shared; ++s) {
            bool is_last = (s == n_shared - 1);
            if (T <= 32) {
                run_shared_impl<32, 4, 2, 4>(
                    x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr,
                    T, H, I, s, is_last, stream
                );
            } else if (T <= 96) {
                run_shared_impl<64, 4, 2, 4>(
                    x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr,
                    T, H, I, s, is_last, stream
                );
            } else {
                run_shared_impl<128, 3, 4, 2>(
                    x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr,
                    T, H, I, s, is_last, stream
                );
            }
        }
    } else {
        size_t n_vec4 = ((size_t)T * (size_t)H) / 4;
        int blocks = static_cast<int>(min((size_t)1024, (n_vec4 + 255) / 256));
        cast_f32_to_bf16_kernel<<<blocks, 256, 0, stream>>>(
            out_f32_ptr, out_bf16_ptr, n_vec4
        );
        C10_CUDA_KERNEL_LAUNCH_CHECK();
    }
}
"""

_CPP_DECLS = """
void launch_decode_moe(
    torch::Tensor x,
    torch::Tensor expert_ids,
    torch::Tensor expert_weights,
    torch::Tensor w1_routed,
    torch::Tensor w2_routed,
    torch::Tensor w1_shared,
    torch::Tensor w2_shared,
    torch::Tensor h_buf,
    torch::Tensor out
);

void launch_grouped_moe(
    torch::Tensor x,
    torch::Tensor expert_ids,
    torch::Tensor expert_weights,
    torch::Tensor w1_routed,
    torch::Tensor w2_routed,
    torch::Tensor w1_shared,
    torch::Tensor w2_shared,
    torch::Tensor h_routed,
    torch::Tensor h_shared,
    torch::Tensor out_fp32,
    torch::Tensor out_bf16,
    torch::Tensor expert_counts,
    torch::Tensor expert_offsets,
    torch::Tensor write_cursors,
    torch::Tensor expert_tile_prefix,
    torch::Tensor m_tile_to_expert,
    torch::Tensor sorted_token_ids,
    torch::Tensor sorted_weights,
    int bm_routed
);
"""

_EXT = None


def _get_ext():
    global _EXT
    if _EXT is None:
        _EXT = load_inline(
            name="glm52_fused_moe_sm120_v1",
            cpp_sources=_CPP_DECLS,
            cuda_sources=_CUDA_SRC,
            functions=["launch_decode_moe", "launch_grouped_moe"],
            extra_cuda_cflags=["-O3", "-arch=sm_120a", "--expt-relaxed-constexpr"],
            verbose=False,
        )
    return _EXT


class _Workspace:
    """Reusable GPU scratch buffers to avoid per-forward allocations."""

    def __init__(self):
        self.device = None
        self.h_decode = None
        self.h_routed = None
        self.h_shared = None
        self.out_fp32 = None
        self.expert_counts = None
        self.expert_offsets = None
        self.write_cursors = None
        self.expert_tile_prefix = None
        self.m_tile_to_expert = None
        self.sorted_token_ids = None
        self.sorted_weights = None

    def ensure_decode(self, device, T: int, total_slots: int, I: int):
        need = T * total_slots * I
        if self.device != device or self.h_decode is None or self.h_decode.numel() < need:
            self.device = device
            self.h_decode = torch.empty(need, dtype=torch.bfloat16, device=device)
        return self.h_decode

    def ensure_grouped(self, device, T: int, E: int, top_k: int, H: int, I: int, bm: int):
        M_total = T * top_k
        max_m_tiles = (M_total + bm - 1) // bm + E
        if self.device != device:
            self.device = device
            self.h_routed = None
            self.h_shared = None
            self.out_fp32 = None
            self.expert_counts = None
            self.expert_offsets = None
            self.write_cursors = None
            self.expert_tile_prefix = None
            self.m_tile_to_expert = None
            self.sorted_token_ids = None
            self.sorted_weights = None

        if self.h_routed is None or self.h_routed.numel() < M_total * I:
            self.h_routed = torch.empty((M_total, I), dtype=torch.bfloat16, device=device)
        if self.h_shared is None or self.h_shared.numel() < T * I:
            self.h_shared = torch.empty((T, I), dtype=torch.bfloat16, device=device)
        if self.out_fp32 is None or self.out_fp32.numel() < T * H:
            self.out_fp32 = torch.empty((T, H), dtype=torch.float32, device=device)
        if self.expert_counts is None or self.expert_counts.numel() < E:
            self.expert_counts = torch.empty(E, dtype=torch.int32, device=device)
        if self.expert_offsets is None or self.expert_offsets.numel() < E + 1:
            self.expert_offsets = torch.empty(E + 1, dtype=torch.int32, device=device)
        if self.write_cursors is None or self.write_cursors.numel() < E:
            self.write_cursors = torch.empty(E, dtype=torch.int32, device=device)
        if self.expert_tile_prefix is None or self.expert_tile_prefix.numel() < E + 1:
            self.expert_tile_prefix = torch.empty(E + 1, dtype=torch.int32, device=device)
        if self.m_tile_to_expert is None or self.m_tile_to_expert.numel() < max_m_tiles:
            self.m_tile_to_expert = torch.empty(max_m_tiles, dtype=torch.int32, device=device)
        if self.sorted_token_ids is None or self.sorted_token_ids.numel() < M_total:
            self.sorted_token_ids = torch.empty(M_total, dtype=torch.int32, device=device)
        if self.sorted_weights is None or self.sorted_weights.numel() < M_total:
            self.sorted_weights = torch.empty(M_total, dtype=torch.float32, device=device)


_WORKSPACES: dict[int, _Workspace] = {}


def _get_workspace(device: torch.device) -> _Workspace:
    idx = device.index if device.index is not None else 0
    ws = _WORKSPACES.get(idx)
    if ws is None:
        ws = _Workspace()
        _WORKSPACES[idx] = ws
    return ws


class Model(nn.Module):
    def __init__(
        self,
        T: int,
        E: int,
        top_k: int,
        n_shared: int,
        H: int,
        I: int,
    ):
        super().__init__()
        self.T, self.E, self.top_k = T, E, top_k
        self.n_shared, self.H, self.I = n_shared, H, I
        self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
        self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
        self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
        self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
        for p in self.parameters():
            nn.init.normal_(p, std=0.02)
        # Pre-compile extension at init time (outside timed forward)
        _get_ext()

    def forward(
        self,
        x: torch.Tensor,
        expert_ids: torch.Tensor,
        expert_weights: torch.Tensor,
    ) -> torch.Tensor:
        T, H = x.shape
        E = self.E
        top_k = self.top_k
        n_shared = self.n_shared
        I = self.I

        ext = _get_ext()
        ws = _get_workspace(x.device)
        out = torch.empty((T, H), dtype=torch.bfloat16, device=x.device)

        x_c = x.contiguous()
        ids_c = expert_ids.contiguous()
        wts_c = expert_weights.contiguous()

        if T <= 4 and (H % 8 == 0) and (I % 8 == 0):
            h_buf = ws.ensure_decode(x.device, T, top_k + n_shared, I)
            ext.launch_decode_moe(
                x_c, ids_c, wts_c,
                self.w1_routed, self.w2_routed,
                self.w1_shared, self.w2_shared,
                h_buf, out,
            )
            return out

        # Choose routed BM based on average tokens per expert
        m_avg = (T * top_k) / max(E, 1)
        bm_override = os.environ.get("MOE_BM_OVERRIDE")
        if bm_override is not None:
            bm = int(bm_override)
        elif m_avg <= 20.0:
            bm = 32
        elif m_avg <= 160.0:
            bm = 64
        else:
            bm = 64

        ws.ensure_grouped(x.device, T, E, top_k, H, I, bm)
        ext.launch_grouped_moe(
            x_c, ids_c, wts_c,
            self.w1_routed, self.w2_routed,
            self.w1_shared, self.w2_shared,
            ws.h_routed, ws.h_shared, ws.out_fp32, out,
            ws.expert_counts, ws.expert_offsets, ws.write_cursors,
            ws.expert_tile_prefix, ws.m_tile_to_expert,
            ws.sorted_token_ids, ws.sorted_weights,
            bm,
        )
        return out

20261118_150445_agy_gemini-4-neon-xhigh_01_glm52_fused_moe