KernelBench cuda · RTX PRO 6000
GLM-5.3 Fused MoE Gemini 4 Neon (XHigh)
manually audited: clean
Gemini 4 Neon (XHigh) wrote the GLM-5.2 MoE layer in hand-written SM120 CUDA C++ with inline PTX (cp.async.cg, ldmatrix.x4, bf16 mma.sync m16n8k16, red.relaxed.gpu.global.add.v4.f32) and a 2-kernel streaming GEMV path for small batches (T <= 4). Notably, 0.1010 is its uniterated V1 first draft: after passing an initial GPU smoke test, a sibling cell's deadlock held the shared cuda GPU lock for 2h42m, so this session exited cleanly at 5:29 pm without ever timing or optimizing on GPU.
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(37.9% · 36.5% · 0.3% · 41.8% · 5.7% · 10.8%) = 10.1%
Kernel source (redacted)
"""Fused GLM-5.2 MoE layer in CUDA C++/PTX for RTX PRO 6000 (SM120 Blackwell).
Architecture:
- Small-batch decode path (T <= 4):
2-kernel fused GEMV covering all active routed + shared experts in one shot:
1) moe_decode_gemv1_silu_mul: vectorized 128-bit streaming gate+up+SiLU*mul -> h_buf (fp32)
2) moe_decode_gemv2_reduce: vectorized 128-bit down-proj + weighted sum -> out (bf16)
- Grouped prefill/batch path (T > 4):
0) Zero-sync GPU routing histogram + prefix sum + scatter + M-tile lookup table
1) Fused Grouped GEMM1 (SM120 PTX mma.sync.aligned.m16n8k16 + cp.async.cg + 128B XOR swizzle
+ ldmatrix.x4): gathers x[token_id] on-the-fly from L2, computes gate & up simultaneously,
applies SiLU(gate)*up in FP32 registers, writes packed h (bf16) via SMEM coalescing.
2) Fused Grouped GEMM2: computes h @ w2^T, scales by routing weight in FP32, and scatters
directly into out_fp32 via 128-bit vectorized red.relaxed.gpu.global.add.v4.f32.
3) Fused Shared Expert GEMM1 + GEMM2: computes shared expert(s), fusing the final addition
with out_fp32 and FP32->BF16 conversion into out (bf16).
"""
from __future__ import annotations
import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
_CUDA_SRC = r"""
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
#include <torch/extension.h>
#include <c10/cuda/CUDAStream.h>
#include <c10/cuda/CUDAException.h>
// ============================================================================
// PTX & Low-Level Device Primitives (SM120 Blackwell)
// ============================================================================
__device__ __forceinline__ uint32_t smem_u32(const void* ptr) {
return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}
__device__ __forceinline__ void cp_async_cg_16(uint32_t dst_smem, const void* src_gmem) {
asm volatile(
"cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\n"
:: "r"(dst_smem), "l"(src_gmem)
);
}
__device__ __forceinline__ void cp_async_cg_16_zfill(uint32_t dst_smem, const void* src_gmem, int src_bytes) {
asm volatile(
"cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n"
:: "r"(dst_smem), "l"(src_gmem), "r"(src_bytes)
);
}
__device__ __forceinline__ void cp_async_commit() {
asm volatile("cp.async.commit_group;\n" ::: "memory");
}
template <int N>
__device__ __forceinline__ void cp_async_wait() {
asm volatile("cp.async.wait_group %0;\n" :: "n"(N) : "memory");
}
__device__ __forceinline__ void ldmatrix_x4(
uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t smem_addr
) {
asm volatile(
"ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
: "r"(smem_addr)
);
}
__device__ __forceinline__ void mma_m16n8k16_bf16(
float& d0, float& d1, float& d2, float& d3,
uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
uint32_t b0, uint32_t b1
) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
: "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3)
: "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1)
);
}
__device__ __forceinline__ void red_global_add_v4_f32(float* addr, float4 v) {
asm volatile(
"red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n"
:: "l"(addr), "f"(v.x), "f"(v.y), "f"(v.z), "f"(v.w)
);
}
__device__ __forceinline__ float silu_f32(float x) {
return x / (1.0f + expf(-x));
}
__device__ __forceinline__ float warp_reduce_sum(float val) {
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
val += __shfl_down_sync(0xffffffff, val, offset);
}
return val;
}
__device__ __forceinline__ float dot8_bf16_f32(uint4 a, uint4 b) {
const __nv_bfloat162* a2 = reinterpret_cast<const __nv_bfloat162*>(&a);
const __nv_bfloat162* b2 = reinterpret_cast<const __nv_bfloat162*>(&b);
float acc = 0.0f;
#pragma unroll
for (int i = 0; i < 4; ++i) {
float2 fa = __bfloat1622float2(a2[i]);
float2 fb = __bfloat1622float2(b2[i]);
acc = fmaf(fa.x, fb.x, acc);
acc = fmaf(fa.y, fb.y, acc);
}
return acc;
}
// ============================================================================
// REGIME A: Ultra-Fast Fused Decode GEMV (T <= 4, e.g. T = 1)
// ============================================================================
// Kernel D1: Computes h_buf[t, slot, col] = silu(dot(x[t], gate)) * dot(x[t], up)
// Grid: x = I / WARPS_PER_BLOCK, y = total_slots (top_k + n_shared), z = T
// Block: WARPS_PER_BLOCK * 32 threads (default 4 warps = 128 threads)
template <int WARPS_PER_BLOCK>
__global__ void moe_decode_gemv1_silu_mul_kernel(
const __nv_bfloat16* __restrict__ x, // (T, H)
const int64_t* __restrict__ expert_ids, // (T, top_k)
const __nv_bfloat16* __restrict__ w1_routed, // (E, 2*I, H)
const __nv_bfloat16* __restrict__ w1_shared, // (n_shared, 2*I, H)
__nv_bfloat16* __restrict__ h_buf, // (T, top_k + n_shared, I)
int T, int E, int top_k, int n_shared, int H, int I
) {
extern __shared__ __nv_bfloat16 s_x[]; // H bf16 (8 KB for H=4096)
const int t = blockIdx.z;
const int slot = blockIdx.y;
const int warp_id = threadIdx.x >> 5;
const int lane = threadIdx.x & 31;
const int col = blockIdx.x * WARPS_PER_BLOCK + warp_id;
// Cooperatively load x[t, 0..H-1] into SMEM using 128-bit loads
const int H_u4 = H >> 3;
const uint4* x_u4 = reinterpret_cast<const uint4*>(x + (size_t)t * H);
uint4* sx_u4 = reinterpret_cast<uint4*>(s_x);
for (int i = threadIdx.x; i < H_u4; i += WARPS_PER_BLOCK * 32) {
sx_u4[i] = x_u4[i];
}
__syncthreads();
if (col >= I) return;
const __nv_bfloat16* w1_base;
if (slot < top_k) {
int64_t e = expert_ids[(size_t)t * top_k + slot];
w1_base = w1_routed + (size_t)e * (size_t)(2 * I) * (size_t)H;
} else {
int s = slot - top_k;
w1_base = w1_shared + (size_t)s * (size_t)(2 * I) * (size_t)H;
}
const uint4* wg_u4 = reinterpret_cast<const uint4*>(w1_base + (size_t)col * H);
const uint4* wu_u4 = reinterpret_cast<const uint4*>(w1_base + (size_t)(I + col) * H);
float acc_g0 = 0.0f, acc_g1 = 0.0f;
float acc_u0 = 0.0f, acc_u1 = 0.0f;
// 2x unrolled streaming 128-bit loads along H
int i = lane;
for (; i + 32 < H_u4; i += 64) {
uint4 x0 = sx_u4[i];
uint4 x1 = sx_u4[i + 32];
uint4 g0 = __ldcs(&wg_u4[i]);
uint4 u0 = __ldcs(&wu_u4[i]);
uint4 g1 = __ldcs(&wg_u4[i + 32]);
uint4 u1 = __ldcs(&wu_u4[i + 32]);
acc_g0 += dot8_bf16_f32(x0, g0);
acc_u0 += dot8_bf16_f32(x0, u0);
acc_g1 += dot8_bf16_f32(x1, g1);
acc_u1 += dot8_bf16_f32(x1, u1);
}
for (; i < H_u4; i += 32) {
uint4 x0 = sx_u4[i];
uint4 g0 = __ldcs(&wg_u4[i]);
uint4 u0 = __ldcs(&wu_u4[i]);
acc_g0 += dot8_bf16_f32(x0, g0);
acc_u0 += dot8_bf16_f32(x0, u0);
}
float g_sum = warp_reduce_sum(acc_g0 + acc_g1);
float u_sum = warp_reduce_sum(acc_u0 + acc_u1);
if (lane == 0) {
int total_slots = top_k + n_shared;
float h = silu_f32(g_sum) * u_sum;
h_buf[((size_t)t * total_slots + slot) * I + col] = __float2bfloat16(h);
}
}
// Kernel D2: Computes out[t, col] = sum_slot w_slot * dot(h_buf[t, slot, :], w2_slot[col, :])
// Grid: x = H / WARPS_PER_BLOCK, z = T
// Block: WARPS_PER_BLOCK * 32 threads (2 warps = 64 threads -> 2048 CTAs for H=4096 -> 10.9 waves on 188 SMs!)
template <int WARPS_PER_BLOCK>
__global__ void moe_decode_gemv2_reduce_kernel(
const __nv_bfloat16* __restrict__ h_buf, // (T, total_slots, I)
const int64_t* __restrict__ expert_ids, // (T, top_k)
const __nv_bfloat16* __restrict__ expert_weights, // (T, top_k)
const __nv_bfloat16* __restrict__ w2_routed, // (E, H, I)
const __nv_bfloat16* __restrict__ w2_shared, // (n_shared, H, I)
__nv_bfloat16* __restrict__ out, // (T, H)
int T, int E, int top_k, int n_shared, int H, int I
) {
const int t = blockIdx.z;
const int warp_id = threadIdx.x >> 5;
const int lane = threadIdx.x & 31;
const int col = blockIdx.x * WARPS_PER_BLOCK + warp_id;
if (col >= H) return;
const int total_slots = top_k + n_shared;
const int I_u4 = I >> 3;
float total_acc = 0.0f;
for (int slot = 0; slot < total_slots; ++slot) {
const __nv_bfloat16* w2_row;
float weight;
if (slot < top_k) {
int64_t e = expert_ids[(size_t)t * top_k + slot];
weight = __bfloat162float(expert_weights[(size_t)t * top_k + slot]);
w2_row = w2_routed + ((size_t)e * (size_t)H + (size_t)col) * (size_t)I;
} else {
int s = slot - top_k;
weight = 1.0f;
w2_row = w2_shared + ((size_t)s * (size_t)H + (size_t)col) * (size_t)I;
}
const uint4* h_u4 = reinterpret_cast<const uint4*>(
h_buf + ((size_t)t * total_slots + slot) * I
);
const uint4* w_u4 = reinterpret_cast<const uint4*>(w2_row);
float acc0 = 0.0f, acc1 = 0.0f;
int i = lane;
#pragma unroll 4
for (; i + 32 < I_u4; i += 64) {
uint4 hv0 = __ldg(&h_u4[i]);
uint4 wv0 = __ldcs(&w_u4[i]);
uint4 hv1 = __ldg(&h_u4[i + 32]);
uint4 wv1 = __ldcs(&w_u4[i + 32]);
acc0 += dot8_bf16_f32(hv0, wv0);
acc1 += dot8_bf16_f32(hv1, wv1);
}
for (; i < I_u4; i += 32) {
uint4 hv0 = __ldg(&h_u4[i]);
uint4 wv0 = __ldcs(&w_u4[i]);
acc0 += dot8_bf16_f32(hv0, wv0);
}
total_acc += weight * (acc0 + acc1);
}
float final_sum = warp_reduce_sum(total_acc);
if (lane == 0) {
out[(size_t)t * H + col] = __float2bfloat16(final_sum);
}
}
// ============================================================================
// REGIME B: Zero-Sync Routing Sort + M-Tile Table Construction
// ============================================================================
__global__ void moe_count_experts_kernel(
const int64_t* __restrict__ expert_ids,
int32_t* __restrict__ expert_counts,
int M_total, int E
) {
extern __shared__ int32_t s_counts[];
for (int i = threadIdx.x; i < E; i += blockDim.x) {
s_counts[i] = 0;
}
__syncthreads();
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
for (; idx < M_total; idx += stride) {
int e = static_cast<int>(expert_ids[idx]);
atomicAdd(&s_counts[e], 1);
}
__syncthreads();
for (int i = threadIdx.x; i < E; i += blockDim.x) {
int c = s_counts[i];
if (c > 0) {
atomicAdd(&expert_counts[i], c);
}
}
}
// Single-block kernel (up to E <= 1024) computing prefix sums & tile table
__global__ void moe_build_offsets_and_tiles_kernel(
const int32_t* __restrict__ expert_counts,
int32_t* __restrict__ expert_offsets, // (E + 1)
int32_t* __restrict__ write_cursors, // (E)
int32_t* __restrict__ expert_tile_prefix, // (E + 1)
int32_t* __restrict__ m_tile_to_expert, // (max_m_tiles)
int E, int BM
) {
// Simple warp/thread scan (E=256 takes < 2 microseconds)
if (threadIdx.x == 0) {
int off = 0;
int tile_off = 0;
for (int e = 0; e < E; ++e) {
int c = expert_counts[e];
expert_offsets[e] = off;
write_cursors[e] = off;
expert_tile_prefix[e] = tile_off;
int mt = (c + BM - 1) / BM;
for (int m = 0; m < mt; ++m) {
m_tile_to_expert[tile_off + m] = e;
}
off += c;
tile_off += mt;
}
expert_offsets[E] = off;
expert_tile_prefix[E] = tile_off;
}
}
__global__ void moe_scatter_tokens_kernel(
const int64_t* __restrict__ expert_ids,
const __nv_bfloat16* __restrict__ expert_weights,
int32_t* __restrict__ write_cursors,
int32_t* __restrict__ sorted_token_ids,
float* __restrict__ sorted_weights,
int M_total, int top_k
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
for (; idx < M_total; idx += stride) {
int e = static_cast<int>(expert_ids[idx]);
int pos = atomicAdd(&write_cursors[e], 1);
sorted_token_ids[pos] = idx / top_k;
sorted_weights[pos] = __bfloat162float(expert_weights[idx]);
}
}
// ============================================================================
// REGIME B: Fused Grouped GEMM1 (Gather X + Gate + Up + SiLU*Mul -> h)
// ============================================================================
// Tile: BM x BN_OUT (BN_OUT = 64 -> 64 gate + 64 up = 128 weight rows), BK = 64
// 8 warps (256 threads):
// - BM = 32: WARPS_M = 2 (WM = 16, 1 tile), WARPS_N = 4 (WN = 16, 1 tile of 16)
// - BM = 64: WARPS_M = 2 (WM = 32, 2 tiles), WARPS_N = 4 (WN = 16, 1 tile of 16)
// - BM = 128: WARPS_M = 4 (WM = 32, 2 tiles), WARPS_N = 2 (WN = 32, 2 tiles of 16)
template <int BM, int BN_OUT, int BK, int STAGES, int WARPS_M, int WARPS_N, bool IS_SHARED>
__global__ __launch_bounds__(WARPS_M * WARPS_N * 32, 1)
void moe_gemm1_silu_mul_kernel(
const __nv_bfloat16* __restrict__ x, // (T, H)
const __nv_bfloat16* __restrict__ w1, // (E or n_shared, 2*I, H)
__nv_bfloat16* __restrict__ h_out, // (M_total or T, I)
const int32_t* __restrict__ expert_offsets, // (E + 1) [routed]
const int32_t* __restrict__ expert_tile_prefix, // (E + 1) [routed]
const int32_t* __restrict__ m_tile_to_expert, // (max_m_tiles) [routed]
const int32_t* __restrict__ sorted_token_ids, // (M_total) [routed]
int T, int E, int H, int I, int shared_idx
) {
static_assert(BK == 64, "BK must be 64 for 128B swizzle");
static_assert(BN_OUT == 64, "BN_OUT must be 64");
constexpr int NWARPS = WARPS_M * WARPS_N;
constexpr int NTHREADS = NWARPS * 32;
constexpr int WM = BM / WARPS_M;
constexpr int WN = BN_OUT / WARPS_N;
constexpr int M_TILES = WM / 16;
constexpr int N_TILES = WN / 16; // each N_TILE is 16 cols = two m16n8k16 MMAs
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int warp_m = warp_id / WARPS_N;
const int warp_n = warp_id % WARPS_N;
const int n_tiles_total = I / BN_OUT; // 32 for I=2048
int e, row_base, m_valid, n_blk;
if constexpr (IS_SHARED) {
const int work_id = blockIdx.x;
const int m_tiles_shared = (T + BM - 1) / BM;
if (work_id >= m_tiles_shared * n_tiles_total) return;
e = shared_idx;
n_blk = work_id / m_tiles_shared;
int m_blk = work_id - n_blk * m_tiles_shared;
row_base = m_blk * BM;
m_valid = min(BM, T - row_base);
} else {
const int work_id = blockIdx.x;
const int total_m_tiles = expert_tile_prefix[E];
if (work_id >= total_m_tiles * n_tiles_total) return;
int m_tile_approx = work_id / n_tiles_total;
e = m_tile_to_expert[m_tile_approx];
int p0 = expert_tile_prefix[e];
int p1 = expert_tile_prefix[e + 1];
int m_tiles_e = p1 - p0;
int local = work_id - p0 * n_tiles_total;
n_blk = local / m_tiles_e;
int m_blk = local - n_blk * m_tiles_e;
int off0 = expert_offsets[e];
int off1 = expert_offsets[e + 1];
row_base = off0 + m_blk * BM;
m_valid = min(BM, off1 - row_base);
}
const int col_base = n_blk * BN_OUT;
// Shared memory layout:
// int32 s_tok[BM] (at the end or separate? Put s_tok in small static smem!)
// Dynamic SMEM (16B aligned):
// For each stage s in [0, STAGES):
// A[s]: BM * 64 bf16 = BM * 128 B
// B_gate[s]: BN_OUT * 64 bf16 = 8192 B
// B_up[s]: BN_OUT * 64 bf16 = 8192 B
// Epilogue reuses dynamic SMEM: s_h[BM][BN_OUT + 8] bf16
__shared__ int32_t s_tok[BM];
if constexpr (!IS_SHARED) {
if (tid < BM) {
s_tok[tid] = (tid < m_valid) ? sorted_token_ids[row_base + tid] : 0;
}
__syncthreads();
}
extern __shared__ char smem_raw[];
constexpr uint32_t A_STAGE_BYTES = BM * 128;
constexpr uint32_t B_STAGE_BYTES = BN_OUT * 128; // 8192 B
constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_STAGE_BYTES;
const uint32_t smem_base = smem_u32(smem_raw);
const __nv_bfloat16* w1_e = w1 + (size_t)e * (size_t)(2 * I) * (size_t)H;
const __nv_bfloat16* wg_base = w1_e + (size_t)col_base * (size_t)H;
const __nv_bfloat16* wu_base = w1_e + (size_t)(I + col_base) * (size_t)H;
// Accumulators in FP32
float acc_g[M_TILES][N_TILES * 2][4];
float acc_u[M_TILES][N_TILES * 2][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
#pragma unroll
for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
acc_g[mi][n8][0] = 0.0f; acc_g[mi][n8][1] = 0.0f;
acc_g[mi][n8][2] = 0.0f; acc_g[mi][n8][3] = 0.0f;
acc_u[mi][n8][0] = 0.0f; acc_u[mi][n8][1] = 0.0f;
acc_u[mi][n8][2] = 0.0f; acc_u[mi][n8][3] = 0.0f;
}
}
// Helper lambda to issue async loads for a K-tile `kt` into `stage`
auto load_stage = [&](int kt, int stage) {
const int k_off = kt * BK;
const uint32_t st_base = smem_base + stage * STAGE_BYTES;
const uint32_t a_base = st_base;
const uint32_t bg_base = st_base + A_STAGE_BYTES;
const uint32_t bu_base = bg_base + B_STAGE_BYTES;
// Load A: BM * 8 chunks of 16B
constexpr int A_CHUNKS = BM * 8;
#pragma unroll
for (int idx = tid; idx < A_CHUNKS; idx += NTHREADS) {
int r = idx >> 3;
int c8 = idx & 7;
int swiz = c8 ^ (r & 7);
uint32_t dst = a_base + (r * 8 + swiz) * 16;
int valid_bytes = (r < m_valid) ? 16 : 0;
const __nv_bfloat16* src;
if constexpr (IS_SHARED) {
int r_clamped = (r < m_valid) ? (row_base + r) : row_base;
src = x + (size_t)r_clamped * H + k_off + c8 * 8;
} else {
int tok = s_tok[r];
src = x + (size_t)tok * H + k_off + c8 * 8;
}
cp_async_cg_16_zfill(dst, src, valid_bytes);
}
// Load B_gate & B_up: BN_OUT * 8 = 512 chunks each
constexpr int B_CHUNKS = BN_OUT * 8;
#pragma unroll
for (int idx = tid; idx < B_CHUNKS; idx += NTHREADS) {
int r = idx >> 3;
int c8 = idx & 7;
int swiz = c8 ^ (r & 7);
uint32_t off = (r * 8 + swiz) * 16;
cp_async_cg_16(bg_base + off, wg_base + (size_t)r * H + k_off + c8 * 8);
cp_async_cg_16(bu_base + off, wu_base + (size_t)r * H + k_off + c8 * 8);
}
};
const int num_k_tiles = H / BK;
// Prologue: fill STAGES - 1 buffers
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < num_k_tiles) {
load_stage(s, s);
}
cp_async_commit();
}
const int lane_r16 = lane & 15;
const int lane_r7 = lane & 7;
const int lane_c_bit = lane >> 4; // 0 or 1
const int warp_m_base = warp_m * WM;
const int warp_n_base = warp_n * WN;
// Main K-loop
for (int kt = 0; kt < num_k_tiles; ++kt) {
const int read_stage = kt % STAGES;
const int write_stage = (kt + STAGES - 1) % STAGES;
const int next_kt = kt + STAGES - 1;
cp_async_wait<STAGES - 2>();
__syncthreads();
// Issue next async stage immediately
if (next_kt < num_k_tiles) {
load_stage(next_kt, write_stage);
}
cp_async_commit();
const uint32_t st_base = smem_base + read_stage * STAGE_BYTES;
const uint32_t a_base = st_base;
const uint32_t bg_base = st_base + A_STAGE_BYTES;
const uint32_t bu_base = bg_base + B_STAGE_BYTES;
// Compute 4 K-steps of 16 with register double-buffering on A
uint32_t ra[2][M_TILES][4];
// Prefetch A for ks = 0
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r = warp_m_base + mi * 16 + lane_r16;
int c8 = lane_c_bit;
uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
ldmatrix_x4(ra[0][mi][0], ra[0][mi][1], ra[0][mi][2], ra[0][mi][3], addr);
}
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
const int cur = ks & 1;
const int nxt = cur ^ 1;
// Prefetch A for ks + 1
if (ks < 3) {
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r = warp_m_base + mi * 16 + lane_r16;
int c8 = (ks + 1) * 2 + lane_c_bit;
uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
ldmatrix_x4(ra[nxt][mi][0], ra[nxt][mi][1], ra[nxt][mi][2], ra[nxt][mi][3], addr);
}
}
// Stream B_gate and B_up per N-tile and run MMA
#pragma unroll
for (int ni = 0; ni < N_TILES; ++ni) {
int nb = warp_n_base + ni * 16 + lane_r16;
int c8 = ks * 2 + lane_c_bit;
uint32_t off = (nb * 8 + (c8 ^ lane_r7)) * 16;
uint32_t bg0, bg1, bg2, bg3;
uint32_t bu0, bu1, bu2, bu3;
ldmatrix_x4(bg0, bg1, bg2, bg3, bg_base + off);
ldmatrix_x4(bu0, bu1, bu2, bu3, bu_base + off);
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
mma_m16n8k16_bf16(
acc_g[mi][2 * ni + 0][0], acc_g[mi][2 * ni + 0][1],
acc_g[mi][2 * ni + 0][2], acc_g[mi][2 * ni + 0][3],
ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
bg0, bg2
);
mma_m16n8k16_bf16(
acc_g[mi][2 * ni + 1][0], acc_g[mi][2 * ni + 1][1],
acc_g[mi][2 * ni + 1][2], acc_g[mi][2 * ni + 1][3],
ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
bg1, bg3
);
mma_m16n8k16_bf16(
acc_u[mi][2 * ni + 0][0], acc_u[mi][2 * ni + 0][1],
acc_u[mi][2 * ni + 0][2], acc_u[mi][2 * ni + 0][3],
ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
bu0, bu2
);
mma_m16n8k16_bf16(
acc_u[mi][2 * ni + 1][0], acc_u[mi][2 * ni + 1][1],
acc_u[mi][2 * ni + 1][2], acc_u[mi][2 * ni + 1][3],
ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
bu1, bu3
);
}
}
}
}
// Drain any remaining async ops before reusing SMEM for epilogue
cp_async_wait<0>();
__syncthreads();
// Epilogue: compute SiLU(gate) * up in FP32, stage bf16 to SMEM with +8 padding
constexpr int S_H_STRIDE = BN_OUT + 8; // 72 bf16 = 144B (16B aligned, bank-conflict-free!)
__nv_bfloat16* s_h = reinterpret_cast<__nv_bfloat16*>(smem_raw);
const int g = lane >> 2;
const int t = lane & 3;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r0 = warp_m_base + mi * 16 + g;
int r1 = r0 + 8;
#pragma unroll
for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
int c0 = warp_n_base + n8 * 8 + t * 2;
float h0 = silu_f32(acc_g[mi][n8][0]) * acc_u[mi][n8][0];
float h1 = silu_f32(acc_g[mi][n8][1]) * acc_u[mi][n8][1];
float h2 = silu_f32(acc_g[mi][n8][2]) * acc_u[mi][n8][2];
float h3 = silu_f32(acc_g[mi][n8][3]) * acc_u[mi][n8][3];
*reinterpret_cast<__nv_bfloat162*>(&s_h[r0 * S_H_STRIDE + c0]) = __floats2bfloat162_rn(h0, h1);
*reinterpret_cast<__nv_bfloat162*>(&s_h[r1 * S_H_STRIDE + c0]) = __floats2bfloat162_rn(h2, h3);
}
}
__syncthreads();
// Coalesced 128-bit stores to global h_out[row_base + r, col_base + c]
constexpr int H_CHUNKS = BM * (BN_OUT / 8); // BM * 8
#pragma unroll
for (int idx = tid; idx < H_CHUNKS; idx += NTHREADS) {
int r = idx >> 3;
int c8 = idx & 7;
if (r < m_valid) {
uint4 v = reinterpret_cast<const uint4*>(&s_h[r * S_H_STRIDE])[c8];
reinterpret_cast<uint4*>(h_out + (size_t)(row_base + r) * I + col_base)[c8] = v;
}
}
}
// ============================================================================
// REGIME B: Fused Grouped GEMM2 (Down Proj + Weight Scale + v4.f32 Scatter-Add)
// Also supports Shared Expert (FUSE_ADD_CAST -> direct bf16 output!)
// ============================================================================
// Tile: BM x BN (BN = 128), BK = 64
// 8 warps (256 threads):
// - BM = 32: WARPS_M = 2 (WM = 16, 1 tile), WARPS_N = 4 (WN = 32, 2 tiles of 16)
// - BM = 64: WARPS_M = 2 (WM = 32, 2 tiles), WARPS_N = 4 (WN = 32, 2 tiles of 16)
// - BM = 128: WARPS_M = 4 (WM = 32, 2 tiles), WARPS_N = 2 (WN = 64, 4 tiles of 16)
// EPILOGUE_MODE:
// 0 = Routed scatter-add: scale by sorted_weights[r], red.global.add.v4.f32 -> out_fp32[tok]
// 1 = Shared add to out_fp32: red.global.add.v4.f32 -> out_fp32[row] (for multi-shared s < n_shared-1)
// 2 = Shared final fused add+cast: out_bf16[row] = bf16(acc + out_fp32[row]) (for last shared expert!)
template <int BM, int BN, int BK, int STAGES, int WARPS_M, int WARPS_N, int EPILOGUE_MODE>
__global__ __launch_bounds__(WARPS_M * WARPS_N * 32, 1)
void moe_gemm2_kernel(
const __nv_bfloat16* __restrict__ h_in, // (M_total or T, I)
const __nv_bfloat16* __restrict__ w2, // (E or n_shared, H, I)
float* __restrict__ out_fp32, // (T, H)
__nv_bfloat16* __restrict__ out_bf16, // (T, H) [mode 2]
const int32_t* __restrict__ expert_offsets, // (E + 1) [mode 0]
const int32_t* __restrict__ expert_tile_prefix, // (E + 1) [mode 0]
const int32_t* __restrict__ m_tile_to_expert, // (max_m_tiles) [mode 0]
const int32_t* __restrict__ sorted_token_ids, // (M_total) [mode 0]
const float* __restrict__ sorted_weights, // (M_total) [mode 0]
int T, int E, int H, int I, int shared_idx
) {
static_assert(BK == 64, "BK must be 64");
static_assert(BN == 128, "BN must be 128");
constexpr bool IS_SHARED = (EPILOGUE_MODE != 0);
constexpr int NWARPS = WARPS_M * WARPS_N;
constexpr int NTHREADS = NWARPS * 32;
constexpr int WM = BM / WARPS_M;
constexpr int WN = BN / WARPS_N;
constexpr int M_TILES = WM / 16;
constexpr int N_TILES = WN / 16;
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int warp_m = warp_id / WARPS_N;
const int warp_n = warp_id % WARPS_N;
const int n_tiles_total = H / BN; // 32 for H=4096
int e, row_base, m_valid, n_blk;
if constexpr (IS_SHARED) {
const int work_id = blockIdx.x;
const int m_tiles_shared = (T + BM - 1) / BM;
if (work_id >= m_tiles_shared * n_tiles_total) return;
e = shared_idx;
n_blk = work_id / m_tiles_shared;
int m_blk = work_id - n_blk * m_tiles_shared;
row_base = m_blk * BM;
m_valid = min(BM, T - row_base);
} else {
const int work_id = blockIdx.x;
const int total_m_tiles = expert_tile_prefix[E];
if (work_id >= total_m_tiles * n_tiles_total) return;
int m_tile_approx = work_id / n_tiles_total;
e = m_tile_to_expert[m_tile_approx];
int p0 = expert_tile_prefix[e];
int p1 = expert_tile_prefix[e + 1];
int m_tiles_e = p1 - p0;
int local = work_id - p0 * n_tiles_total;
n_blk = local / m_tiles_e;
int m_blk = local - n_blk * m_tiles_e;
int off0 = expert_offsets[e];
int off1 = expert_offsets[e + 1];
row_base = off0 + m_blk * BM;
m_valid = min(BM, off1 - row_base);
}
const int col_base = n_blk * BN;
__shared__ int32_t s_tok[BM];
__shared__ float s_wt[BM];
if constexpr (!IS_SHARED) {
if (tid < BM) {
if (tid < m_valid) {
s_tok[tid] = sorted_token_ids[row_base + tid];
s_wt[tid] = sorted_weights[row_base + tid];
} else {
s_tok[tid] = 0;
s_wt[tid] = 0.0f;
}
}
__syncthreads();
}
extern __shared__ char smem_raw[];
constexpr uint32_t A_STAGE_BYTES = BM * 128;
constexpr uint32_t B_STAGE_BYTES = BN * 128; // 16384 B
constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES;
const uint32_t smem_base = smem_u32(smem_raw);
const __nv_bfloat16* w2_base = w2 + ((size_t)e * (size_t)H + (size_t)col_base) * (size_t)I;
float acc[M_TILES][N_TILES * 2][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
#pragma unroll
for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
acc[mi][n8][0] = 0.0f; acc[mi][n8][1] = 0.0f;
acc[mi][n8][2] = 0.0f; acc[mi][n8][3] = 0.0f;
}
}
auto load_stage = [&](int kt, int stage) {
const int k_off = kt * BK;
const uint32_t st_base = smem_base + stage * STAGE_BYTES;
const uint32_t a_base = st_base;
const uint32_t b_base = st_base + A_STAGE_BYTES;
// Load A (h_in is contiguous!)
constexpr int A_CHUNKS = BM * 8;
#pragma unroll
for (int idx = tid; idx < A_CHUNKS; idx += NTHREADS) {
int r = idx >> 3;
int c8 = idx & 7;
int swiz = c8 ^ (r & 7);
uint32_t dst = a_base + (r * 8 + swiz) * 16;
int valid_bytes = (r < m_valid) ? 16 : 0;
int r_clamped = (r < m_valid) ? (row_base + r) : row_base;
const __nv_bfloat16* src = h_in + (size_t)r_clamped * I + k_off + c8 * 8;
cp_async_cg_16_zfill(dst, src, valid_bytes);
}
// Load B (w2)
constexpr int B_CHUNKS = BN * 8; // 1024 chunks
#pragma unroll
for (int idx = tid; idx < B_CHUNKS; idx += NTHREADS) {
int r = idx >> 3;
int c8 = idx & 7;
int swiz = c8 ^ (r & 7);
uint32_t dst = b_base + (r * 8 + swiz) * 16;
cp_async_cg_16(dst, w2_base + (size_t)r * I + k_off + c8 * 8);
}
};
const int num_k_tiles = I / BK;
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < num_k_tiles) {
load_stage(s, s);
}
cp_async_commit();
}
const int lane_r16 = lane & 15;
const int lane_r7 = lane & 7;
const int lane_c_bit = lane >> 4;
const int warp_m_base = warp_m * WM;
const int warp_n_base = warp_n * WN;
for (int kt = 0; kt < num_k_tiles; ++kt) {
const int read_stage = kt % STAGES;
const int write_stage = (kt + STAGES - 1) % STAGES;
const int next_kt = kt + STAGES - 1;
cp_async_wait<STAGES - 2>();
__syncthreads();
if (next_kt < num_k_tiles) {
load_stage(next_kt, write_stage);
}
cp_async_commit();
const uint32_t st_base = smem_base + read_stage * STAGE_BYTES;
const uint32_t a_base = st_base;
const uint32_t b_base = st_base + A_STAGE_BYTES;
uint32_t ra[2][M_TILES][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r = warp_m_base + mi * 16 + lane_r16;
int c8 = lane_c_bit;
uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
ldmatrix_x4(ra[0][mi][0], ra[0][mi][1], ra[0][mi][2], ra[0][mi][3], addr);
}
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
const int cur = ks & 1;
const int nxt = cur ^ 1;
if (ks < 3) {
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r = warp_m_base + mi * 16 + lane_r16;
int c8 = (ks + 1) * 2 + lane_c_bit;
uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16;
ldmatrix_x4(ra[nxt][mi][0], ra[nxt][mi][1], ra[nxt][mi][2], ra[nxt][mi][3], addr);
}
}
#pragma unroll
for (int ni = 0; ni < N_TILES; ++ni) {
int nb = warp_n_base + ni * 16 + lane_r16;
int c8 = ks * 2 + lane_c_bit;
uint32_t b0, b1, b2, b3;
ldmatrix_x4(b0, b1, b2, b3, b_base + (nb * 8 + (c8 ^ lane_r7)) * 16);
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
mma_m16n8k16_bf16(
acc[mi][2 * ni + 0][0], acc[mi][2 * ni + 0][1],
acc[mi][2 * ni + 0][2], acc[mi][2 * ni + 0][3],
ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
b0, b2
);
mma_m16n8k16_bf16(
acc[mi][2 * ni + 1][0], acc[mi][2 * ni + 1][1],
acc[mi][2 * ni + 1][2], acc[mi][2 * ni + 1][3],
ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3],
b1, b3
);
}
}
}
}
cp_async_wait<0>();
__syncthreads();
// Epilogue: stage FP32 results to SMEM (padded stride 136 floats = 544B, 16B aligned)
// Max size: BM=128 -> 128 * 136 * 4 = 69,632 B (fits in 96KB SMEM!)
// BM=64 -> 64 * 136 * 4 = 34,816 B (fits in 96KB SMEM!)
// BM=32 -> 32 * 136 * 4 = 17,408 B (fits in 80KB SMEM!)
constexpr int S_OUT_STRIDE = BN + 8; // 136 floats
float* s_out = reinterpret_cast<float*>(smem_raw);
const int g = lane >> 2;
const int t = lane & 3;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r0 = warp_m_base + mi * 16 + g;
int r1 = r0 + 8;
float w0 = IS_SHARED ? 1.0f : s_wt[r0];
float w1 = IS_SHARED ? 1.0f : s_wt[r1];
#pragma unroll
for (int n8 = 0; n8 < N_TILES * 2; ++n8) {
int c0 = warp_n_base + n8 * 8 + t * 2;
float2 v0 = make_float2(acc[mi][n8][0] * w0, acc[mi][n8][1] * w0);
float2 v1 = make_float2(acc[mi][n8][2] * w1, acc[mi][n8][3] * w1);
*reinterpret_cast<float2*>(&s_out[r0 * S_OUT_STRIDE + c0]) = v0;
*reinterpret_cast<float2*>(&s_out[r1 * S_OUT_STRIDE + c0]) = v1;
}
}
__syncthreads();
// 1 warp writes 1 full row of 128 floats (32 float4s = 512B) in a single coalesced step!
for (int r = warp_id; r < m_valid; r += NWARPS) {
float4 v = reinterpret_cast<const float4*>(&s_out[r * S_OUT_STRIDE])[lane];
if constexpr (EPILOGUE_MODE == 0) {
int tok = s_tok[r];
float* dst = out_fp32 + (size_t)tok * H + col_base + lane * 4;
red_global_add_v4_f32(dst, v);
} else if constexpr (EPILOGUE_MODE == 1) {
int row = row_base + r;
float* dst = out_fp32 + (size_t)row * H + col_base + lane * 4;
red_global_add_v4_f32(dst, v);
} else {
int row = row_base + r;
const float4 prev = reinterpret_cast<const float4*>(
out_fp32 + (size_t)row * H + col_base
)[lane];
__nv_bfloat162 b0 = __floats2bfloat162_rn(v.x + prev.x, v.y + prev.y);
__nv_bfloat162 b1 = __floats2bfloat162_rn(v.z + prev.z, v.w + prev.w);
uint2 packed;
packed.x = *reinterpret_cast<const uint32_t*>(&b0);
packed.y = *reinterpret_cast<const uint32_t*>(&b1);
reinterpret_cast<uint2*>(out_bf16 + (size_t)row * H + col_base)[lane] = packed;
}
}
}
__global__ void cast_f32_to_bf16_kernel(
const float* __restrict__ src,
__nv_bfloat16* __restrict__ dst,
size_t n_vec4
) {
size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
size_t stride = (size_t)gridDim.x * blockDim.x;
const float4* src4 = reinterpret_cast<const float4*>(src);
uint2* dst2 = reinterpret_cast<uint2*>(dst);
for (; idx < n_vec4; idx += stride) {
float4 v = src4[idx];
__nv_bfloat162 b0 = __floats2bfloat162_rn(v.x, v.y);
__nv_bfloat162 b1 = __floats2bfloat162_rn(v.z, v.w);
uint2 packed;
packed.x = *reinterpret_cast<const uint32_t*>(&b0);
packed.y = *reinterpret_cast<const uint32_t*>(&b1);
dst2[idx] = packed;
}
}
// ============================================================================
// C++ Launch Wrappers
// ============================================================================
template <typename KernelT>
static inline void ensure_smem_once(bool& configured, KernelT kernel, size_t smem_bytes) {
if (!configured && smem_bytes > 48 * 1024) {
C10_CUDA_CHECK(cudaFuncSetAttribute(
kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast<int>(smem_bytes)
));
configured = true;
}
}
void launch_decode_moe(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared,
torch::Tensor h_buf,
torch::Tensor out
) {
const int T = static_cast<int>(x.size(0));
const int H = static_cast<int>(x.size(1));
const int E = static_cast<int>(w1_routed.size(0));
const int I = static_cast<int>(w2_routed.size(2));
const int top_k = static_cast<int>(expert_ids.size(1));
const int n_shared = static_cast<int>(w1_shared.size(0));
const int total_slots = top_k + n_shared;
cudaStream_t stream = c10::cuda::getCurrentCUDAStream();
// Kernel D1: 4 warps/block -> 4608 blocks for T=1, 9 slots, I=2048
{
constexpr int WPB = 4;
dim3 grid((I + WPB - 1) / WPB, total_slots, T);
dim3 block(WPB * 32);
size_t smem = (size_t)H * sizeof(__nv_bfloat16);
moe_decode_gemv1_silu_mul_kernel<WPB><<<grid, block, smem, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
expert_ids.data_ptr<int64_t>(),
reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(h_buf.data_ptr()),
T, E, top_k, n_shared, H, I
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
// Kernel D2: 2 warps/block -> 2048 blocks for T=1, H=4096 (10.9 waves on 188 SMs)
{
constexpr int WPB = 2;
dim3 grid((H + WPB - 1) / WPB, 1, T);
dim3 block(WPB * 32);
moe_decode_gemv2_reduce_kernel<WPB><<<grid, block, 0, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(h_buf.data_ptr()),
expert_ids.data_ptr<int64_t>(),
reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(out.data_ptr()),
T, E, top_k, n_shared, H, I
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
}
template <int BM, int STAGES, int WARPS_M, int WARPS_N>
static void run_routed_grouped_impl(
const __nv_bfloat16* x_ptr,
const __nv_bfloat16* w1_ptr,
const __nv_bfloat16* w2_ptr,
__nv_bfloat16* h_ptr,
float* out_fp32_ptr,
const int32_t* offsets_ptr,
const int32_t* tile_prefix_ptr,
const int32_t* m_tile_to_expert_ptr,
const int32_t* sorted_tok_ptr,
const float* sorted_wt_ptr,
int T, int E, int H, int I, int max_m_tiles,
cudaStream_t stream
) {
constexpr int BN1_OUT = 64;
constexpr int BN2 = 128;
constexpr int BK = 64;
constexpr int NTHREADS = WARPS_M * WARPS_N * 32;
// GEMM1
{
static bool cfg1 = false;
constexpr size_t smem1 = STAGES * (BM * 128 + 2 * BN1_OUT * 128);
auto k1 = moe_gemm1_silu_mul_kernel<BM, BN1_OUT, BK, STAGES, WARPS_M, WARPS_N, false>;
ensure_smem_once(cfg1, k1, smem1);
int grid1 = max_m_tiles * (I / BN1_OUT);
k1<<<grid1, NTHREADS, smem1, stream>>>(
x_ptr, w1_ptr, h_ptr,
offsets_ptr, tile_prefix_ptr, m_tile_to_expert_ptr, sorted_tok_ptr,
T, E, H, I, 0
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
// GEMM2
{
static bool cfg2 = false;
constexpr size_t smem2_pipe = STAGES * (BM * 128 + BN2 * 128);
constexpr size_t smem2_epi = BM * (BN2 + 8) * sizeof(float);
constexpr size_t smem2 = smem2_pipe > smem2_epi ? smem2_pipe : smem2_epi;
auto k2 = moe_gemm2_kernel<BM, BN2, BK, STAGES, WARPS_M, WARPS_N, 0>;
ensure_smem_once(cfg2, k2, smem2);
int grid2 = max_m_tiles * (H / BN2);
k2<<<grid2, NTHREADS, smem2, stream>>>(
h_ptr, w2_ptr, out_fp32_ptr, nullptr,
offsets_ptr, tile_prefix_ptr, m_tile_to_expert_ptr, sorted_tok_ptr, sorted_wt_ptr,
T, E, H, I, 0
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
}
template <int BM, int STAGES, int WARPS_M, int WARPS_N>
static void run_shared_impl(
const __nv_bfloat16* x_ptr,
const __nv_bfloat16* w1_s_ptr,
const __nv_bfloat16* w2_s_ptr,
__nv_bfloat16* h_s_ptr,
float* out_fp32_ptr,
__nv_bfloat16* out_bf16_ptr,
int T, int H, int I, int s_idx, bool is_last,
cudaStream_t stream
) {
constexpr int BN1_OUT = 64;
constexpr int BN2 = 128;
constexpr int BK = 64;
constexpr int NTHREADS = WARPS_M * WARPS_N * 32;
const int m_tiles = (T + BM - 1) / BM;
// Shared GEMM1
{
static bool cfg_s1 = false;
constexpr size_t smem1 = STAGES * (BM * 128 + 2 * BN1_OUT * 128);
auto k1 = moe_gemm1_silu_mul_kernel<BM, BN1_OUT, BK, STAGES, WARPS_M, WARPS_N, true>;
ensure_smem_once(cfg_s1, k1, smem1);
int grid1 = m_tiles * (I / BN1_OUT);
k1<<<grid1, NTHREADS, smem1, stream>>>(
x_ptr, w1_s_ptr, h_s_ptr,
nullptr, nullptr, nullptr, nullptr,
T, 1, H, I, s_idx
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
// Shared GEMM2
{
static bool cfg_s2_last = false;
static bool cfg_s2_mid = false;
constexpr size_t smem2_pipe = STAGES * (BM * 128 + BN2 * 128);
constexpr size_t smem2_epi = BM * (BN2 + 8) * sizeof(float);
constexpr size_t smem2 = smem2_pipe > smem2_epi ? smem2_pipe : smem2_epi;
int grid2 = m_tiles * (H / BN2);
if (is_last) {
auto k2 = moe_gemm2_kernel<BM, BN2, BK, STAGES, WARPS_M, WARPS_N, 2>;
ensure_smem_once(cfg_s2_last, k2, smem2);
k2<<<grid2, NTHREADS, smem2, stream>>>(
h_s_ptr, w2_s_ptr, out_fp32_ptr, out_bf16_ptr,
nullptr, nullptr, nullptr, nullptr, nullptr,
T, 1, H, I, s_idx
);
} else {
auto k2 = moe_gemm2_kernel<BM, BN2, BK, STAGES, WARPS_M, WARPS_N, 1>;
ensure_smem_once(cfg_s2_mid, k2, smem2);
k2<<<grid2, NTHREADS, smem2, stream>>>(
h_s_ptr, w2_s_ptr, out_fp32_ptr, nullptr,
nullptr, nullptr, nullptr, nullptr, nullptr,
T, 1, H, I, s_idx
);
}
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
}
void launch_grouped_moe(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared,
torch::Tensor h_routed,
torch::Tensor h_shared,
torch::Tensor out_fp32,
torch::Tensor out_bf16,
torch::Tensor expert_counts,
torch::Tensor expert_offsets,
torch::Tensor write_cursors,
torch::Tensor expert_tile_prefix,
torch::Tensor m_tile_to_expert,
torch::Tensor sorted_token_ids,
torch::Tensor sorted_weights,
int bm_routed
) {
const int T = static_cast<int>(x.size(0));
const int H = static_cast<int>(x.size(1));
const int E = static_cast<int>(w1_routed.size(0));
const int I = static_cast<int>(w2_routed.size(2));
const int top_k = static_cast<int>(expert_ids.size(1));
const int n_shared = static_cast<int>(w1_shared.size(0));
const int M_total = T * top_k;
const int max_m_tiles = (M_total + bm_routed - 1) / bm_routed + E;
cudaStream_t stream = c10::cuda::getCurrentCUDAStream();
// 1. Zero expert_counts and out_fp32
C10_CUDA_CHECK(cudaMemsetAsync(expert_counts.data_ptr(), 0, (size_t)E * sizeof(int32_t), stream));
C10_CUDA_CHECK(cudaMemsetAsync(out_fp32.data_ptr(), 0, (size_t)T * (size_t)H * sizeof(float), stream));
// 2. Histogram expert counts
{
int blocks = min(256, (M_total + 255) / 256);
moe_count_experts_kernel<<<blocks, 256, (size_t)E * sizeof(int32_t), stream>>>(
expert_ids.data_ptr<int64_t>(),
expert_counts.data_ptr<int32_t>(),
M_total, E
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
// 3. Prefix sum + tile lookup table
moe_build_offsets_and_tiles_kernel<<<1, 1, 0, stream>>>(
expert_counts.data_ptr<int32_t>(),
expert_offsets.data_ptr<int32_t>(),
write_cursors.data_ptr<int32_t>(),
expert_tile_prefix.data_ptr<int32_t>(),
m_tile_to_expert.data_ptr<int32_t>(),
E, bm_routed
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
// 4. Scatter sorted token_ids and weights
{
int blocks = min(256, (M_total + 255) / 256);
moe_scatter_tokens_kernel<<<blocks, 256, 0, stream>>>(
expert_ids.data_ptr<int64_t>(),
reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
write_cursors.data_ptr<int32_t>(),
sorted_token_ids.data_ptr<int32_t>(),
sorted_weights.data_ptr<float>(),
M_total, top_k
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
const __nv_bfloat16* x_ptr = reinterpret_cast<const __nv_bfloat16*>(x.data_ptr());
const __nv_bfloat16* w1_r_ptr = reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr());
const __nv_bfloat16* w2_r_ptr = reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr());
__nv_bfloat16* h_r_ptr = reinterpret_cast<__nv_bfloat16*>(h_routed.data_ptr());
float* out_f32_ptr = out_fp32.data_ptr<float>();
__nv_bfloat16* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr());
// 5. Routed Experts (GEMM1 fused SiLU*mul -> GEMM2 fused weight-scale + v4.f32 scatter-add)
if (bm_routed == 32) {
run_routed_grouped_impl<32, 4, 2, 4>(
x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr,
expert_offsets.data_ptr<int32_t>(),
expert_tile_prefix.data_ptr<int32_t>(),
m_tile_to_expert.data_ptr<int32_t>(),
sorted_token_ids.data_ptr<int32_t>(),
sorted_weights.data_ptr<float>(),
T, E, H, I, max_m_tiles, stream
);
} else if (bm_routed == 64) {
run_routed_grouped_impl<64, 4, 2, 4>(
x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr,
expert_offsets.data_ptr<int32_t>(),
expert_tile_prefix.data_ptr<int32_t>(),
m_tile_to_expert.data_ptr<int32_t>(),
sorted_token_ids.data_ptr<int32_t>(),
sorted_weights.data_ptr<float>(),
T, E, H, I, max_m_tiles, stream
);
} else {
run_routed_grouped_impl<128, 3, 4, 2>(
x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr,
expert_offsets.data_ptr<int32_t>(),
expert_tile_prefix.data_ptr<int32_t>(),
m_tile_to_expert.data_ptr<int32_t>(),
sorted_token_ids.data_ptr<int32_t>(),
sorted_weights.data_ptr<float>(),
T, E, H, I, max_m_tiles, stream
);
}
// 6. Shared Expert(s) — last shared expert fuses add(out_fp32) + cast(bf16) -> out_bf16!
if (n_shared > 0) {
const __nv_bfloat16* w1_s_ptr = reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr());
const __nv_bfloat16* w2_s_ptr = reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr());
__nv_bfloat16* h_s_ptr = reinterpret_cast<__nv_bfloat16*>(h_shared.data_ptr());
for (int s = 0; s < n_shared; ++s) {
bool is_last = (s == n_shared - 1);
if (T <= 32) {
run_shared_impl<32, 4, 2, 4>(
x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr,
T, H, I, s, is_last, stream
);
} else if (T <= 96) {
run_shared_impl<64, 4, 2, 4>(
x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr,
T, H, I, s, is_last, stream
);
} else {
run_shared_impl<128, 3, 4, 2>(
x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr,
T, H, I, s, is_last, stream
);
}
}
} else {
size_t n_vec4 = ((size_t)T * (size_t)H) / 4;
int blocks = static_cast<int>(min((size_t)1024, (n_vec4 + 255) / 256));
cast_f32_to_bf16_kernel<<<blocks, 256, 0, stream>>>(
out_f32_ptr, out_bf16_ptr, n_vec4
);
C10_CUDA_KERNEL_LAUNCH_CHECK();
}
}
"""
_CPP_DECLS = """
void launch_decode_moe(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared,
torch::Tensor h_buf,
torch::Tensor out
);
void launch_grouped_moe(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared,
torch::Tensor h_routed,
torch::Tensor h_shared,
torch::Tensor out_fp32,
torch::Tensor out_bf16,
torch::Tensor expert_counts,
torch::Tensor expert_offsets,
torch::Tensor write_cursors,
torch::Tensor expert_tile_prefix,
torch::Tensor m_tile_to_expert,
torch::Tensor sorted_token_ids,
torch::Tensor sorted_weights,
int bm_routed
);
"""
_EXT = None
def _get_ext():
global _EXT
if _EXT is None:
_EXT = load_inline(
name="glm52_fused_moe_sm120_v1",
cpp_sources=_CPP_DECLS,
cuda_sources=_CUDA_SRC,
functions=["launch_decode_moe", "launch_grouped_moe"],
extra_cuda_cflags=["-O3", "-arch=sm_120a", "--expt-relaxed-constexpr"],
verbose=False,
)
return _EXT
class _Workspace:
"""Reusable GPU scratch buffers to avoid per-forward allocations."""
def __init__(self):
self.device = None
self.h_decode = None
self.h_routed = None
self.h_shared = None
self.out_fp32 = None
self.expert_counts = None
self.expert_offsets = None
self.write_cursors = None
self.expert_tile_prefix = None
self.m_tile_to_expert = None
self.sorted_token_ids = None
self.sorted_weights = None
def ensure_decode(self, device, T: int, total_slots: int, I: int):
need = T * total_slots * I
if self.device != device or self.h_decode is None or self.h_decode.numel() < need:
self.device = device
self.h_decode = torch.empty(need, dtype=torch.bfloat16, device=device)
return self.h_decode
def ensure_grouped(self, device, T: int, E: int, top_k: int, H: int, I: int, bm: int):
M_total = T * top_k
max_m_tiles = (M_total + bm - 1) // bm + E
if self.device != device:
self.device = device
self.h_routed = None
self.h_shared = None
self.out_fp32 = None
self.expert_counts = None
self.expert_offsets = None
self.write_cursors = None
self.expert_tile_prefix = None
self.m_tile_to_expert = None
self.sorted_token_ids = None
self.sorted_weights = None
if self.h_routed is None or self.h_routed.numel() < M_total * I:
self.h_routed = torch.empty((M_total, I), dtype=torch.bfloat16, device=device)
if self.h_shared is None or self.h_shared.numel() < T * I:
self.h_shared = torch.empty((T, I), dtype=torch.bfloat16, device=device)
if self.out_fp32 is None or self.out_fp32.numel() < T * H:
self.out_fp32 = torch.empty((T, H), dtype=torch.float32, device=device)
if self.expert_counts is None or self.expert_counts.numel() < E:
self.expert_counts = torch.empty(E, dtype=torch.int32, device=device)
if self.expert_offsets is None or self.expert_offsets.numel() < E + 1:
self.expert_offsets = torch.empty(E + 1, dtype=torch.int32, device=device)
if self.write_cursors is None or self.write_cursors.numel() < E:
self.write_cursors = torch.empty(E, dtype=torch.int32, device=device)
if self.expert_tile_prefix is None or self.expert_tile_prefix.numel() < E + 1:
self.expert_tile_prefix = torch.empty(E + 1, dtype=torch.int32, device=device)
if self.m_tile_to_expert is None or self.m_tile_to_expert.numel() < max_m_tiles:
self.m_tile_to_expert = torch.empty(max_m_tiles, dtype=torch.int32, device=device)
if self.sorted_token_ids is None or self.sorted_token_ids.numel() < M_total:
self.sorted_token_ids = torch.empty(M_total, dtype=torch.int32, device=device)
if self.sorted_weights is None or self.sorted_weights.numel() < M_total:
self.sorted_weights = torch.empty(M_total, dtype=torch.float32, device=device)
_WORKSPACES: dict[int, _Workspace] = {}
def _get_workspace(device: torch.device) -> _Workspace:
idx = device.index if device.index is not None else 0
ws = _WORKSPACES.get(idx)
if ws is None:
ws = _Workspace()
_WORKSPACES[idx] = ws
return ws
class Model(nn.Module):
def __init__(
self,
T: int,
E: int,
top_k: int,
n_shared: int,
H: int,
I: int,
):
super().__init__()
self.T, self.E, self.top_k = T, E, top_k
self.n_shared, self.H, self.I = n_shared, H, I
self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
for p in self.parameters():
nn.init.normal_(p, std=0.02)
# Pre-compile extension at init time (outside timed forward)
_get_ext()
def forward(
self,
x: torch.Tensor,
expert_ids: torch.Tensor,
expert_weights: torch.Tensor,
) -> torch.Tensor:
T, H = x.shape
E = self.E
top_k = self.top_k
n_shared = self.n_shared
I = self.I
ext = _get_ext()
ws = _get_workspace(x.device)
out = torch.empty((T, H), dtype=torch.bfloat16, device=x.device)
x_c = x.contiguous()
ids_c = expert_ids.contiguous()
wts_c = expert_weights.contiguous()
if T <= 4 and (H % 8 == 0) and (I % 8 == 0):
h_buf = ws.ensure_decode(x.device, T, top_k + n_shared, I)
ext.launch_decode_moe(
x_c, ids_c, wts_c,
self.w1_routed, self.w2_routed,
self.w1_shared, self.w2_shared,
h_buf, out,
)
return out
# Choose routed BM based on average tokens per expert
m_avg = (T * top_k) / max(E, 1)
bm_override = os.environ.get("MOE_BM_OVERRIDE")
if bm_override is not None:
bm = int(bm_override)
elif m_avg <= 20.0:
bm = 32
elif m_avg <= 160.0:
bm = 64
else:
bm = 64
ws.ensure_grouped(x.device, T, E, top_k, H, I, bm)
ext.launch_grouped_moe(
x_c, ids_c, wts_c,
self.w1_routed, self.w2_routed,
self.w1_shared, self.w2_shared,
ws.h_routed, ws.h_shared, ws.out_fp32, out,
ws.expert_counts, ws.expert_offsets, ws.write_cursors,
ws.expert_tile_prefix, ws.m_tile_to_expert,
ws.sorted_token_ids, ws.sorted_weights,
bm,
)
return out
20261118_150445_agy_gemini-4-neon-xhigh_01_glm52_fused_moe