KernelBench cuda · RTX PRO 6000
GLM-5.3 Fused MoE Claude Fable 5.2
10.8%geomean peak fraction across shapes
manually audited: clean
Fable 5.2 treated the MoE layer as a weight-streaming problem. One prep CTA counting-sorts (token, slot) pairs by expert and builds balanced work lists; persistent CTAs then stream each expert's weights from DRAM once with warp-wide cp.async and bf16 mma.sync, fusing SiLU(gate) * up * route weight into the first GEMM's epilogue, and a combine kernel sums shared plus top-k outputs in fp32. The warp grid is picked per expert from its token count.
harnessclaudeagent session1h 57mtotal wall2h 12mcheck5mbenchmark4moutput tokens315,951cost$23.07gpu-lock wait16mgpu-lock held1h 8mregimecompute
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
shape 09.272 ms40.0%1.40 TB/s · 78% of 1.8 TB/s HBM · also 200 TFLOPS (40% of compute)
shape 19.281 ms40.3%1.40 TB/s · 78% of 1.8 TB/s HBM · also 201 TFLOPS (40% of compute)
shape 20.318 ms0.3%40.73 TB/s · 100% of 1.8 TB/s HBM · also 1 TFLOPS (0% of compute)
shape 313.208 ms56.2%281 TFLOPS · 56% of 500 TF bf16 peak · also 0.99 TB/s (55% of HBM)
shape 48.054 ms5.8%1.61 TB/s · 89% of 1.8 TB/s HBM · also 29 TFLOPS (6% of compute)
shape 58.257 ms11.0%1.57 TB/s · 87% of 1.8 TB/s HBM · also 55 TFLOPS (11% of compute)
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(40.0% · 40.3% · 0.3% · 56.2% · 5.8% · 11.0%) = 10.9%
Kernel source (redacted)
"""GLM-5.2-class fused MoE layer — hand-written CUDA (mma.sync + cp.async) for SM120.
Design (RTX PRO 6000 Blackwell: 188 SMs, 100 KB smem/SM, 128 MB L2, ~1.6 TB/s DRAM)
---------------------------------------------------------------------------------
Every call has to touch every routed expert's weights once (12.9 GB for T >= 512), so the
layer is a *weight-streaming* problem first and a tensor-core problem second.
prep_k one CTA: counting-sort the (token, slot) pairs by expert, pick a warp layout
per expert from its token count, and emit balanced work lists for both stages.
moe_k<1> persistent CTAs pull work items (expert, token tile, neuron range):
[gate|up] = X_e @ W1_e^T with bf16 mma.sync (fp32 accumulate),
fused epilogue h = silu(gate) * up * route_weight -> bf16.
moe_k<2> same streaming GEMM for y = h @ W2_e^T -> bf16 scratch[token][slot].
combine_k out[t] = sum over slots (shared + top-k routed), fp32 sum -> bf16.
The pybind wrapper validates the inputs, keeps grow-only scratch buffers per device and
launches the four kernels on torch's current stream; anything it does not cover (CPU tensors,
H or I not a multiple of 64, ...) falls back to a plain PyTorch path.
Streaming GEMM details that matter on this card:
* weights go DRAM -> smem exactly once per expert; all tokens of an expert share the pass
(256-token tiles; the L2 de-duplicates the re-reads of multi-tile experts);
* every 128-byte line is requested by ONE warp-wide cp.async instruction (8 lanes x 16 B),
which is worth ~2x DRAM throughput versus per-row chunk loops;
* 8 warps x (64x64) accumulator tiles, warp grid chosen per item (4x2 / 2x4 / 1x5) so the
tile matches the expert's token count; the work list is sized to a multiple of the SM
count so the last wave of CTAs is full (matters for T=1 decode).
"""
from __future__ import annotations
import torch
import torch.nn as nn
import torch.nn.functional as F
import torch.utils.cpp_extension
OP_TYPE = "glm52_fused_moe"
SUPPORTED_PRECISIONS = ["bf16"]
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
T, E, top_k, n_shared, H, I = 4096, 256, 8, 1, 4096, 2048
# ----------------------------------------------------------------------------------------
# CUDA source
# ----------------------------------------------------------------------------------------
_CUDA_SRC = r"""
#include <cuda_runtime.h>
#include <cstdint>
#include <cstdio>
namespace {
struct Item { int e, p_off, m_cnt, pk; }; // pk = n0 | n_cnt << 16 | WN << 25
#define MAXE 512
#define NWARP 8
#define NTHR (NWARP * 32)
#define MTMAX 256
__device__ __forceinline__ float bf16_to_float(uint16_t v) { return __uint_as_float(((uint32_t)v) << 16); }
__device__ __forceinline__ uint32_t float_to_bf16(float f) { // round-to-nearest-even
uint32_t u = __float_as_uint(f);
u += 0x7FFFu + ((u >> 16) & 1u);
return u >> 16;
}
// ---------------------------------------------------------------------------
// prep: counting sort of (token, slot) pairs by expert + balanced work lists.
// Each expert -> up to two "units": U0 = tiles of MTMAX tokens (warp layout 4x2),
// U1 = small remainder tile with a layout matched to its token count.
// ---------------------------------------------------------------------------
struct UInfo { int mt, WN; };
__device__ __forceinline__ UInfo unit_info(int n, int odd, int th1, int th2) {
UInfo u; u.mt = 0; u.WN = 2;
if (n <= 0) return u;
const int mtf = (n - 1) / MTMAX, rem = n - mtf * MTMAX; // rem in 1..MTMAX
const bool small = rem <= th2;
if (!odd) { u.mt = mtf + (small ? 0 : 1); }
else if (small) { u.mt = 1; u.WN = (rem <= th1) ? 5 : 4; }
return u;
}
__global__ void prep_k(const long long* __restrict__ ids, const uint16_t* __restrict__ wts,
int T, int topk, int E, int NS,
int* __restrict__ tok, int* __restrict__ dst, float* __restrict__ wgt,
unsigned* __restrict__ vmask,
Item* __restrict__ items1, Item* __restrict__ items2,
int* queue1, int* queue2, int U1, int U2, int G, int th1, int th2)
{
__shared__ int cnt[MAXE], off[MAXE], cur[MAXE];
__shared__ int A1[2 * MAXE], A2[2 * MAXE], B1[2 * MAXE], B2[2 * MAXE], io1[2 * MAXE + 1], io2[2 * MAXE + 1];
__shared__ int tot[2];
const int tid = threadIdx.x, nth = blockDim.x;
const int P = T * topk, ET = E + NS, SL = topk + NS, NU = 2 * ET;
for (int e = tid; e < ET; e += nth) { cnt[e] = (e >= E) ? T : 0; cur[e] = 0; }
__syncthreads();
for (int p = tid; p < P; p += nth) {
const long long e = ids[p];
if (e >= 0 && e < E) atomicAdd(&cnt[(int)e], 1);
}
__syncthreads();
// minimal item counts per unit
for (int u = tid; u < NU; u += nth) {
const UInfo ui = unit_info(cnt[u >> 1], u & 1, th1, th2);
A1[u] = ui.mt * ((U1 + 4 * ui.WN - 1) / (4 * ui.WN));
A2[u] = ui.mt * ((U2 + 8 * ui.WN - 1) / (8 * ui.WN));
}
__syncthreads();
for (int u = tid; u < NU; u += nth) {
int a1 = 0, a2 = 0;
for (int j = 0; j < u; j++) { a1 += A1[j]; a2 += A2[j]; }
B1[u] = a1; B2[u] = a2;
if (u == NU - 1) { tot[0] = a1 + A1[u]; tot[1] = a2 + A2[u]; }
}
for (int e = tid; e < ET; e += nth) {
int o = 0;
for (int j = 0; j < e; j++) o += cnt[j];
off[e] = o;
}
__syncthreads();
// scale item counts so that the total is a multiple of the grid size (full last wave)
for (int u = tid; u < NU; u += nth) {
const UInfo ui = unit_info(cnt[u >> 1], u & 1, th1, th2);
int k1 = 0, k2 = 0;
if (ui.mt > 0) {
const long long n01 = tot[0], n02 = tot[1];
const long long t1 = ((n01 + G - 1) / G) * G, t2 = ((n02 + G - 1) / G) * G;
const int k01 = A1[u] / ui.mt, k02 = A2[u] / ui.mt;
k1 = (int)(((long long)(B1[u] + A1[u]) * t1 / n01 - (long long)B1[u] * t1 / n01) / ui.mt);
k2 = (int)(((long long)(B2[u] + A2[u]) * t2 / n02 - (long long)B2[u] * t2 / n02) / ui.mt);
k1 = min(max(k1, k01), U1); k2 = min(max(k2, k02), U2);
}
A1[u] = k1; A2[u] = k2;
}
__syncthreads();
for (int u = tid; u < NU; u += nth) {
const UInfo ui = unit_info(cnt[u >> 1], u & 1, th1, th2);
B1[u] = A1[u] * ui.mt; B2[u] = A2[u] * ui.mt;
}
__syncthreads();
for (int u = tid; u < NU; u += nth) {
int a1 = 0, a2 = 0;
for (int j = 0; j < u; j++) { a1 += B1[j]; a2 += B2[j]; }
io1[u] = a1; io2[u] = a2;
if (u == NU - 1) {
const int t1 = a1 + B1[u], t2 = a2 + B2[u];
io1[NU] = t1; io2[NU] = t2;
queue1[0] = 0; queue1[1] = t1; queue2[0] = 0; queue2[1] = t2;
}
}
__syncthreads();
// scatter pairs (order inside an expert is irrelevant: rows are independent)
for (int t = tid; t < T; t += nth) {
unsigned vm = 0;
for (int k = 0; k < topk; k++) {
const int p = t * topk + k;
const long long e = ids[p];
if (e >= 0 && e < E) {
const int pos = off[(int)e] + atomicAdd(&cur[(int)e], 1);
tok[pos] = t; dst[pos] = t * SL + k; wgt[pos] = bf16_to_float(wts[p]);
vm |= 1u << k;
}
}
for (int s = 0; s < NS; s++) {
const int pos = off[E + s] + t;
tok[pos] = t; dst[pos] = t * SL + topk + s; wgt[pos] = 1.0f;
vm |= 1u << (topk + s);
}
vmask[t] = vm;
}
#pragma unroll
for (int st = 0; st < 2; st++) {
const int* io = st ? io2 : io1; const int* kA = st ? A2 : A1; const int* pr = st ? B2 : B1;
Item* items = st ? items2 : items1;
const int U = st ? U2 : U1;
const int total = io[NU];
for (int q = tid; q < total; q += nth) {
int lo = 0, hi = NU; // largest u with io[u] <= q
while (hi - lo > 1) { int mid = (lo + hi) >> 1; if (io[mid] <= q) lo = mid; else hi = mid; }
int u = lo;
while (pr[u] == 0) u++; // skip empty units sharing the same offset
const int e = u >> 1, n = cnt[e];
const UInfo ui = unit_info(n, u & 1, th1, th2);
const int k = kA[u];
const int local = q - io[u], j = local / ui.mt, i = local - j * ui.mt;
const int mtf = (n - 1) / MTMAX;
const int tile = (u & 1) ? mtf : i;
const int base = U / k, r2 = U - base * k;
const int start = j * base + min(j, r2), size = base + (j < r2);
Item it;
it.e = e; it.p_off = off[e] + tile * MTMAX; it.m_cnt = min(MTMAX, n - tile * MTMAX);
it.pk = (8 * start) | ((8 * size) << 16) | (ui.WN << 25);
items[q] = it;
}
}
}
// ---------------------------------------------------------------------------
// streaming GEMM: C[m][n] = sum_k A[m][k] * B[n][k] (both K-contiguous bf16)
// 8 warps, 64x64 warp tiles, per-item warp layout (WM x WN), compile-time specialised.
// A and B rings: 2 slots x 64 k.
// ---------------------------------------------------------------------------
__device__ __forceinline__ void cp_async16(unsigned sa, const void* gsrc) {
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;\n" :: "r"(sa), "l"(gsrc));
}
__device__ __forceinline__ void ldsm4(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, unsigned sa) {
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(sa));
}
__device__ __forceinline__ void mma16(float* c, const uint32_t* a, const uint32_t* b) {
asm volatile("mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 {%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n"
: "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
}
#define SMEM_TILES 98304
#define SMEM_TOTAL (SMEM_TILES + 16)
struct KCtx {
const uint16_t* A; long ldA; const int* a_row;
const uint16_t* Wr; const uint16_t* Ws; int E, K, NOUT;
uint16_t* out; long ldO; const int* out_row; const float* wgt;
int tid, lane, warp;
unsigned sm_base;
};
template<int STAGE, int WM, int WN>
__device__ __forceinline__ void run_item(const KCtx& c, const Item it)
{
constexpr int MT = 64 * WM, NT = 64 * WN;
constexpr unsigned a_slot = MT * 128, b_off = 2 * a_slot, b_slot = NT * 128, up_off = (NT / 2) * 128;
const int tid = c.tid, lane = c.lane, warp = c.warp;
const int K = c.K, NOUT = c.NOUT;
const long NROWS = (STAGE == 1) ? 2L * NOUT : (long)NOUT;
const int NK = K / 64;
const int m_cnt = it.m_cnt;
const int n0 = it.pk & 0xFFFF, n_cnt = (it.pk >> 16) & 0x1FF;
const uint16_t* Wb = (it.e < c.E) ? c.Wr + (long)it.e * NROWS * K : c.Ws + (long)(it.e - c.E) * NROWS * K;
const int wm = warp / WN, wn = warp % WN;
const int a_lrow = lane & 15, a_lhi = lane >> 4;
const int b_lrow = (lane & 7) + 8 * (lane >> 4), b_lhi = (lane >> 3) & 1;
const int a_sw = a_lrow & 7, b_sw = b_lrow & 7;
// loader geometry: thread -> chunk c8 of rows r0 + 32*i; lanes 8a..8a+7 cover one full
// 128B line, so every line is requested by exactly one warp-wide cp.async instruction
const int c8 = tid & 7, r0 = tid >> 3;
const unsigned ld_dst0 = c.sm_base + r0 * 128 + ((c8 ^ (r0 & 7)) << 4);
const long strideB = 64L * K; // bytes between rows r and r+32
uint32_t aoff[2 * WM]; // A row offsets in 16B units
#pragma unroll
for (int i = 0; i < 2 * WM; i++) {
const int r = r0 + 32 * i;
aoff[i] = 0;
if (r < m_cnt) {
const long gr = (STAGE == 1) ? (long)c.a_row[it.p_off + r] : (long)(it.p_off + r);
aoff[i] = (uint32_t)(gr * (c.ldA >> 3));
}
}
const char* pa = (const char*)c.A + c8 * 16;
const char* pg = (const char*)(Wb + (long)(n0 + r0) * K) + c8 * 16;
const char* pu = pg + (long)((STAGE == 1) ? NOUT : NT / 2) * K * 2;
const int n_cnt1 = min(n_cnt, NT / 2);
const int n_cnt2 = (STAGE == 1) ? n_cnt : n_cnt - NT / 2;
auto load = [&](int slot) {
const unsigned da = ld_dst0 + slot * a_slot;
#pragma unroll
for (int i = 0; i < 2 * WM; i++)
if (r0 + 32 * i < m_cnt) cp_async16(da + i * 4096, pa + ((size_t)aoff[i] << 4));
const unsigned db = ld_dst0 + b_off + slot * b_slot;
#pragma unroll
for (int i = 0; i < WN; i++) {
if (r0 + 32 * i < n_cnt1) cp_async16(db + i * 4096, pg + i * strideB);
if (r0 + 32 * i < n_cnt2) cp_async16(db + up_off + i * 4096, pu + i * strideB);
}
pa += 128; pg += 128; pu += 128;
asm volatile("cp.async.commit_group;\n" ::);
};
int a_cnt = (m_cnt - wm * 64 + 15) >> 4; // 16-row A fragments this warp really needs
a_cnt = a_cnt < 0 ? 0 : (a_cnt > 4 ? 4 : a_cnt);
if (wm >= WM || wn * ((STAGE == 1) ? 32 : 64) >= n_cnt) a_cnt = 0;
float acc[4][8][4];
#pragma unroll
for (int i = 0; i < 4; i++)
#pragma unroll
for (int j = 0; j < 8; j++) { acc[i][j][0] = 0.f; acc[i][j][1] = 0.f; acc[i][j][2] = 0.f; acc[i][j][3] = 0.f; }
load(0);
const unsigned a_lane_off = c.sm_base + (wm * 64 + a_lrow) * 128;
// B fragment rows: stage 1 -> [gate half | up half], stage 2 -> contiguous
const unsigned b_lane_lo = c.sm_base + b_off + (((STAGE == 1) ? wn * 32 : wn * 64) + b_lrow) * 128;
const unsigned b_lane_hi = (STAGE == 1) ? b_lane_lo + up_off : b_lane_lo + 32 * 128;
int slot = 0;
for (int kt = 0; kt < NK; kt++) {
asm volatile("cp.async.wait_group 0;\n" ::);
__syncthreads();
if (kt + 1 < NK) load(slot ^ 1);
if (a_cnt == 4) { // full 64-row warp tile: branch-free hot path
const unsigned sa = a_lane_off + slot * a_slot;
const unsigned sbl = b_lane_lo + slot * b_slot, sbh = b_lane_hi + slot * b_slot;
#pragma unroll
for (int ks = 0; ks < 4; ks++) {
uint32_t af[4][4], bfr[8][2];
#pragma unroll
for (int i = 0; i < 4; i++)
ldsm4(af[i][0], af[i][1], af[i][2], af[i][3], sa + i * 16 * 128 + (((2 * ks + a_lhi) ^ a_sw) << 4));
#pragma unroll
for (int j = 0; j < 2; j++) {
ldsm4(bfr[2 * j][0], bfr[2 * j][1], bfr[2 * j + 1][0], bfr[2 * j + 1][1],
sbl + j * 16 * 128 + (((2 * ks + b_lhi) ^ b_sw) << 4));
ldsm4(bfr[2 * j + 4][0], bfr[2 * j + 4][1], bfr[2 * j + 5][0], bfr[2 * j + 5][1],
sbh + j * 16 * 128 + (((2 * ks + b_lhi) ^ b_sw) << 4));
}
#pragma unroll
for (int i = 0; i < 4; i++) {
#pragma unroll
for (int j = 0; j < 8; j++) mma16(acc[i][j], af[i], bfr[j]);
}
}
} else if (a_cnt > 0) { // partially filled tile: skip unused 16-row fragments
const unsigned sa = a_lane_off + slot * a_slot;
const unsigned sbl = b_lane_lo + slot * b_slot, sbh = b_lane_hi + slot * b_slot;
#pragma unroll
for (int ks = 0; ks < 4; ks++) {
uint32_t af[4][4], bfr[8][2];
#pragma unroll
for (int i = 0; i < 3; i++)
if (i < a_cnt)
ldsm4(af[i][0], af[i][1], af[i][2], af[i][3], sa + i * 16 * 128 + (((2 * ks + a_lhi) ^ a_sw) << 4));
#pragma unroll
for (int j = 0; j < 2; j++) {
ldsm4(bfr[2 * j][0], bfr[2 * j][1], bfr[2 * j + 1][0], bfr[2 * j + 1][1],
sbl + j * 16 * 128 + (((2 * ks + b_lhi) ^ b_sw) << 4));
ldsm4(bfr[2 * j + 4][0], bfr[2 * j + 4][1], bfr[2 * j + 5][0], bfr[2 * j + 5][1],
sbh + j * 16 * 128 + (((2 * ks + b_lhi) ^ b_sw) << 4));
}
#pragma unroll
for (int i = 0; i < 3; i++)
if (i < a_cnt) {
#pragma unroll
for (int j = 0; j < 8; j++) mma16(acc[i][j], af[i], bfr[j]);
}
}
}
slot ^= 1;
}
// ---------------- epilogue ----------------
const int r_base = wm * 64 + (lane >> 2);
const int c_lane = 2 * (lane & 3);
if (STAGE == 1) {
const int cb = wn * 32;
#pragma unroll
for (int i = 0; i < 4; i++) {
if (i < a_cnt) {
#pragma unroll
for (int h = 0; h < 2; h++) {
int r = r_base + 16 * i + 8 * h;
if (r < m_cnt) {
float w = c.wgt[it.p_off + r];
uint32_t* o = (uint32_t*)(c.out + (long)(it.p_off + r) * c.ldO + n0 + cb + c_lane);
#pragma unroll
for (int j = 0; j < 4; j++) {
if (cb + 8 * j < n_cnt) {
float g0 = acc[i][j][2 * h], g1 = acc[i][j][2 * h + 1];
float u0 = acc[i][j + 4][2 * h], u1 = acc[i][j + 4][2 * h + 1];
float h0 = g0 / (1.f + __expf(-g0)) * u0 * w; // silu(gate) * up * route weight
float h1 = g1 / (1.f + __expf(-g1)) * u1 * w;
o[4 * j] = float_to_bf16(h0) | (float_to_bf16(h1) << 16);
}
}
}
}
}
}
} else {
const int cb = wn * 64;
#pragma unroll
for (int i = 0; i < 4; i++) {
if (i < a_cnt) {
#pragma unroll
for (int h = 0; h < 2; h++) {
int r = r_base + 16 * i + 8 * h;
if (r < m_cnt) {
long orow = (long)c.out_row[it.p_off + r];
uint32_t* o = (uint32_t*)(c.out + orow * c.ldO + n0 + cb + c_lane);
#pragma unroll
for (int j = 0; j < 8; j++)
if (cb + 8 * j < n_cnt)
o[4 * j] = float_to_bf16(acc[i][j][2 * h]) | (float_to_bf16(acc[i][j][2 * h + 1]) << 16);
}
}
}
}
}
}
template<int STAGE>
__global__ void __launch_bounds__(NTHR)
moe_k(const uint16_t* __restrict__ A, long ldA, const int* __restrict__ a_row,
const uint16_t* __restrict__ Wr, const uint16_t* __restrict__ Ws, int E, int K, int NOUT,
const Item* __restrict__ items, int* queue,
uint16_t* __restrict__ out, long ldO, const int* __restrict__ out_row,
const float* __restrict__ wgt)
{
extern __shared__ __align__(128) unsigned char smem[];
int* s_q = (int*)(smem + SMEM_TILES);
KCtx c;
c.A = A; c.ldA = ldA; c.a_row = a_row; c.Wr = Wr; c.Ws = Ws; c.E = E; c.K = K; c.NOUT = NOUT;
c.out = out; c.ldO = ldO; c.out_row = out_row; c.wgt = wgt;
c.tid = threadIdx.x; c.lane = c.tid & 31; c.warp = c.tid >> 5;
c.sm_base = (unsigned)__cvta_generic_to_shared(smem);
const int qcount = queue[1];
while (true) {
if (c.tid == 0) *s_q = atomicAdd(queue, 1);
__syncthreads();
const int q = *s_q;
if (q >= qcount) return;
const Item it = items[q];
const int WN = (it.pk >> 25) & 7;
if (WN == 2) run_item<STAGE, 4, 2>(c, it);
else if (WN == 4) run_item<STAGE, 2, 4>(c, it);
else run_item<STAGE, 1, 5>(c, it);
}
}
// out[t][c] = sum over valid slots s of scratch[t][s][c]
__global__ void combine_k(const uint16_t* __restrict__ scr, const unsigned* __restrict__ vmask,
uint16_t* __restrict__ out, int T, int H, int SL)
{
const int cpr = H / 8;
long i = (long)blockIdx.x * blockDim.x + threadIdx.x;
if (i >= (long)T * cpr) return;
int t = (int)(i / cpr), c = (int)(i - (long)t * cpr);
const uint4* base = (const uint4*)(scr + ((long)t * SL) * H) + c;
const unsigned vm = vmask[t];
float acc[8] = {0, 0, 0, 0, 0, 0, 0, 0};
for (int s = 0; s < SL; s++) {
if (!((vm >> s) & 1u)) continue;
uint4 v = base[(long)s * cpr];
uint32_t w[4] = {v.x, v.y, v.z, v.w};
#pragma unroll
for (int j = 0; j < 4; j++) {
acc[2 * j] += bf16_to_float((uint16_t)(w[j] & 0xFFFF));
acc[2 * j + 1] += bf16_to_float((uint16_t)(w[j] >> 16));
}
}
uint4 o;
o.x = float_to_bf16(acc[0]) | (float_to_bf16(acc[1]) << 16);
o.y = float_to_bf16(acc[2]) | (float_to_bf16(acc[3]) << 16);
o.z = float_to_bf16(acc[4]) | (float_to_bf16(acc[5]) << 16);
o.w = float_to_bf16(acc[6]) | (float_to_bf16(acc[7]) << 16);
((uint4*)(out + (long)t * H))[c] = o;
}
} // namespace
// ---------------------------------------------------------------------------
// host entry points (plain C ABI, called from the pybind wrapper)
// ---------------------------------------------------------------------------
extern "C" int kb_moe_init(int* num_sm)
{
int dev = 0;
if (cudaGetDevice(&dev) != cudaSuccess) return 1;
cudaDeviceProp p;
if (cudaGetDeviceProperties(&p, dev) != cudaSuccess) return 2;
if (p.major < 8) return 3; // needs cp.async + bf16 mma.sync
if ((size_t)p.sharedMemPerBlockOptin < (size_t)SMEM_TOTAL) return 4;
if (cudaFuncSetAttribute(moe_k<1>, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_TOTAL) != cudaSuccess) return 5;
if (cudaFuncSetAttribute(moe_k<2>, cudaFuncAttributeMaxDynamicSharedMemorySize, SMEM_TOTAL) != cudaSuccess) return 6;
*num_sm = p.multiProcessorCount;
return 0;
}
// prep + stage 1 + stage 2 (scratch[token][slot][H] holds every expert's contribution)
extern "C" int kb_moe_stages(const void* x, const void* ids, const void* wts,
const void* w1r, const void* w2r, const void* w1s, const void* w2s,
void* tok, void* dst, void* wgt, void* vmask,
void* items1, void* items2, void* queues, void* hbuf, void* scr,
int T, int topk, int E, int NS, int H, int I, int G, int th1, int th2, void* stream_)
{
cudaStream_t stream = (cudaStream_t)stream_;
int* q1 = (int*)queues; int* q2 = q1 + 2;
prep_k<<<1, 1024, 0, stream>>>((const long long*)ids, (const uint16_t*)wts, T, topk, E, NS,
(int*)tok, (int*)dst, (float*)wgt, (unsigned*)vmask,
(Item*)items1, (Item*)items2, q1, q2, I / 8, H / 8, G, th1, th2);
// stage 1: h[pair][I] = silu(x W_gate^T) * (x W_up^T) * route_weight
moe_k<1><<<G, NTHR, SMEM_TOTAL, stream>>>((const uint16_t*)x, (long)H, (const int*)tok,
(const uint16_t*)w1r, (const uint16_t*)w1s, E, H, I,
(const Item*)items1, q1, (uint16_t*)hbuf, (long)I, nullptr, (const float*)wgt);
// stage 2: scratch[token][slot][H] = h W_down^T
moe_k<2><<<G, NTHR, SMEM_TOTAL, stream>>>((const uint16_t*)hbuf, (long)I, nullptr,
(const uint16_t*)w2r, (const uint16_t*)w2s, E, I, H,
(const Item*)items2, q2, (uint16_t*)scr, (long)H, (const int*)dst, (const float*)wgt);
cudaError_t err = cudaGetLastError();
if (err != cudaSuccess) { fprintf(stderr, "kb_moe_stages: %s\n", cudaGetErrorString(err)); return 1; }
return 0;
}
extern "C" int kb_moe_combine(const void* scr, const void* vmask, void* out, int T, int H, int SL, void* stream_)
{
cudaStream_t stream = (cudaStream_t)stream_;
const long n = (long)T * (H / 8);
combine_k<<<(unsigned)((n + 255) / 256), 256, 0, stream>>>((const uint16_t*)scr, (const unsigned*)vmask, (uint16_t*)out, T, H, SL);
cudaError_t err = cudaGetLastError();
if (err != cudaSuccess) { fprintf(stderr, "kb_moe_combine: %s\n", cudaGetErrorString(err)); return 1; }
return 0;
}
"""
_CPP_SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include <unordered_map>
extern "C" int kb_moe_init(int* num_sm);
extern "C" int kb_moe_stages(const void* x, const void* ids, const void* wts,
const void* w1r, const void* w2r, const void* w1s, const void* w2s,
void* tok, void* dst, void* wgt, void* vmask,
void* items1, void* items2, void* queues, void* hbuf, void* scr,
int T, int topk, int E, int NS, int H, int I, int G, int th1, int th2, void* stream);
extern "C" int kb_moe_combine(const void* scr, const void* vmask, void* out, int T, int H, int SL, void* stream);
namespace {
constexpr int64_t kMTMAX = 256; // must match MTMAX in the CUDA source
constexpr int kTH1 = 64, kTH2 = 128; // remainder-tile layout thresholds (tokens): 1x5 / 2x4, else 4x2
struct Buf {
torch::Tensor t; int64_t cap = -1;
void* get(int64_t n, torch::ScalarType dt, const torch::Device& dev) {
if (cap < n) { t = torch::Tensor(); t = torch::empty({n}, torch::TensorOptions().dtype(dt).device(dev)); cap = n; }
return t.data_ptr();
}
};
struct DevState {
int num_sm = 0; // 0: not initialised, < 0: unsupported device
Buf tok, dst, wgt, vmask, items1, items2, queues, hbuf, scr; // grow-only scratch
};
// Intentionally leaked: the scratch tensors must not be destroyed during static teardown.
std::unordered_map<int, DevState>& dev_states() {
static auto* m = new std::unordered_map<int, DevState>();
return *m;
}
bool is_w(const torch::Tensor& w, const torch::Device& dev, int64_t a, int64_t b, int64_t c) {
return w.defined() && w.dim() == 3 && w.size(0) == a && w.size(1) == b && w.size(2) == c &&
w.scalar_type() == torch::kBFloat16 && w.device() == dev && w.is_contiguous();
}
} // namespace
// Returns the (T, H) bf16 output, or an undefined tensor (None) when the inputs are outside
// what the CUDA kernels cover (the caller then uses the generic path).
torch::Tensor moe_forward(torch::Tensor x, torch::Tensor ids, torch::Tensor wts,
torch::Tensor w1r, torch::Tensor w2r, torch::Tensor w1s, torch::Tensor w2s,
int64_t E, int64_t topk, int64_t NS, int64_t H, int64_t I) {
if (!x.defined() || !x.is_cuda() || x.dim() != 2 || x.scalar_type() != torch::kBFloat16) return torch::Tensor();
const int64_t T = x.size(0), SL = topk + NS;
const torch::Device dev = x.device();
if (T <= 0 || x.size(1) != H) return torch::Tensor();
if (H % 64 || I % 64 || H < 64 || I < 64 || H > 32768 || I > 32768) return torch::Tensor();
if (E < 1 || NS < 0 || topk < 0 || E + NS > 512 || SL < 1 || SL > 32 || T * SL >= (1LL << 28)) return torch::Tensor();
if (T * SL * (std::max(H, I) / 8) >= (1LL << 32)) return torch::Tensor(); // 32-bit row offsets (16 B units)
if (ids.dim() != 2 || ids.size(0) != T || ids.size(1) != topk || ids.device() != dev) return torch::Tensor();
if (wts.dim() != 2 || wts.size(0) != T || wts.size(1) != topk || wts.device() != dev) return torch::Tensor();
if (!is_w(w1r, dev, E, 2 * I, H) || !is_w(w2r, dev, E, H, I)) return torch::Tensor();
if (NS > 0 && (!is_w(w1s, dev, NS, 2 * I, H) || !is_w(w2s, dev, NS, H, I))) return torch::Tensor();
c10::cuda::CUDAGuard guard(dev);
DevState& st = dev_states()[dev.index()];
if (st.num_sm == 0) { int n = 0; int rc = kb_moe_init(&n); st.num_sm = (rc == 0 && n > 0) ? n : -1; }
if (st.num_sm < 0) return torch::Tensor();
const int G = st.num_sm;
if (!x.is_contiguous()) x = x.contiguous();
if (ids.scalar_type() != torch::kLong) ids = ids.to(torch::kLong);
if (!ids.is_contiguous()) ids = ids.contiguous();
if (wts.scalar_type() != torch::kBFloat16) wts = wts.to(torch::kBFloat16);
if (!wts.is_contiguous()) wts = wts.contiguous();
const int64_t P = T * SL;
const int64_t n_items = (std::max(I / 64, H / 128) + 1) * (P / kMTMAX + 2 * (E + NS) + 2) + 2 * G + 64;
void* tok = st.tok.get(P, torch::kInt32, dev);
void* dst = st.dst.get(P, torch::kInt32, dev);
void* wgt = st.wgt.get(P, torch::kFloat32, dev);
void* vmask = st.vmask.get(T, torch::kInt32, dev);
void* items1 = st.items1.get(4 * n_items, torch::kInt32, dev);
void* items2 = st.items2.get(4 * n_items, torch::kInt32, dev);
void* queues = st.queues.get(4, torch::kInt32, dev);
void* hbuf = st.hbuf.get(P * I, torch::kBFloat16, dev);
void* scr = st.scr.get(P * H, torch::kBFloat16, dev);
void* stream = (void*)at::cuda::getCurrentCUDAStream(dev.index()).stream();
int rc = kb_moe_stages(x.data_ptr(), ids.data_ptr(), wts.data_ptr(),
w1r.data_ptr(), w2r.data_ptr(), NS > 0 ? w1s.data_ptr() : nullptr, NS > 0 ? w2s.data_ptr() : nullptr,
tok, dst, wgt, vmask, items1, items2, queues, hbuf, scr,
(int)T, (int)topk, (int)E, (int)NS, (int)H, (int)I, G, kTH1, kTH2, stream);
if (rc != 0) throw std::runtime_error("glm52 fused MoE: CUDA launch failed (stages)");
// the output is allocated while the GPU is already busy with the stages
torch::Tensor out = torch::empty({T, H}, torch::TensorOptions().dtype(torch::kBFloat16).device(dev));
rc = kb_moe_combine(scr, vmask, out.data_ptr(), (int)T, (int)H, (int)SL, stream);
if (rc != 0) throw std::runtime_error("glm52 fused MoE: CUDA launch failed (combine)");
return out;
}
"""
_FWD = None
def _load():
"""Build (once, cached on disk by torch) the CUDA extension and return its entry point."""
global _FWD
if _FWD is None:
ext = torch.utils.cpp_extension.load_inline(
name="kb_glm52_fused_moe",
cpp_sources=[_CPP_SRC],
cuda_sources=[_CUDA_SRC],
functions=["moe_forward"],
extra_cuda_cflags=["-O3"],
verbose=False,
)
_FWD = ext.moe_forward
return _FWD
class Model(nn.Module):
def __init__(
self,
T: int,
E: int,
top_k: int,
n_shared: int,
H: int,
I: int,
):
super().__init__()
self.T, self.E, self.top_k = T, E, top_k
self.n_shared, self.H, self.I = n_shared, H, I
self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
for p in self.parameters():
nn.init.normal_(p, std=0.02)
# -- generic (slow) path for shapes / devices the CUDA kernels do not cover ------------
def _forward_generic(self, x, expert_ids, expert_weights):
def ffn(xe, w1, w2):
half = w1.shape[0] // 2
h = F.silu(xe @ w1[:half].float().T) * (xe @ w1[half:].float().T)
return h @ w2.float().T
xf = x.float()
out = torch.zeros(x.shape[0], x.shape[1], device=x.device, dtype=torch.float32)
for s in range(self.n_shared):
out = out + ffn(xf, self.w1_shared[s], self.w2_shared[s])
wts = expert_weights.float()
for e in range(self.E):
token_idx, k_idx = (expert_ids == e).nonzero(as_tuple=True)
if token_idx.numel() == 0:
continue
y = ffn(xf[token_idx], self.w1_routed[e], self.w2_routed[e])
out.index_add_(0, token_idx, y * wts[token_idx, k_idx].unsqueeze(1))
return out.to(torch.bfloat16)
def forward(
self,
x: torch.Tensor,
expert_ids: torch.Tensor,
expert_weights: torch.Tensor,
) -> torch.Tensor:
"""
x: (T, H) bf16
expert_ids: (T, top_k) int64 in [0, E)
expert_weights: (T, top_k) bf16 (routed only; shared is unweighted sum)
returns: (T, H) bf16
"""
if x.is_cuda:
fwd = _FWD or _load()
out = fwd(x, expert_ids, expert_weights,
self.w1_routed, self.w2_routed, self.w1_shared, self.w2_shared,
self.E, self.top_k, self.n_shared, self.H, self.I)
if out is not None:
return out
return self._forward_generic(x, expert_ids, expert_weights)
def get_init_inputs():
return [T, E, top_k, n_shared, H, I]
def get_inputs():
x = torch.randn(T, H, dtype=torch.bfloat16)
logits = torch.randn(T, E) + torch.linspace(0.3, 0.0, E).unsqueeze(0)
vals, ids = torch.topk(logits, k=top_k, dim=-1)
weights = torch.softmax(vals, dim=-1).to(torch.bfloat16)
return [x, ids.to(torch.int64), weights]
20261109_143718_claude_claude-fable-5-2_01_glm52_fused_moe