KernelBench cuda · RTX PRO 6000
GLM-5.3 Fused MoE DeepSeek V4.1 Flash (1017)
9.65%geomean peak fraction across shapes
manually audited: clean
Two grouped bf16 tensor-core GEMMs (mma.sync m16n8k16) over expert-sorted rows with the shared expert as an extra group: the up GEMM fuses silu(gate)*up, the down GEMM scales each row by its routing weight, and a last kernel sums the top-k rows plus the shared row. Routing histogram, prefix sums and scatter are recomputed every call; the intermediate h and per-expert outputs are rounded to bf16 before the fp32 final sum.
harnessor-fableagent session4h 54mtotal wall5h 8mcheck5mbenchmark5moutput tokens353,020cost$22.50gpu-lock wait1h 46mgpu-lock held60mregimecompute
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
shape 010.407 ms35.7%1.25 TB/s · 69% of 1.8 TB/s HBM · also 178 TFLOPS (36% of compute)
shape 110.367 ms36.1%1.25 TB/s · 70% of 1.8 TB/s HBM · also 180 TFLOPS (36% of compute)
shape 20.403 ms0.2%32.08 TB/s · 100% of 1.8 TB/s HBM · also 1 TFLOPS (0% of compute)
shape 313.995 ms53.0%265 TFLOPS · 53% of 500 TF bf16 peak · also 0.93 TB/s (52% of HBM)
shape 48.755 ms5.3%1.48 TB/s · 82% of 1.8 TB/s HBM · also 26 TFLOPS (5% of compute)
shape 59.094 ms10.0%1.42 TB/s · 79% of 1.8 TB/s HBM · also 50 TFLOPS (10% of compute)
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(35.7% · 36.1% · 0.2% · 53.0% · 5.3% · 10.0%) = 9.6%
Kernel source (redacted)
"""GLM-5.2 fused MoE layer, hand-written CUDA (SM120 Blackwell).
Structure
---------
Everything is expressed as two grouped GEMMs over a "sorted pair" row space:
rows = [expert 0 tokens][expert 1 tokens] ... [expert E-1 tokens][shared tokens]
* up GEMM : h[row, :I] = silu(x[t] @ gate_e) * (x[t] @ up_e) (fused epilogue)
* down GEMM : y[row, :H] = w[row] * (h[row] @ down_e^T) (fused scaling)
The shared expert is simply group index E with weight 1.0 and T rows, so a
single launch covers routed + shared experts. A final reduction kernel sums the
top_k routed contributions plus the shared one:
out[t] = sum_k y[pos[t,k]] + y[shared_off + t]
The GEMM core is an Ampere-style bf16 tensor-core kernel (mma.sync.m16n8k16)
with cp.async multi-stage pipelining, 128B-XOR-swizzled shared memory and a
coalesced shared-memory epilogue. Both operands are K-major (NT GEMM), so the
same ldmatrix pattern serves A and B.
Real CUDA only: no Triton, no vLLM, no torch matmul on the hot path.
"""
from __future__ import annotations
import os
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
_CUDA_SRC = r"""
#include <torch/extension.h>
#include <ATen/cuda/CUDAContext.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <stdint.h>
using bf16_t = __nv_bfloat16;
using u32 = uint32_t;
#define DI __device__ __forceinline__
DI u32 sm_addr(const void* p) { return (u32)__cvta_generic_to_shared(p); }
DI void cp_async16(u32 dst, const void* src) {
asm volatile("cp.async.cg.shared.global [%0], [%1], 16;" ::"r"(dst), "l"(src));
}
DI void cp_commit() { asm volatile("cp.async.commit_group;"); }
template <int N>
DI void cp_wait() { asm volatile("cp.async.wait_group %0;" ::"n"(N)); }
DI void ldm_x4(u32& a, u32& b, u32& c, u32& d, u32 addr) {
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];"
: "=r"(a), "=r"(b), "=r"(c), "=r"(d)
: "r"(addr));
}
DI void mma16816(float* c, const u32* a, const u32* b) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0,%1,%2,%3},{%4,%5,%6,%7},{%8,%9},{%0,%1,%2,%3};"
: "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
}
DI float silu_f(float v) { return v / (1.f + __expf(-v)); }
// ---------------------------------------------------------------------------
// routing / packing
// ---------------------------------------------------------------------------
__global__ void k_hist(const long* __restrict__ ids, int npairs, int E, int T,
int* __restrict__ counts) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
if (i == 0) counts[E] = T; // shared expert group carries every token
int stride = gridDim.x * blockDim.x;
for (; i < npairs; i += stride) atomicAdd(&counts[(int)ids[i]], 1);
}
// one block: padded offsets + block (tile-row) map
__global__ void k_prefix(const int* __restrict__ counts, int E, int BM,
int* __restrict__ pad_off, int* __restrict__ cursor,
int* __restrict__ block_expert,
int* __restrict__ block_row0,
int* __restrict__ block_nrows,
int* __restrict__ meta) {
int g = threadIdx.x;
if (g <= E) {
int off = 0, blk = 0;
for (int i = 0; i < g; i++) {
int c = counts[i];
off += c;
blk += (c + BM - 1) / BM;
}
int c = counts[g];
pad_off[g] = off;
cursor[g] = off;
if (g == E) pad_off[E + 1] = off + c;
int nb = (c + BM - 1) / BM;
for (int i = 0; i < nb; i++) {
block_expert[blk + i] = g;
block_row0[blk + i] = off + i * BM;
int rem = c - i * BM;
block_nrows[blk + i] = rem < BM ? rem : BM;
}
}
__syncthreads();
if (g == 0) {
int tot = 0;
for (int i = 0; i <= E; i++) tot += (counts[i] + BM - 1) / BM;
meta[0] = tot;
meta[1] = pad_off[E] + counts[E];
}
}
__global__ void k_scatter(const long* __restrict__ ids, const bf16_t* __restrict__ wts,
int npairs, int topk, int T,
int* __restrict__ cursor,
int* __restrict__ sorted_token,
float* __restrict__ sorted_weight,
int* __restrict__ pos_of_pair, const int* __restrict__ pad_off,
int E) {
int i = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
int total = npairs + T;
const int sh_off = pad_off[E];
for (; i < total; i += stride) {
if (i < npairs) {
int e = (int)ids[i];
int p = atomicAdd(&cursor[e], 1);
sorted_token[p] = i / topk;
sorted_weight[p] = __bfloat162float(wts[i]);
pos_of_pair[i] = p;
} else {
int t = i - npairs;
sorted_token[sh_off + t] = t;
sorted_weight[sh_off + t] = 1.0f;
}
}
}
// ---------------------------------------------------------------------------
// grouped GEMM
// ---------------------------------------------------------------------------
struct GK {
const bf16_t* A;
const int* arow;
int lda;
const bf16_t* Br;
const bf16_t* Bs;
int ldb;
int E;
int b_delta;
const int* meta; // [nb_total, pad_total]
const int* block_expert;
const int* block_row0;
const int* block_nrows;
size_t w_stride;
bf16_t* C;
int ldc;
const float* rw;
};
template <int BM>
struct Cfg {
static constexpr int WM = (BM >= 256) ? 8 : ((BM >= 64) ? 4 : 2);
static constexpr int WT = BM / WM;
static constexpr int NW = WM * 2;
static constexpr int NT = NW * 32;
};
// MODE 0: up-projection, twin gate|up halves of OUTW columns + silu-mul epilogue
// MODE 1: down-projection, OUTW contiguous columns, scaled by routing weight
template <int BM, int BK, int STAGES, int MODE, int OUTW, int WT, int MINB>
__global__ void __launch_bounds__((BM / WT) * 64, MINB) moe_gemm_kernel(GK g) {
constexpr int WM = BM / WT; // warps along M
constexpr int NT = WM * 64; // 2 warps along N per M-warp
constexpr int NCH = BK / 8;
constexpr int ASZ = BM * BK;
// MODE 0 stages the twin gate|up halves of OUTW columns each (2*OUTW rows);
// MODE 1 stages OUTW contiguous columns.
constexpr int BROW = (MODE == 0) ? (2 * OUTW) : OUTW;
constexpr int BSZ = BROW * BK;
constexpr int CW = OUTW / 2; // columns per warp
constexpr int NF = (MODE == 0) ? (2 * CW / 8) : (CW / 8); // B fragments
extern __shared__ __align__(16) bf16_t smem[];
bf16_t* sA = smem;
bf16_t* sB = smem + STAGES * ASZ;
// persistent CTAs: walk the (row-tile, n-tile) work list so the grid is
// exactly the machine width and no partial wave is left at the end.
const int ntiles = g.ldc / OUTW;
const int nitems = g.meta[0] * ntiles;
for (int item = blockIdx.x; item < nitems; item += gridDim.x) {
const int b = item / ntiles;
const int j = item - b * ntiles;
const int n_base = j * OUTW;
const int gid = g.block_expert[b];
const int row0 = g.block_row0[b];
const int nrows = g.block_nrows[b];
const int pad_total = g.meta[1];
const bf16_t* Bg = (gid < g.E) ? (g.Br + (size_t)gid * g.w_stride) : g.Bs;
const int lda = g.lda;
const int ldb = g.ldb;
const int K = ldb;
const int lane = threadIdx.x & 31;
const int warp = threadIdx.x >> 5;
const int wm = warp >> 1;
const int wn = warp & 1;
const int cl = lane & 15;
const int ck = lane >> 4;
auto load_stage = [&](int s, int k0) {
bf16_t* dA = sA + s * ASZ;
bf16_t* dB = sB + s * BSZ;
#pragma unroll
for (int i = 0; i < (BM * NCH + NT - 1) / NT; i++) {
int idx = threadIdx.x + i * NT;
if (BM * NCH % NT == 0 || idx < BM * NCH) {
int row = idx / NCH, ch = idx % NCH;
int grow = row0 + row;
int tok = (MODE == 0) ? ((grow < pad_total) ? g.arow[grow] : 0) : grow;
cp_async16(sm_addr(dA + row * BK + ((ch ^ (row & (NCH - 1))) * 8)),
g.A + (size_t)tok * lda + k0 + ch * 8);
}
}
#pragma unroll
for (int i = 0; i < (BROW * NCH + NT - 1) / NT; i++) {
int idx = threadIdx.x + i * NT;
if (BROW * NCH % NT == 0 || idx < BROW * NCH) {
int row = idx / NCH, ch = idx % NCH;
int grow = n_base + row;
if (MODE == 0 && row >= OUTW) grow += g.b_delta - OUTW;
cp_async16(sm_addr(dB + row * BK + ((ch ^ (row & (NCH - 1))) * 8)),
Bg + (size_t)grow * ldb + k0 + ch * 8);
}
}
};
float acc[WT / 16][NF][4];
#pragma unroll
for (int mi = 0; mi < WT / 16; mi++)
#pragma unroll
for (int nf = 0; nf < NF; nf++)
#pragma unroll
for (int e = 0; e < 4; e++) acc[mi][nf][e] = 0.f;
const int nk = K / BK;
for (int s = 0; s < STAGES - 1; s++) {
if (s < nk) load_stage(s, s * BK);
cp_commit();
}
cp_wait<STAGES - 2>();
__syncthreads();
for (int ki = 0; ki < nk; ki++) {
int s = ki % STAGES;
int pf = ki + STAGES - 1;
if (pf < nk) load_stage(pf % STAGES, pf * BK);
cp_commit();
const bf16_t* pA = sA + s * ASZ;
const bf16_t* pB = sB + s * BSZ;
u32 afr[WT / 16][4];
u32 bfr[NF][2];
#pragma unroll
for (int kk = 0; kk < BK / 16; kk++) {
const int kc = kk * 2;
#pragma unroll
for (int mi = 0; mi < WT / 16; mi++) {
int row = wm * WT + mi * 16 + cl;
int ch = kc + ck;
ldm_x4(afr[mi][0], afr[mi][1], afr[mi][2], afr[mi][3],
sm_addr(pA + row * BK + ((ch ^ (row & (NCH - 1))) * 8)));
}
#pragma unroll
for (int f = 0; f < NF / 2; f++) {
int base;
if (MODE == 0)
base = (f < NF / 4) ? (wn * CW + f * 16)
: (OUTW + wn * CW + (f - NF / 4) * 16);
else
base = wn * CW + f * 16;
int row = base + (lane & 7) + (((lane >> 4) & 1) * 8);
int ch = kc + ((lane >> 3) & 1);
u32 d0, d1, d2, d3;
ldm_x4(d0, d1, d2, d3, sm_addr(pB + row * BK + ((ch ^ (row & (NCH - 1))) * 8)));
bfr[2 * f][0] = d0;
bfr[2 * f][1] = d1;
bfr[2 * f + 1][0] = d2;
bfr[2 * f + 1][1] = d3;
}
#pragma unroll
for (int mi = 0; mi < WT / 16; mi++)
#pragma unroll
for (int nf = 0; nf < NF; nf++) mma16816(acc[mi][nf], afr[mi], bfr[nf]);
}
cp_wait<STAGES - 2>();
__syncthreads();
}
cp_wait<0>();
// ---------------- epilogue ----------------
__syncthreads();
bf16_t* sEpi = smem; // reuse pipeline smem
#pragma unroll
for (int mi = 0; mi < WT / 16; mi++) {
int r = wm * WT + mi * 16 + (lane >> 2);
float w0 = 1.f, w1 = 1.f;
if (MODE == 1) {
w0 = g.rw[row0 + r];
w1 = g.rw[row0 + r + 8];
}
#pragma unroll
for (int nf = 0; nf < CW / 8; nf++) {
int c = wn * CW + nf * 8 + (lane & 3) * 2;
float v0, v1, v2, v3;
if (MODE == 0) {
v0 = silu_f(acc[mi][nf][0]) * acc[mi][nf + NF / 2][0];
v1 = silu_f(acc[mi][nf][1]) * acc[mi][nf + NF / 2][1];
v2 = silu_f(acc[mi][nf][2]) * acc[mi][nf + NF / 2][2];
v3 = silu_f(acc[mi][nf][3]) * acc[mi][nf + NF / 2][3];
} else {
v0 = acc[mi][nf][0] * w0;
v1 = acc[mi][nf][1] * w0;
v2 = acc[mi][nf][2] * w1;
v3 = acc[mi][nf][3] * w1;
}
__nv_bfloat162 p0 = __floats2bfloat162_rn(v0, v1);
__nv_bfloat162 p1 = __floats2bfloat162_rn(v2, v3);
*reinterpret_cast<u32*>(sEpi + r * OUTW + c) = *reinterpret_cast<u32*>(&p0);
*reinterpret_cast<u32*>(sEpi + (r + 8) * OUTW + c) = *reinterpret_cast<u32*>(&p1);
}
}
__syncthreads();
constexpr int TPR = OUTW * 2 / 16; // 16B chunks per output row
constexpr int RPP = NT / TPR;
const int colbase = j * OUTW;
#pragma unroll
for (int r0i = 0; r0i < BM; r0i += RPP) {
int rr = r0i + threadIdx.x / TPR;
int c = (threadIdx.x % TPR) * 8;
if (rr < nrows) {
uint4 v = *reinterpret_cast<const uint4*>(sEpi + rr * OUTW + c);
*reinterpret_cast<uint4*>(g.C + (size_t)(row0 + rr) * g.ldc + colbase + c) = v;
}
}
__syncthreads();
}
}
// ---------------------------------------------------------------------------
// final reduction: out[t] = sum_k y[pos[t,k]] + y[shared_row + t]
// ---------------------------------------------------------------------------
__global__ void k_reduce(const bf16_t* __restrict__ y, const int* __restrict__ pos,
int topk, int H, const int* __restrict__ meta,
bf16_t* __restrict__ out) {
int t = blockIdx.x;
const int* p = pos + (size_t)t * topk;
const int sh_off = meta[1] - gridDim.x; // shared rows are the last T rows
const bf16_t* sh = y + (size_t)(sh_off + t) * H;
bf16_t* o = out + (size_t)t * H;
const int tid = blockIdx.y * blockDim.x + threadIdx.x;
const int tstride = gridDim.y * blockDim.x;
for (int c = tid * 8; c < H; c += tstride * 8) {
uint4 acc = *reinterpret_cast<const uint4*>(sh + c);
__nv_bfloat162* a2 = reinterpret_cast<__nv_bfloat162*>(&acc);
float s[8];
#pragma unroll
for (int i = 0; i < 4; i++) {
float2 f = __bfloat1622float2(a2[i]);
s[2 * i] = f.x;
s[2 * i + 1] = f.y;
}
for (int k = 0; k < topk; k++) {
uint4 v = *reinterpret_cast<const uint4*>(y + (size_t)p[k] * H + c);
__nv_bfloat162* v2 = reinterpret_cast<__nv_bfloat162*>(&v);
#pragma unroll
for (int i = 0; i < 4; i++) {
float2 f = __bfloat1622float2(v2[i]);
s[2 * i] += f.x;
s[2 * i + 1] += f.y;
}
}
uint4 res;
__nv_bfloat162* r2 = reinterpret_cast<__nv_bfloat162*>(&res);
#pragma unroll
for (int i = 0; i < 4; i++) r2[i] = __floats2bfloat162_rn(s[2 * i], s[2 * i + 1]);
*reinterpret_cast<uint4*>(o + c) = res;
}
}
// ---------------------------------------------------------------------------
// host side
// ---------------------------------------------------------------------------
struct Kcfg { int bm, bk, st, w0, wt, w1; };
static Kcfg choose_kcfg(int T, int topk, int E) {
if (const char* e = getenv("GLM52_CFG")) {
Kcfg c{0, 0, 0, 64, 32, 128};
if (sscanf(e, "%d:%d:%d:%d:%d:%d", &c.bm, &c.bk, &c.st, &c.w0, &c.wt, &c.w1) >= 3) {
if (c.w0 == 0) c.w0 = 64;
if (c.wt == 0) c.wt = 32;
if (c.w1 == 0) c.w1 = 128;
return c;
}
}
if (const char* e = getenv("GLM52_BM")) {
Kcfg c{atoi(e), 64, 0, 64, 32, 128};
if (const char* s = getenv("GLM52_ST")) c.st = atoi(s);
return c;
}
// rows per expert decides the regime:
// a handful of rows -> small tiles (everything is DRAM-streaming bound and
// the padding waste of a 128-row tile is pure loss)
// otherwise -> 128-row tiles, 128-wide up / 256-wide down: the
// widest slices that still fit the smem budget, which
// minimises weight re-reads per flop.
double ne = (double)T * (double)topk / (double)E;
if (ne <= 8.0) return Kcfg{32, 64, 4, 64, 16, 128};
return Kcfg{128, 64, 2, 128, 32, 256};
}
#define NT_OF(BM, WT) (((BM) / (WT)) * 64)
template <int BM, int BK, int STAGES, int MODE, int OUTW, int WT, int MINB>
static void launch_one(GK& g, int n_tiles, int ub, size_t smem, cudaStream_t st) {
auto kfn = moe_gemm_kernel<BM, BK, STAGES, MODE, OUTW, WT, MINB>;
constexpr int NT = NT_OF(BM, WT);
// Persistent grid: one CTA per resident slot, each walking the work list.
static int nblk = 0;
if (!nblk) {
cudaFuncSetAttribute(kfn, cudaFuncAttributeMaxDynamicSharedMemorySize, (int)smem);
int dev = 0, sms = 0, occ = 0;
cudaGetDevice(&dev);
cudaDeviceGetAttribute(&sms, cudaDevAttrMultiProcessorCount, dev);
cudaOccupancyMaxActiveBlocksPerMultiprocessor(&occ, kfn, NT, smem);
if (occ < 1) occ = 1;
nblk = sms * occ;
if (getenv("GLM52_DBG")) {
fprintf(stderr, "[glm52] BM=%d BK=%d ST=%d MODE=%d OUTW=%d WT=%d smem=%zu nt=%d occ=%d blk=%d\n",
BM, BK, STAGES, MODE, OUTW, WT, smem, NT, occ, nblk);
}
}
int items = n_tiles * ub;
kfn<<<(nblk < items ? nblk : items), NT, smem, st>>>(g);
}
#define PIPE_SMEM(BM, BROW, BK, ST) (((size_t)(ST) * ((BM) + (BROW)) * (BK) * 2))
#define EPI_SMEM(BM, W) ((size_t)(BM) * (W) * 2)
#define MAX2(a, b) ((a) > (b) ? (a) : (b))
template <int BM, int BK, int STAGES, int W0, int W1, int WT, int MB0, int MB1>
static void launch_mode(GK& g, int n_tiles, int ub, int mode, cudaStream_t st) {
if (mode == 0) {
size_t smem = MAX2(PIPE_SMEM(BM, 2 * W0, BK, STAGES), EPI_SMEM(BM, W0));
launch_one<BM, BK, STAGES, 0, W0, WT, MB0>(g, n_tiles, ub, smem, st);
} else {
size_t smem = MAX2(PIPE_SMEM(BM, W1, BK, STAGES), EPI_SMEM(BM, W1));
launch_one<BM, BK, STAGES, 1, W1, WT, MB1>(g, n_tiles, ub, smem, st);
}
}
static void launch_gemm(GK& g, Kcfg c, int n_tiles, int ub, int mode, cudaStream_t st) {
int w0 = c.w0 ? c.w0 : 64;
switch (c.bm) {
case 32: launch_mode<32, 64, 4, 64, 128, 16, 1, 1>(g, n_tiles, ub, mode, st); break;
case 64:
if (c.st == 2) launch_mode<64, 64, 2, 64, 128, 32, 2, 2>(g, n_tiles, ub, mode, st);
else launch_mode<64, 64, 4, 64, 128, 32, 1, 1>(g, n_tiles, ub, mode, st);
break;
case 96:
if (c.st == 2) launch_mode<96, 64, 2, 64, 128, 32, 2, 2>(g, n_tiles, ub, mode, st);
else launch_mode<96, 64, 3, 64, 128, 32, 1, 1>(g, n_tiles, ub, mode, st);
break;
case 128:
if (c.wt == 64) {
if (c.st == 2) launch_mode<128, 64, 2, 128, 128, 64, 1, 1>(g, n_tiles, ub, mode, st);
else launch_mode<128, 64, 3, 64, 128, 64, 1, 1>(g, n_tiles, ub, mode, st);
} else if (c.bk == 32) {
// deep stage count: more prefetch in flight against L2/DRAM latency
if (c.st >= 6) launch_mode<128, 32, 6, 64, 128, 32, 1, 1>(g, n_tiles, ub, mode, st);
else launch_mode<128, 32, 4, 128, 256, 32, 1, 1>(g, n_tiles, ub, mode, st);
} else if (c.st == 2 && w0 == 128) {
if (c.w1 == 256) launch_mode<128, 64, 2, 128, 256, 32, 1, 1>(g, n_tiles, ub, mode, st);
else launch_mode<128, 64, 2, 128, 128, 32, 1, 1>(g, n_tiles, ub, mode, st);
} else if (c.st == 2) {
if (c.w1 == 256) launch_mode<128, 64, 2, 64, 256, 32, 1, 1>(g, n_tiles, ub, mode, st);
else launch_mode<128, 64, 2, 64, 128, 32, 1, 1>(g, n_tiles, ub, mode, st);
} else {
launch_mode<128, 64, 3, 64, 128, 32, 1, 1>(g, n_tiles, ub, mode, st);
}
break;
default:
launch_mode<256, 64, 2, 64, 128, 64, 1, 1>(g, n_tiles, ub, mode, st);
break;
}
}
torch::Tensor glm52_moe_forward(torch::Tensor x, torch::Tensor ids, torch::Tensor wts,
torch::Tensor w1r, torch::Tensor w2r,
torch::Tensor w1s, torch::Tensor w2s,
torch::Tensor counts, torch::Tensor pad_off,
torch::Tensor cursor, torch::Tensor block_expert,
torch::Tensor block_row0, torch::Tensor block_nrows,
torch::Tensor meta, torch::Tensor sorted_token,
torch::Tensor sorted_weight, torch::Tensor pos_of_pair,
torch::Tensor h_buf, torch::Tensor y_buf) {
const int T = (int)x.size(0);
const int H = (int)x.size(1);
const int topk = (int)ids.size(1);
const int E = (int)w1r.size(0);
const int I = (int)w2r.size(2);
const int npairs = T * topk;
cudaStream_t st = at::cuda::getCurrentCUDAStream();
int* counts_p = counts.data_ptr<int>();
int* meta_p = meta.data_ptr<int>();
cudaMemsetAsync(counts_p, 0, sizeof(int) * (E + 1), st);
{
int nb = (npairs + 255) / 256;
if (nb > 2048) nb = 2048;
k_hist<<<nb, 256, 0, st>>>(ids.data_ptr<long>(), npairs, E, T, counts_p);
}
const Kcfg kc = choose_kcfg(T, topk, E);
const int BM = kc.bm;
k_prefix<<<1, 1024, 0, st>>>(counts_p, E, BM, pad_off.data_ptr<int>(),
cursor.data_ptr<int>(), block_expert.data_ptr<int>(),
block_row0.data_ptr<int>(), block_nrows.data_ptr<int>(),
meta_p);
{
int nb = (npairs + T + 255) / 256;
if (nb > 2048) nb = 2048;
k_scatter<<<nb, 256, 0, st>>>(ids.data_ptr<long>(), (const bf16_t*)wts.data_ptr(),
npairs, topk, T, cursor.data_ptr<int>(),
sorted_token.data_ptr<int>(),
sorted_weight.data_ptr<float>(),
pos_of_pair.data_ptr<int>(), pad_off.data_ptr<int>(), E);
}
const int ub = (npairs + T) / BM + E + 2;
GK g{};
g.A = (const bf16_t*)x.data_ptr();
g.arow = sorted_token.data_ptr<int>();
g.lda = H;
g.Br = (const bf16_t*)w1r.data_ptr();
g.Bs = (const bf16_t*)w1s.data_ptr();
g.ldb = H;
g.E = E;
g.b_delta = I;
g.meta = meta_p;
g.block_expert = block_expert.data_ptr<int>();
g.block_row0 = block_row0.data_ptr<int>();
g.block_nrows = block_nrows.data_ptr<int>();
g.w_stride = (size_t)(2 * I) * H;
g.C = (bf16_t*)h_buf.data_ptr();
g.ldc = I;
g.rw = nullptr;
GK gd = g;
gd.A = (const bf16_t*)h_buf.data_ptr();
gd.arow = nullptr;
gd.lda = I;
gd.Br = (const bf16_t*)w2r.data_ptr();
gd.Bs = (const bf16_t*)w2s.data_ptr();
gd.ldb = I;
gd.b_delta = 64;
gd.w_stride = (size_t)H * I;
gd.C = (bf16_t*)y_buf.data_ptr();
gd.ldc = H;
gd.rw = sorted_weight.data_ptr<float>();
launch_gemm(g, kc, I / (kc.w0 ? kc.w0 : 64), ub, 0, st);
launch_gemm(gd, kc, H / (kc.w1 ? kc.w1 : 128), ub, 1, st);
auto out = torch::empty({T, H}, x.options());
{
int threads = 256;
int gy = (H + threads * 8 - 1) / (threads * 8);
k_reduce<<<dim3(T, gy), threads, 0, st>>>(
(const bf16_t*)y_buf.data_ptr(), pos_of_pair.data_ptr<int>(), topk, H,
meta_p, (bf16_t*)out.data_ptr());
}
return out;
}
"""
_CPP_SRC = r"""
#include <torch/extension.h>
torch::Tensor glm52_moe_forward(torch::Tensor x, torch::Tensor ids, torch::Tensor wts,
torch::Tensor w1r, torch::Tensor w2r,
torch::Tensor w1s, torch::Tensor w2s,
torch::Tensor counts, torch::Tensor pad_off,
torch::Tensor cursor, torch::Tensor block_expert,
torch::Tensor block_row0, torch::Tensor block_nrows,
torch::Tensor meta, torch::Tensor sorted_token,
torch::Tensor sorted_weight, torch::Tensor pos_of_pair,
torch::Tensor h_buf, torch::Tensor y_buf);
"""
def _build():
# torch's JIT builder shells out to `ninja`; it may live in the venv's bin
# dir without being on PATH.
import shutil, sysconfig
if shutil.which("ninja") is None:
sd = sysconfig.get_paths()["scripts"]
if os.path.exists(os.path.join(sd, "ninja")):
os.environ["PATH"] = sd + os.pathsep + os.environ.get("PATH", "")
return load_inline(
name="glm52_moe_ext",
cpp_sources=_CPP_SRC,
cuda_sources=_CUDA_SRC,
functions=["glm52_moe_forward"],
extra_cuda_cflags=["-O3", "-std=c++17", "--use_fast_math"],
verbose=False,
)
_ext = _build()
class Model(nn.Module):
def __init__(self, T: int, E: int, top_k: int, n_shared: int, H: int, I: int):
super().__init__()
self.T, self.E, self.top_k = T, E, top_k
self.n_shared, self.H, self.I = n_shared, H, I
self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
for p in self.parameters():
nn.init.normal_(p, std=0.02)
self._ws = {}
def _get_ws(self, T: int, device):
key = (T, device.index)
b = self._ws.get(key)
if b is not None:
return b
E, top_k, H, I = self.E, self.top_k, self.H, self.I
npairs = T * top_k
max_rows = npairs + T + 512
ub = max_rows // 32 + E + 4
i32 = dict(dtype=torch.int32, device=device)
b = [
torch.zeros(E + 1, **i32), # counts
torch.zeros(E + 2, **i32), # pad_off
torch.zeros(E + 2, **i32), # cursor
torch.zeros(ub, **i32), # block_expert
torch.zeros(ub, **i32), # block_row0
torch.zeros(ub, **i32), # block_nrows
torch.zeros(8, **i32), # meta
torch.zeros(max_rows, **i32), # sorted_token
torch.zeros(max_rows, dtype=torch.float32, device=device), # sorted_weight
torch.zeros(npairs, **i32), # pos_of_pair
torch.zeros(max_rows, I, dtype=torch.bfloat16, device=device), # h_buf
torch.zeros(max_rows, H, dtype=torch.bfloat16, device=device), # y_buf
]
self._ws[key] = b
return b
def forward(self, x, expert_ids, expert_weights):
T = x.shape[0]
bufs = self._get_ws(T, x.device)
return _ext.glm52_moe_forward(
x, expert_ids, expert_weights,
self.w1_routed, self.w2_routed, self.w1_shared, self.w2_shared,
*bufs,
)
def get_init_inputs():
return [4096, 256, 8, 1, 4096, 2048]
def get_inputs():
T, E, top_k, H = 4096, 256, 8, 4096
x = torch.randn(T, H, dtype=torch.bfloat16)
logits = torch.randn(T, E) + torch.linspace(0.3, 0.0, E).unsqueeze(0)
vals, ids = torch.topk(logits, k=top_k, dim=-1)
weights = torch.softmax(vals, dim=-1).to(torch.bfloat16)
return [x, ids.to(torch.int64), weights]
20261018_065729_or-fable_deepseek_deepseek-v4.1-flash-1017_01_glm52_fused_moe