KernelBench cuda · RTX PRO 6000
GLM-5.3 Fused MoE Gemini 4 Neon (XHigh)
manually audited: clean
Solo rerun of Gemini 4 Neon (XHigh) on GLM-5.3 Fused MoE (following the lock-blocked V1 0.1010 run). Wrote a self-contained 1,977-line SM120 CUDA C++/PTX/TMA kernel suite with zero CPU-GPU synchronization across four adaptive regimes: split-warp 128-thread streaming GEMV for decode (T <= 4); single-barrier hybrid cp.async + 3D-TMA (L2::evict_first when nm==1) with high-priority direct-store shared-expert overlap for small batches; disjoint 2-bucket [BM=128, BM=192] hybrid 3D-TMA with low-priority shared overlap for medium batches; and sequential direct-store shared + universal 3-bucket [BM=128, BM=96, BM=192] schedule minimizing (padded_rows, m_tiles) for large batches, all reducing via SM120 vectorized red.relaxed.gpu.global.add.v4.f32. Lands 0.1105 (#2 overall, +9.4% over V1, fastest on board at T=1, T=512, and T=1000).
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(42.0% · 41.2% · 0.3% · 52.2% · 5.9% · 11.3%) = 11.0%
Kernel source (redacted)
"""Fast SM120 Blackwell CUDA/PTX/TMA Fused MoE Layer for GLM-5.2 (v4).
Architecture:
- Self-contained CUDA C++ / inline PTX / 2D & 3D TMA kernels via load_inline.
- Zero-overhead 7-arg C++ entrypoint with O(1) static per-device workspace cache.
- 4 adaptive regimes (zero CPU-GPU sync in all regimes):
1. Decode / Tiny-T (T <= 4):
Split-warp 128-thread GEMV1 (2 warps/row contiguous streaming ld.global.cs.v4,
zero SMEM x copy, fused FP32 SwiGLU -> BF16 h) + 4-warp/row GEMV2 (3-slot
unrolled streaming loads, L1-resident h, 99% SM balance, atomic-free).
2. Small avg M_e (T*top_k <= E*48, e.g. T=512, 1000):
BM=64 BK=64 S=4 single-barrier hybrid cp.async + 3D-TMA (SWIZZLE_128B,
L2::evict_first) GEMM1 + pure 3D-TMA GEMM2; direct-store shared expert
overlapped on high-priority stream after routed GEMM1 launch (zero memset).
3. Medium avg M_e (E*48 < T*top_k <= E*180, e.g. T=4096, 4127):
Disjoint 2-bucket [BM=128 (BK=64, S=3), BM=192 (BK=32, S=4)] GPU schedule
eliminating padding compute waste and weight re-reads, using 1-barrier
hybrid cp.async + 3D-TMA (L2::evict_first when nm==1) GEMM1 + pure 3D-TMA
GEMM2; direct-store shared expert overlapped on low-priority stream after
routed GEMM1 launch (zero memset).
4. Large avg M_e (T*top_k > E*180, e.g. T=8192):
Sequential pure-TMA direct-store shared expert (zero memset, zero contention)
+ universal 3-bucket [BM=128, BM=96, BM=192] GPU schedule minimizing
(padded_rows, m_tiles) per expert with zero inner-loop branches.
- GEMM2 epilogue uses SM120 vectorized red.relaxed.gpu.global.add.v4.f32.
"""
from __future__ import annotations
import hashlib
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
_CUDA_SRC = r'''
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include <stdint.h>
#include <math.h>
namespace {
// ============================================================================
// Low-level SM120 PTX / TMA Primitives
// ============================================================================
__device__ __forceinline__ uint32_t to_smem_addr(const void* ptr) {
return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}
__device__ __forceinline__ void cp_async_16(uint32_t smem_addr, const void* gmem_ptr, int src_bytes) {
asm volatile(
"cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n"
:: "r"(smem_addr), "l"(gmem_ptr), "r"(src_bytes)
);
}
__device__ __forceinline__ void cp_async_16_full(uint32_t smem_addr, const void* gmem_ptr) {
asm volatile(
"cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\n"
:: "r"(smem_addr), "l"(gmem_ptr)
);
}
__device__ __forceinline__ void cp_async_commit() {
asm volatile("cp.async.commit_group;\n");
}
template <int N>
__device__ __forceinline__ void cp_async_wait() {
asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}
__device__ __forceinline__ void mbarrier_init(uint32_t mbar, int count) {
asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(mbar), "r"(count));
}
__device__ __forceinline__ void mbarrier_arrive_expect_tx(uint32_t mbar, int tx_bytes) {
asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" :: "r"(mbar), "r"(tx_bytes));
}
__device__ __forceinline__ void mbarrier_wait(uint32_t mbar, int phase) {
asm volatile(
"{\n"
".reg .pred P1;\n"
"LAB_WAIT:\n"
"mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n"
"@!P1 bra LAB_WAIT;\n"
"}\n"
:: "r"(mbar), "r"(phase)
);
}
__device__ __forceinline__ void tma_load_2d(
uint32_t smem_addr, const CUtensorMap* tmap, int c0, int c1, uint32_t mbar
) {
asm volatile(
"cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes "
"[%0], [%1, {%2, %3}], [%4];\n"
:: "r"(smem_addr), "l"(tmap), "r"(c0), "r"(c1), "r"(mbar)
: "memory"
);
}
__device__ __forceinline__ void tma_load_3d_hint(
uint32_t smem_addr, const CUtensorMap* tmap, int c0, int c1, int c2, uint32_t mbar, uint64_t policy
) {
asm volatile(
"cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint "
"[%0], [%1, {%2, %3, %4}], [%5], %6;\n"
:: "r"(smem_addr), "l"(tmap), "r"(c0), "r"(c1), "r"(c2), "r"(mbar), "l"(policy)
: "memory"
);
}
__device__ __forceinline__ void ldmatrix_x4(
uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t smem_addr
) {
asm volatile(
"ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
: "r"(smem_addr)
);
}
__device__ __forceinline__ void mma_m16n8k16_bf16(
float& d0, float& d1, float& d2, float& d3,
uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
uint32_t b0, uint32_t b1,
float c0, float c1, float c2, float c3
) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
: "=f"(d0), "=f"(d1), "=f"(d2), "=f"(d3)
: "r"(a0), "r"(a1), "r"(a2), "r"(a3),
"r"(b0), "r"(b1),
"f"(c0), "f"(c1), "f"(c2), "f"(c3)
);
}
__device__ __forceinline__ uint4 ld_global_cs_v4(const uint4* ptr) {
uint4 v;
asm volatile(
"ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr)
);
return v;
}
__device__ __forceinline__ void red_global_add_v4_f32(float* ptr, float4 v) {
asm volatile(
"red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n"
:: "l"(ptr), "f"(v.x), "f"(v.y), "f"(v.z), "f"(v.w)
);
}
__device__ __forceinline__ float silu_mul_f32(float g, float u) {
return (g / (1.0f + expf(-g))) * u;
}
template <int BK>
__device__ __forceinline__ int swizzle_k(int r, int c) {
if constexpr (BK == 64) {
return (r * 8 + (c ^ (r & 7))) * 16; // Matches CU_TENSOR_MAP_SWIZZLE_128B
} else {
return (r * 4 + (c ^ ((r >> 1) & 3))) * 16; // Matches CU_TENSOR_MAP_SWIZZLE_64B
}
}
// ============================================================================
// Regime 1: Decode / Tiny-T Fused MoE Kernels (T <= 4)
// ============================================================================
__device__ __forceinline__ float dot8_bf16_bf16(uint4 a, uint4 b) {
const __nv_bfloat162* ap = reinterpret_cast<const __nv_bfloat162*>(&a);
const __nv_bfloat162* bp = reinterpret_cast<const __nv_bfloat162*>(&b);
float2 a0 = __bfloat1622float2(ap[0]), b0 = __bfloat1622float2(bp[0]);
float2 a1 = __bfloat1622float2(ap[1]), b1 = __bfloat1622float2(bp[1]);
float2 a2 = __bfloat1622float2(ap[2]), b2 = __bfloat1622float2(bp[2]);
float2 a3 = __bfloat1622float2(ap[3]), b3 = __bfloat1622float2(bp[3]);
float acc = a0.x * b0.x;
acc = fmaf(a0.y, b0.y, acc);
acc = fmaf(a1.x, b1.x, acc);
acc = fmaf(a1.y, b1.y, acc);
acc = fmaf(a2.x, b2.x, acc);
acc = fmaf(a2.y, b2.y, acc);
acc = fmaf(a3.x, b3.x, acc);
acc = fmaf(a3.y, b3.y, acc);
return acc;
}
__device__ __forceinline__ float warp_reduce_sum(float v) {
#pragma unroll
for (int offset = 16; offset > 0; offset >>= 1) {
v += __shfl_down_sync(0xffffffff, v, offset);
}
return v;
}
// Split-warp 128-thread GEMV1:
// Warps 0..1 stream gate row i (64t = 1KB coalesced burst x 8 unrolled loads = 8KB);
// Warps 2..3 stream up row I+i simultaneously. Zero SMEM x copy; hits ~1546 GB/s!
__global__ __launch_bounds__(128)
void decode_gemv1_splitwarp_128t_kernel(
const __nv_bfloat16* __restrict__ x,
const int64_t* __restrict__ expert_ids,
const __nv_bfloat16* __restrict__ w1_routed,
const __nv_bfloat16* __restrict__ w1_shared,
__nv_bfloat16* __restrict__ h_out,
int T, int top_k, int n_shared, int H, int I
) {
__shared__ float s_part[4];
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
int S = n_shared + top_k;
int i = blockIdx.x % I;
int rem = blockIdx.x / I;
int slot = rem % S;
int t = rem / S;
if (t >= T) return;
int is_up = wid >> 1; // 0: warps 0,1 (gate); 1: warps 2,3 (up)
int tid64 = (wid & 1) * 32 + lane;
const __nv_bfloat16* w1_base = (slot < n_shared)
? (w1_shared + (size_t)slot * (2 * I) * H)
: (w1_routed + (size_t)expert_ids[t * top_k + (slot - n_shared)] * (2 * I) * H);
const uint4* x_vec = reinterpret_cast<const uint4*>(x + (size_t)t * H);
const uint4* w_vec = reinterpret_cast<const uint4*>(w1_base + (size_t)(is_up * I + i) * H);
int n_vec = H >> 3;
float sum = 0.0f;
int base = 0;
for (; base + 512 <= n_vec; base += 512) {
int k = base + tid64;
uint4 w0 = ld_global_cs_v4(w_vec + k);
uint4 w1 = ld_global_cs_v4(w_vec + k + 64);
uint4 w2 = ld_global_cs_v4(w_vec + k + 128);
uint4 w3 = ld_global_cs_v4(w_vec + k + 192);
uint4 w4 = ld_global_cs_v4(w_vec + k + 256);
uint4 w5 = ld_global_cs_v4(w_vec + k + 320);
uint4 w6 = ld_global_cs_v4(w_vec + k + 384);
uint4 w7 = ld_global_cs_v4(w_vec + k + 448);
sum += dot8_bf16_bf16(__ldg(x_vec + k), w0)
+ dot8_bf16_bf16(__ldg(x_vec + k + 64), w1)
+ dot8_bf16_bf16(__ldg(x_vec + k + 128), w2)
+ dot8_bf16_bf16(__ldg(x_vec + k + 192), w3)
+ dot8_bf16_bf16(__ldg(x_vec + k + 256), w4)
+ dot8_bf16_bf16(__ldg(x_vec + k + 320), w5)
+ dot8_bf16_bf16(__ldg(x_vec + k + 384), w6)
+ dot8_bf16_bf16(__ldg(x_vec + k + 448), w7);
}
for (int k = base + tid64; k < n_vec; k += 64) {
sum += dot8_bf16_bf16(__ldg(x_vec + k), ld_global_cs_v4(w_vec + k));
}
sum = warp_reduce_sum(sum);
if (lane == 0) s_part[wid] = sum;
__syncthreads();
if (tid == 0) {
h_out[(size_t)(t * S + slot) * I + i] =
__float2bfloat16_rn(silu_mul_f32(s_part[0] + s_part[1], s_part[2] + s_part[3]));
}
}
// 4-warp/row (128 threads = 1 output row j) GEMV2:
// 4096 blocks at T=1 -> 99% SM balance, 3-slot unrolled ld.global.cs.v4, L1-resident h!
__global__ __launch_bounds__(128)
void decode_gemv2_4w_1row_kernel(
const __nv_bfloat16* __restrict__ h,
const int64_t* __restrict__ expert_ids,
const __nv_bfloat16* __restrict__ expert_weights,
const __nv_bfloat16* __restrict__ w2_routed,
const __nv_bfloat16* __restrict__ w2_shared,
__nv_bfloat16* __restrict__ out,
int T, int top_k, int n_shared, int H, int I
) {
__shared__ const uint4* s_wb[64];
__shared__ float s_wt[64];
__shared__ float s_part[4];
int tid = threadIdx.x;
int wid = tid >> 5;
int lane = tid & 31;
int S = n_shared + top_k;
int j = blockIdx.x % H;
int t = blockIdx.x / H;
if (t >= T) return;
int n_vec = I >> 3;
int smem_S = (S <= 64) ? S : 64;
if (tid < smem_S) {
if (tid < n_shared) {
s_wb[tid] = reinterpret_cast<const uint4*>(w2_shared + (size_t)tid * H * I);
s_wt[tid] = 1.0f;
} else {
int ki = tid - n_shared;
int64_t e = expert_ids[t * top_k + ki];
s_wb[tid] = reinterpret_cast<const uint4*>(w2_routed + (size_t)e * H * I);
s_wt[tid] = __bfloat162float(expert_weights[t * top_k + ki]);
}
}
__syncthreads();
const __nv_bfloat16* h_t = h + (size_t)(t * S) * I;
size_t row_off = (size_t)j * n_vec;
float total = 0.0f;
if (n_vec == 256 && S <= 64) {
// Fast path for I=2048: 3-slot concurrent streaming loads (6 uint4s in flight per thread!)
size_t ro = row_off + tid;
int slot = 0;
for (; slot + 2 < S; slot += 3) {
uint4 w0a = ld_global_cs_v4(s_wb[slot] + ro);
uint4 w0b = ld_global_cs_v4(s_wb[slot] + ro + 128);
uint4 w1a = ld_global_cs_v4(s_wb[slot + 1] + ro);
uint4 w1b = ld_global_cs_v4(s_wb[slot + 1] + ro + 128);
uint4 w2a = ld_global_cs_v4(s_wb[slot + 2] + ro);
uint4 w2b = ld_global_cs_v4(s_wb[slot + 2] + ro + 128);
const uint4* h0 = reinterpret_cast<const uint4*>(h_t + (size_t)slot * 2048);
const uint4* h1 = reinterpret_cast<const uint4*>(h_t + (size_t)(slot + 1) * 2048);
const uint4* h2 = reinterpret_cast<const uint4*>(h_t + (size_t)(slot + 2) * 2048);
total = fmaf(s_wt[slot], dot8_bf16_bf16(__ldg(h0 + tid), w0a) + dot8_bf16_bf16(__ldg(h0 + tid + 128), w0b), total);
total = fmaf(s_wt[slot + 1], dot8_bf16_bf16(__ldg(h1 + tid), w1a) + dot8_bf16_bf16(__ldg(h1 + tid + 128), w1b), total);
total = fmaf(s_wt[slot + 2], dot8_bf16_bf16(__ldg(h2 + tid), w2a) + dot8_bf16_bf16(__ldg(h2 + tid + 128), w2b), total);
}
for (; slot < S; ++slot) {
uint4 w0a = ld_global_cs_v4(s_wb[slot] + ro);
uint4 w0b = ld_global_cs_v4(s_wb[slot] + ro + 128);
const uint4* h0 = reinterpret_cast<const uint4*>(h_t + (size_t)slot * 2048);
total = fmaf(s_wt[slot], dot8_bf16_bf16(__ldg(h0 + tid), w0a) + dot8_bf16_bf16(__ldg(h0 + tid + 128), w0b), total);
}
} else {
// General I / large S fallback
for (int slot = 0; slot < S; ++slot) {
const uint4* wb;
float wt;
if (slot < 64) {
wb = s_wb[slot] + row_off;
wt = s_wt[slot];
} else {
int ki = slot - n_shared;
wb = reinterpret_cast<const uint4*>(w2_routed + ((size_t)expert_ids[t * top_k + ki] * H + j) * I);
wt = __bfloat162float(expert_weights[t * top_k + ki]);
}
const uint4* hv = reinterpret_cast<const uint4*>(h_t + (size_t)slot * I);
float d = 0.0f;
for (int k = tid; k < n_vec; k += 128) {
d += dot8_bf16_bf16(__ldg(hv + k), ld_global_cs_v4(wb + k));
}
total = fmaf(wt, d, total);
}
}
total = warp_reduce_sum(total);
if (lane == 0) s_part[wid] = total;
__syncthreads();
if (tid == 0) {
out[(size_t)t * H + j] = __float2bfloat16_rn(s_part[0] + s_part[1] + s_part[2] + s_part[3]);
}
}
// ============================================================================
// GPU Routing & Multi-Bucket O(1) Tile Schedule (Zero CPU Sync)
// ============================================================================
__global__ void moe_histogram_kernel(
const int64_t* __restrict__ expert_ids,
int32_t* __restrict__ expert_counts,
int total_tokens,
int E
) {
extern __shared__ int32_t s_counts[];
for (int e = threadIdx.x; e < E; e += blockDim.x) s_counts[e] = 0;
__syncthreads();
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
for (int i = idx; i < total_tokens; i += stride) {
int e = static_cast<int>(expert_ids[i]);
atomicAdd(&s_counts[e], 1);
}
__syncthreads();
for (int e = threadIdx.x; e < E; e += blockDim.x) {
int c = s_counts[e];
if (c > 0) atomicAdd(&expert_counts[e], c);
}
}
// SCHED_MODE:
// 0 = Single bucket BM=64 (Regime 2: small avg M_e)
// 1 = Disjoint 2-bucket [BM_A=128, BM_C=192] (Regime 3: medium avg M_e)
// 2 = Universal 3-bucket [BM_A=128, BM_B=96, BM_C=192] (Regime 4: large avg M_e)
template <int SCHED_MODE>
__global__ void moe_prefix_and_schedule_kernel(
const int32_t* __restrict__ expert_counts,
int32_t* __restrict__ expert_offsets,
int32_t* __restrict__ expert_cursors,
int32_t* __restrict__ emto_a,
int32_t* __restrict__ m2e_a,
int32_t* __restrict__ emto_b,
int32_t* __restrict__ m2e_b,
int32_t* __restrict__ emto_c,
int32_t* __restrict__ m2e_c,
int E
) {
__shared__ int32_t s_wt[8], s_wa[8], s_wb[8], s_wc[8];
int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5;
int run_tok = 0, run_a = 0, run_b = 0, run_c = 0;
for (int base = 0; base < E; base += 256) {
int e = base + tid;
int c = (e < E) ? expert_counts[e] : 0;
int mta = 0, mtb = 0, mtc = 0;
if (c > 0) {
if constexpr (SCHED_MODE == 0) {
mta = (c + 63) >> 6;
} else if constexpr (SCHED_MODE == 1) {
if (c > 128 && c <= 192) mtc = 1;
else mta = (c + 127) >> 7;
} else {
// Universal mathematical rule: minimize (padded_rows, m_tiles) over {128, 96, 192}
int t128 = (c + 127) >> 7;
int t96 = (c + 95) / 96;
int t192 = (c + 191) / 192;
int s128 = ((t128 << 7) << 12) + t128;
int s96 = ((t96 * 96) << 12) + t96;
int s192 = ((t192 * 192) << 12) + t192;
if (s192 <= s128 && s192 <= s96) mtc = t192;
else if (s96 < s128) mtb = t96;
else mta = t128;
}
}
int wt = c, wa = mta, wb = mtb, wc = mtc;
#pragma unroll
for (int off = 1; off < 32; off <<= 1) {
int tt = __shfl_up_sync(0xffffffff, wt, off);
int ta = __shfl_up_sync(0xffffffff, wa, off);
int tb = __shfl_up_sync(0xffffffff, wb, off);
int tc = __shfl_up_sync(0xffffffff, wc, off);
if (lane >= off) { wt += tt; wa += ta; wb += tb; wc += tc; }
}
if (lane == 31) { s_wt[warp] = wt; s_wa[warp] = wa; s_wb[warp] = wb; s_wc[warp] = wc; }
__syncthreads();
if (warp == 0) {
int bt = (lane < 8) ? s_wt[lane] : 0;
int ba = (lane < 8) ? s_wa[lane] : 0;
int bb = (lane < 8) ? s_wb[lane] : 0;
int bc = (lane < 8) ? s_wc[lane] : 0;
#pragma unroll
for (int off = 1; off < 8; off <<= 1) {
int tt = __shfl_up_sync(0xffffffff, bt, off);
int ta = __shfl_up_sync(0xffffffff, ba, off);
int tb = __shfl_up_sync(0xffffffff, bb, off);
int tc = __shfl_up_sync(0xffffffff, bc, off);
if (lane >= off) { bt += tt; ba += ta; bb += tb; bc += tc; }
}
if (lane < 8) { s_wt[lane] = bt; s_wa[lane] = ba; s_wb[lane] = bb; s_wc[lane] = bc; }
}
__syncthreads();
int ex_tok = run_tok + ((warp > 0) ? s_wt[warp - 1] : 0) + wt - c;
int ex_a = run_a + ((warp > 0) ? s_wa[warp - 1] : 0) + wa - mta;
int ex_b = run_b + ((warp > 0) ? s_wb[warp - 1] : 0) + wb - mtb;
int ex_c = run_c + ((warp > 0) ? s_wc[warp - 1] : 0) + wc - mtc;
if (e < E) {
expert_offsets[e] = ex_tok;
expert_cursors[e] = ex_tok;
emto_a[e] = ex_a;
for (int m = 0; m < mta; ++m) m2e_a[ex_a + m] = e;
if constexpr (SCHED_MODE == 2) {
emto_b[e] = ex_b;
for (int m = 0; m < mtb; ++m) m2e_b[ex_b + m] = e;
}
if constexpr (SCHED_MODE >= 1) {
emto_c[e] = ex_c;
for (int m = 0; m < mtc; ++m) m2e_c[ex_c + m] = e;
}
}
run_tok += s_wt[7];
run_a += s_wa[7];
run_b += s_wb[7];
run_c += s_wc[7];
__syncthreads();
}
if (tid == 0) {
expert_offsets[E] = run_tok;
emto_a[E] = run_a;
if constexpr (SCHED_MODE == 2) emto_b[E] = run_b;
if constexpr (SCHED_MODE >= 1) emto_c[E] = run_c;
}
}
__global__ void moe_scatter_kernel(
const int64_t* __restrict__ expert_ids,
const __nv_bfloat16* __restrict__ expert_weights,
int32_t* __restrict__ expert_cursors,
int32_t* __restrict__ sorted_token_ids,
float* __restrict__ sorted_weights,
int total_tokens,
int top_k
) {
int idx = blockIdx.x * blockDim.x + threadIdx.x;
int stride = gridDim.x * blockDim.x;
for (int i = idx; i < total_tokens; i += stride) {
int e = static_cast<int>(expert_ids[i]);
int pos = atomicAdd(&expert_cursors[e], 1);
sorted_token_ids[pos] = i / top_k;
sorted_weights[pos] = __bfloat162float(expert_weights[i]);
}
}
// ============================================================================
// Pure 2D-TMA Shared GEMM1 + Fused SwiGLU (BM=128 or 64, BK=64)
// ============================================================================
template <int BM, int WARPS_M, int WARPS_N, int STAGES>
__global__ __launch_bounds__(256, 1)
void shared_gemm1_tma_kernel(
const __grid_constant__ CUtensorMap tmapX,
const __grid_constant__ CUtensorMap tmapW1,
__nv_bfloat16* __restrict__ h_out,
int T, int H, int I
) {
constexpr int BN_HALF = 64, BK = 64;
constexpr int WARP_M = BM / WARPS_M;
constexpr int WARP_N = BN_HALF / WARPS_N;
constexpr int WMT = WARP_M / 16;
constexpr int WNT2 = WARP_N / 16;
constexpr int WNT = WARP_N / 8;
constexpr int STAGE_A = BM * BK * 2;
constexpr int STAGE_BG = BN_HALF * BK * 2;
constexpr int STAGE_BYTES = STAGE_A + 2 * STAGE_BG;
int num_n = I / BN_HALF;
int m_tile = blockIdx.x / num_n;
int n_tile = blockIdx.x % num_n;
int m_start = m_tile * BM;
int valid_m = min(BM, T - m_start);
int bn = n_tile * BN_HALF;
if (valid_m <= 0) return;
extern __shared__ __align__(128) char smem[];
uint32_t smem_base = to_smem_addr(smem);
uint64_t* full_mbar = reinterpret_cast<uint64_t*>(smem + STAGES * STAGE_BYTES);
int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
int warp_m = wid / WARPS_N, warp_n = wid % WARPS_N;
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1);
}
__syncthreads();
int num_k = H / BK;
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < num_k) {
uint32_t sb = smem_base + s * STAGE_BYTES;
uint32_t fm = to_smem_addr(&full_mbar[s]);
mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
tma_load_2d(sb, &tmapX, s * BK, m_start, fm);
tma_load_2d(sb + STAGE_A, &tmapW1, s * BK, bn, fm);
tma_load_2d(sb + STAGE_A + STAGE_BG, &tmapW1, s * BK, I + bn, fm);
}
}
}
float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4];
#pragma unroll
for (int m = 0; m < WMT; ++m)
#pragma unroll
for (int n = 0; n < WNT; ++n) {
acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f;
acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f;
}
int ar = lane & 15, ac = lane >> 4;
int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4];
mbarrier_wait(to_smem_addr(&full_mbar[0]), 0);
{
uint32_t rb = smem_base;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
rb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], rb + STAGE_A + swizzle_k<64>(r, bc));
ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<64>(r, bc));
}
}
for (int k = 0; k < num_k; ++k) {
uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
#pragma unroll
for (int ks = 0; ks < 4; ++ks) {
int c = ks & 1, nx = c ^ 1;
if (ks == 3) {
__syncthreads();
int lk = k + STAGES - 1;
if (tid == 0 && lk < num_k) {
int ws = lk % STAGES;
uint32_t wb = smem_base + ws * STAGE_BYTES;
uint32_t fm = to_smem_addr(&full_mbar[ws]);
mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
tma_load_2d(wb, &tmapX, lk * BK, m_start, fm);
tma_load_2d(wb + STAGE_A, &tmapW1, lk * BK, bn, fm);
tma_load_2d(wb + STAGE_A + STAGE_BG, &tmapW1, lk * BK, I + bn, fm);
}
if (k + 1 < num_k) {
int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1;
mbarrier_wait(to_smem_addr(&full_mbar[ns]), np);
uint32_t nrb = smem_base + ns * STAGE_BYTES;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
nrb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<64>(r, bc));
ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<64>(r, bc));
}
}
} else {
int cb = (ks + 1) * 2;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
rb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, cb + ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<64>(r, cb + bc));
ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<64>(r, cb + bc));
}
}
#pragma unroll
for (int m = 0; m < WMT; ++m) {
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
}
}
}
}
__syncthreads();
constexpr int EPI_STRIDE = 72;
__nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem);
int gi = lane >> 2, ti = lane & 3;
#pragma unroll
for (int m = 0; m < WMT; ++m) {
int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
if (r0 < valid_m) {
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
*reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) =
__floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1]));
}
}
if (r1 < valid_m) {
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
*reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) =
__floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3]));
}
}
}
__syncthreads();
int er = tid >> 3, ec = tid & 7;
for (int r = er; r < valid_m; r += 32) {
*reinterpret_cast<uint4*>(h_out + (size_t)(m_start + r) * I + bn + ec * 8) =
*reinterpret_cast<const uint4*>(smem_h + r * EPI_STRIDE + ec * 8);
}
}
// ============================================================================
// Regimes 2 & 3 Routed GEMM1 (1-Barrier Hybrid cp.async A + 3D-TMA w1 + SwiGLU)
// Single __syncthreads() per K-step + L2::evict_first when nm == 1!
// Supports:
// BM=64, BK=64, WM=2, WN=4, STAGES=4 (Regime 2: T=512, 1000 -> ~1650 GB/s!)
// BM=128, BK=64, WM=4, WN=2, STAGES=3 (Regime 3 bucket A: T=4096, 4127 -> ~1626 GB/s!)
// BM=192, BK=32, WM=4, WN=2, STAGES=4 (Regime 3 bucket C: T=4096, 4127 -> ~1549 GB/s!)
// ============================================================================
template <int BM, int BK, int WM, int WN, int STAGES>
__global__ __launch_bounds__(256, 1)
void routed_gemm1_hybrid_tma_kernel(
const __nv_bfloat16* __restrict__ x,
const __grid_constant__ CUtensorMap tmapW1,
__nv_bfloat16* __restrict__ h_out,
const int32_t* __restrict__ expert_offsets,
const int32_t* __restrict__ emto,
const int32_t* __restrict__ m2e,
const int32_t* __restrict__ sorted_token_ids,
int E, int H, int I
) {
constexpr int BN_HALF = 64;
constexpr int WARP_M = BM / WM;
constexpr int WARP_N = BN_HALF / WN;
constexpr int WMT = WARP_M / 16;
constexpr int WNT2 = WARP_N / 16;
constexpr int WNT = WARP_N / 8;
constexpr int KSTEPS = BK / 16;
constexpr int CHUNKS_K = BK / 8;
constexpr int RPS = 256 / CHUNKS_K;
constexpr int A_ITERS = BM / RPS;
constexpr int STAGE_A = BM * BK * 2;
constexpr int STAGE_BG = BN_HALF * BK * 2;
constexpr int STAGE_B_BYTES = 2 * STAGE_BG;
constexpr int STAGE_BYTES = STAGE_A + STAGE_B_BYTES;
int num_n = I / BN_HALF;
int total_mt = emto[E];
int tile_id = blockIdx.x;
if (tile_id >= total_mt * num_n) return;
int e = m2e[tile_id / num_n];
int e_mt0 = emto[e];
int nm = emto[e + 1] - e_mt0;
int local = tile_id - e_mt0 * num_n;
int n_tile = local / nm;
int m_local = local % nm;
int e_tok0 = expert_offsets[e];
int e_cnt = expert_offsets[e + 1] - e_tok0;
int m_start = e_tok0 + m_local * BM;
int valid_m = min(BM, e_cnt - m_local * BM);
int bn = n_tile * BN_HALF;
extern __shared__ __align__(128) char smem[];
uint32_t smem_base = to_smem_addr(smem);
uint64_t* full_mbar = reinterpret_cast<uint64_t*>(smem + STAGES * STAGE_BYTES);
int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
int warp_m = wid / WN, warp_n = wid % WN;
uint64_t b_policy;
if (nm == 1) asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(b_policy));
else asm volatile("createpolicy.fractional.L2::evict_normal.b64 %0, 1.0;" : "=l"(b_policy));
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1);
}
__syncthreads();
int lr = tid / CHUNKS_K, lc = tid % CHUNKS_K;
int swz_a[A_ITERS], valid_a[A_ITERS];
const char* gA[A_ITERS];
#pragma unroll
for (int i = 0; i < A_ITERS; ++i) {
int r = lr + i * RPS;
swz_a[i] = swizzle_k<BK>(r, lc);
if (r < valid_m) {
gA[i] = reinterpret_cast<const char*>(x + (size_t)sorted_token_ids[m_start + r] * H) + lc * 16;
valid_a[i] = 16;
} else {
gA[i] = reinterpret_cast<const char*>(x);
valid_a[i] = 0;
}
}
int num_k = H / BK;
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < num_k) {
uint32_t sb = smem_base + s * STAGE_BYTES;
size_t koff = (size_t)s * BK * 2;
#pragma unroll
for (int i = 0; i < A_ITERS; ++i) cp_async_16(sb + swz_a[i], gA[i] + koff, valid_a[i]);
if (tid == 0) {
uint32_t fm = to_smem_addr(&full_mbar[s]);
mbarrier_arrive_expect_tx(fm, STAGE_B_BYTES);
tma_load_3d_hint(sb + STAGE_A, &tmapW1, s * BK, bn, e, fm, b_policy);
tma_load_3d_hint(sb + STAGE_A + STAGE_BG, &tmapW1, s * BK, I + bn, e, fm, b_policy);
}
}
cp_async_commit();
}
float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4];
#pragma unroll
for (int m = 0; m < WMT; ++m)
#pragma unroll
for (int n = 0; n < WNT; ++n) {
acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f;
acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f;
}
int ar = lane & 15, ac = lane >> 4;
int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4];
cp_async_wait<STAGES - 2>();
mbarrier_wait(to_smem_addr(&full_mbar[0]), 0);
__syncthreads();
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
smem_base + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], smem_base + STAGE_A + swizzle_k<BK>(r, bc));
ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], smem_base + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
}
for (int k = 0; k < num_k; ++k) {
int lk = k + STAGES - 1;
uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
#pragma unroll
for (int ks = 0; ks < KSTEPS; ++ks) {
int c = ks & 1, nx = c ^ 1;
if (ks == KSTEPS - 1) {
// Previous iter's __syncthreads() already guaranteed stage (k-1)%STAGES == lk%STAGES is consumed!
if (lk < num_k) {
int ws = lk % STAGES;
uint32_t wb = smem_base + ws * STAGE_BYTES;
size_t koff = (size_t)lk * BK * 2;
#pragma unroll
for (int i = 0; i < A_ITERS; ++i) cp_async_16(wb + swz_a[i], gA[i] + koff, valid_a[i]);
if (tid == 0) {
uint32_t fm = to_smem_addr(&full_mbar[ws]);
mbarrier_arrive_expect_tx(fm, STAGE_B_BYTES);
tma_load_3d_hint(wb + STAGE_A, &tmapW1, lk * BK, bn, e, fm, b_policy);
tma_load_3d_hint(wb + STAGE_A + STAGE_BG, &tmapW1, lk * BK, I + bn, e, fm, b_policy);
}
}
cp_async_commit();
if (k + 1 < num_k) {
int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1;
cp_async_wait<STAGES - 2>();
mbarrier_wait(to_smem_addr(&full_mbar[ns]), np);
__syncthreads();
uint32_t nrb = smem_base + ns * STAGE_BYTES;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
nrb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<BK>(r, bc));
ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
}
}
} else {
int cb = (ks + 1) * 2;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, cb + ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<BK>(r, cb + bc));
ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, cb + bc));
}
}
#pragma unroll
for (int m = 0; m < WMT; ++m) {
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
}
}
}
}
cp_async_wait<0>();
__syncthreads();
constexpr int EPI_STRIDE = 72;
__nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem);
int gi = lane >> 2, ti = lane & 3;
#pragma unroll
for (int m = 0; m < WMT; ++m) {
int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
if (r0 < valid_m) {
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
*reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) =
__floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1]));
}
}
if (r1 < valid_m) {
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
*reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) =
__floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3]));
}
}
}
__syncthreads();
int er = tid >> 3, ec = tid & 7;
for (int r = er; r < valid_m; r += 32) {
*reinterpret_cast<uint4*>(h_out + (size_t)(m_start + r) * I + bn + ec * 8) =
*reinterpret_cast<const uint4*>(smem_h + r * EPI_STRIDE + ec * 8);
}
}
// ============================================================================
// Regime 4 Routed GEMM1 (Zero-Branch Pipelined cp.async + SwiGLU)
// Supports:
// BM=128, BK=64, WM=4, WN=2, STAGES=3 (Regime 4 bucket A)
// BM=96, BK=64, WM=2, WN=4, STAGES=3 (Regime 4 bucket B)
// BM=192, BK=32, WM=4, WN=2, STAGES=4 (Regime 4 bucket C)
// ============================================================================
template <int BM, int BK, int WM, int WN, int STAGES>
__global__ __launch_bounds__(256, 1)
void routed_gemm1_cpasync_kernel(
const __nv_bfloat16* __restrict__ x,
const __nv_bfloat16* __restrict__ w1,
__nv_bfloat16* __restrict__ h_out,
const int32_t* __restrict__ expert_offsets,
const int32_t* __restrict__ emto,
const int32_t* __restrict__ m2e,
const int32_t* __restrict__ sorted_token_ids,
int E, int H, int I
) {
constexpr int BN_HALF = 64;
constexpr int WARP_M = BM / WM;
constexpr int WARP_N = BN_HALF / WN;
constexpr int WMT = WARP_M / 16;
constexpr int WNT2 = WARP_N / 16;
constexpr int WNT = WARP_N / 8;
constexpr int KSTEPS = BK / 16;
constexpr int CHUNKS_K = BK / 8;
constexpr int RPS = 256 / CHUNKS_K;
constexpr int A_ITERS = BM / RPS;
constexpr int B_ITERS = BN_HALF / RPS;
constexpr int STAGE_A = BM * BK * 2;
constexpr int STAGE_BG = BN_HALF * BK * 2;
constexpr int STAGE_BYTES = STAGE_A + 2 * STAGE_BG;
int num_n = I / BN_HALF;
int total_mt = emto[E];
int tile_id = blockIdx.x;
if (tile_id >= total_mt * num_n) return;
int e = m2e[tile_id / num_n];
int e_mt0 = emto[e];
int nm = emto[e + 1] - e_mt0;
int local = tile_id - e_mt0 * num_n;
int n_tile = local / nm;
int m_local = local % nm;
int e_tok0 = expert_offsets[e];
int e_cnt = expert_offsets[e + 1] - e_tok0;
int m_start = e_tok0 + m_local * BM;
int valid_m = min(BM, e_cnt - m_local * BM);
int bn = n_tile * BN_HALF;
extern __shared__ __align__(128) char smem[];
uint32_t smem_base = to_smem_addr(smem);
int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
int warp_m = wid / WN, warp_n = wid % WN;
int lr = tid / CHUNKS_K, lc = tid % CHUNKS_K;
int swz_a[A_ITERS], valid_a[A_ITERS];
const char* gA[A_ITERS];
#pragma unroll
for (int i = 0; i < A_ITERS; ++i) {
int r = lr + i * RPS;
swz_a[i] = swizzle_k<BK>(r, lc);
if (r < valid_m) {
gA[i] = reinterpret_cast<const char*>(x + (size_t)sorted_token_ids[m_start + r] * H) + lc * 16;
valid_a[i] = 16;
} else {
gA[i] = reinterpret_cast<const char*>(x);
valid_a[i] = 0;
}
}
int swz_bg[B_ITERS], swz_bu[B_ITERS];
const char* gBg[B_ITERS];
const char* gBu[B_ITERS];
const __nv_bfloat16* w1_e = w1 + (size_t)e * (2 * I) * H;
#pragma unroll
for (int i = 0; i < B_ITERS; ++i) {
int r = lr + i * RPS;
swz_bg[i] = STAGE_A + swizzle_k<BK>(r, lc);
swz_bu[i] = STAGE_A + STAGE_BG + swizzle_k<BK>(r, lc);
gBg[i] = reinterpret_cast<const char*>(w1_e + (size_t)(bn + r) * H) + lc * 16;
gBu[i] = reinterpret_cast<const char*>(w1_e + (size_t)(I + bn + r) * H) + lc * 16;
}
int num_k = H / BK;
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < num_k) {
uint32_t sb = smem_base + s * STAGE_BYTES;
size_t koff = (size_t)s * BK * 2;
#pragma unroll
for (int i = 0; i < A_ITERS; ++i) cp_async_16(sb + swz_a[i], gA[i] + koff, valid_a[i]);
#pragma unroll
for (int i = 0; i < B_ITERS; ++i) {
cp_async_16_full(sb + swz_bg[i], gBg[i] + koff);
cp_async_16_full(sb + swz_bu[i], gBu[i] + koff);
}
}
cp_async_commit();
}
float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4];
#pragma unroll
for (int m = 0; m < WMT; ++m)
#pragma unroll
for (int n = 0; n < WNT; ++n) {
acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f;
acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f;
}
int ar = lane & 15, ac = lane >> 4;
int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4];
cp_async_wait<STAGES - 2>();
__syncthreads();
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
smem_base + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], smem_base + STAGE_A + swizzle_k<BK>(r, bc));
ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], smem_base + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
}
for (int k = 0; k < num_k; ++k) {
int lk = k + STAGES - 1;
uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
#pragma unroll
for (int ks = 0; ks < KSTEPS; ++ks) {
int c = ks & 1, nx = c ^ 1;
if (ks == KSTEPS - 1) {
if (lk < num_k) {
uint32_t wb = smem_base + (lk % STAGES) * STAGE_BYTES;
size_t koff = (size_t)lk * BK * 2;
#pragma unroll
for (int i = 0; i < A_ITERS; ++i) cp_async_16(wb + swz_a[i], gA[i] + koff, valid_a[i]);
#pragma unroll
for (int i = 0; i < B_ITERS; ++i) {
cp_async_16_full(wb + swz_bg[i], gBg[i] + koff);
cp_async_16_full(wb + swz_bu[i], gBu[i] + koff);
}
}
cp_async_commit();
cp_async_wait<STAGES - 2>();
__syncthreads();
if (k + 1 < num_k) {
uint32_t nrb = smem_base + ((k + 1) % STAGES) * STAGE_BYTES;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
nrb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<BK>(r, bc));
ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
}
}
} else {
int cb = (ks + 1) * 2;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, cb + ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
int r = warp_n * WARP_N + n * 16 + br;
ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<BK>(r, cb + bc));
ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, cb + bc));
}
}
if constexpr (WMT > WNT2) {
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
#pragma unroll
for (int m = 0; m < WMT; ++m) {
mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
}
}
} else {
#pragma unroll
for (int m = 0; m < WMT; ++m) {
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
}
}
}
}
}
cp_async_wait<0>();
__syncthreads();
constexpr int EPI_STRIDE = 72;
__nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem);
int gi = lane >> 2, ti = lane & 3;
#pragma unroll
for (int m = 0; m < WMT; ++m) {
int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
if (r0 < valid_m) {
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
*reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) =
__floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1]));
}
}
if (r1 < valid_m) {
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
*reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) =
__floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3]));
}
}
}
__syncthreads();
int er = tid >> 3, ec = tid & 7;
for (int r = er; r < valid_m; r += 32) {
*reinterpret_cast<uint4*>(h_out + (size_t)(m_start + r) * I + bn + ec * 8) =
*reinterpret_cast<const uint4*>(smem_h + r * EPI_STRIDE + ec * 8);
}
}
// ============================================================================
// Pure TMA GEMM2 + Weighted Vectorized Global Reduction (or Direct Store)
// Supports:
// Shared: BM=128 (4x2, BK=64, S=3), BM=64 (2x4, BK=64, S=4)
// Routed: BM=64 (2x4, BK=64, S=4), BM=128 (4x2, BK=64, S=3),
// BM=96 (2x4, BK=64, S=3), BM=192 (4x2, BK=32, S=4)
// ============================================================================
template <int BM, int BK, int WM, int WN, int STAGES, bool IS_SHARED, bool DIRECT_STORE>
__global__ __launch_bounds__(256, 1)
void gemm2_tma_kernel(
const __grid_constant__ CUtensorMap tmapH,
const __grid_constant__ CUtensorMap tmapW2,
float* __restrict__ out_fp32,
const int32_t* __restrict__ expert_offsets,
const int32_t* __restrict__ emto,
const int32_t* __restrict__ m2e,
const int32_t* __restrict__ sorted_token_ids,
const float* __restrict__ sorted_weights,
int E_or_T, int H, int I
) {
constexpr int BN = 128;
constexpr int WARP_M = BM / WM;
constexpr int WARP_N = BN / WN;
constexpr int WMT = WARP_M / 16;
constexpr int WNT2 = WARP_N / 16;
constexpr int WNT = WARP_N / 8;
constexpr int KSTEPS = BK / 16;
constexpr int STAGE_A = BM * BK * 2;
constexpr int STAGE_B = BN * BK * 2;
constexpr int STAGE_BYTES = STAGE_A + STAGE_B;
int num_n = H / BN;
int tile_id = blockIdx.x;
int e = 0, nm = 2, m_start = 0, valid_m = 0, bn = 0;
if constexpr (IS_SHARED) {
int m_tile = tile_id / num_n;
int n_tile = tile_id % num_n;
m_start = m_tile * BM;
valid_m = min(BM, E_or_T - m_start);
bn = n_tile * BN;
if (valid_m <= 0) return;
} else {
int total_mt = emto[E_or_T];
if (tile_id >= total_mt * num_n) return;
e = m2e[tile_id / num_n];
int e_mt0 = emto[e];
nm = emto[e + 1] - e_mt0;
int local = tile_id - e_mt0 * num_n;
int n_tile = local / nm;
int m_local = local % nm;
int e_tok0 = expert_offsets[e];
int e_cnt = expert_offsets[e + 1] - e_tok0;
m_start = e_tok0 + m_local * BM;
valid_m = min(BM, e_cnt - m_local * BM);
bn = n_tile * BN;
}
extern __shared__ __align__(128) char smem[];
uint32_t smem_base = to_smem_addr(smem);
uint64_t* full_mbar = reinterpret_cast<uint64_t*>(smem + STAGES * STAGE_BYTES);
int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
int warp_m = wid / WN, warp_n = wid % WN;
uint64_t b_policy;
if constexpr (!IS_SHARED) {
if (nm == 1) asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(b_policy));
else asm volatile("createpolicy.fractional.L2::evict_normal.b64 %0, 1.0;" : "=l"(b_policy));
}
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1);
}
__syncthreads();
int num_k = I / BK;
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES - 1; ++s) {
if (s < num_k) {
uint32_t sb = smem_base + s * STAGE_BYTES;
uint32_t fm = to_smem_addr(&full_mbar[s]);
mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
tma_load_2d(sb, &tmapH, s * BK, m_start, fm);
if constexpr (IS_SHARED) tma_load_2d(sb + STAGE_A, &tmapW2, s * BK, bn, fm);
else tma_load_3d_hint(sb + STAGE_A, &tmapW2, s * BK, bn, e, fm, b_policy);
}
}
}
float acc[WMT][WNT][4];
#pragma unroll
for (int m = 0; m < WMT; ++m)
#pragma unroll
for (int n = 0; n < WNT; ++n)
acc[m][n][0] = acc[m][n][1] = acc[m][n][2] = acc[m][n][3] = 0.0f;
int ar = lane & 15, ac = lane >> 4;
int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
uint32_t aR[2][WMT][4], bR[2][WNT2][4];
mbarrier_wait(to_smem_addr(&full_mbar[0]), 0);
{
uint32_t rb = smem_base;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n)
ldmatrix_x4(bR[0][n][0], bR[0][n][1], bR[0][n][2], bR[0][n][3],
rb + STAGE_A + swizzle_k<BK>(warp_n * WARP_N + n * 16 + br, bc));
}
for (int k = 0; k < num_k; ++k) {
uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
#pragma unroll
for (int ks = 0; ks < KSTEPS; ++ks) {
int c = ks & 1, nx = c ^ 1;
if (ks == KSTEPS - 1) {
__syncthreads();
int lk = k + STAGES - 1;
if (tid == 0 && lk < num_k) {
int ws = lk % STAGES;
uint32_t wb = smem_base + ws * STAGE_BYTES;
uint32_t fm = to_smem_addr(&full_mbar[ws]);
mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
tma_load_2d(wb, &tmapH, lk * BK, m_start, fm);
if constexpr (IS_SHARED) tma_load_2d(wb + STAGE_A, &tmapW2, lk * BK, bn, fm);
else tma_load_3d_hint(wb + STAGE_A, &tmapW2, lk * BK, bn, e, fm, b_policy);
}
if (k + 1 < num_k) {
int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1;
mbarrier_wait(to_smem_addr(&full_mbar[ns]), np);
uint32_t nrb = smem_base + ns * STAGE_BYTES;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
nrb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n)
ldmatrix_x4(bR[nx][n][0], bR[nx][n][1], bR[nx][n][2], bR[nx][n][3],
nrb + STAGE_A + swizzle_k<BK>(warp_n * WARP_N + n * 16 + br, bc));
}
} else {
int cb = (ks + 1) * 2;
#pragma unroll
for (int m = 0; m < WMT; ++m)
ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, cb + ac));
#pragma unroll
for (int n = 0; n < WNT2; ++n)
ldmatrix_x4(bR[nx][n][0], bR[nx][n][1], bR[nx][n][2], bR[nx][n][3],
rb + STAGE_A + swizzle_k<BK>(warp_n * WARP_N + n * 16 + br, cb + bc));
}
if constexpr (WMT > WNT2) {
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
#pragma unroll
for (int m = 0; m < WMT; ++m) {
mma_m16n8k16_bf16(acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][0], bR[c][n][1], acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3]);
mma_m16n8k16_bf16(acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][2], bR[c][n][3], acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3]);
}
}
} else {
#pragma unroll
for (int m = 0; m < WMT; ++m) {
#pragma unroll
for (int n = 0; n < WNT2; ++n) {
mma_m16n8k16_bf16(acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][0], bR[c][n][1], acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3]);
mma_m16n8k16_bf16(acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][2], bR[c][n][3], acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3]);
}
}
}
}
}
__syncthreads();
constexpr int EPI_STRIDE = 136; // 128 + 8 floats (16B aligned for float4!)
constexpr int EPI_ROWS = (BM <= 128) ? BM : (BM / 2);
float* smem_f = reinterpret_cast<float*>(smem);
int32_t* smem_tok = reinterpret_cast<int32_t*>(smem_f + EPI_ROWS * EPI_STRIDE);
float* smem_wt = reinterpret_cast<float*>(smem_tok + BM);
if (tid < valid_m) {
if constexpr (IS_SHARED) {
smem_tok[tid] = m_start + tid;
smem_wt[tid] = 1.0f;
} else {
smem_tok[tid] = sorted_token_ids[m_start + tid];
smem_wt[tid] = sorted_weights[m_start + tid];
}
}
__syncthreads();
int gi = lane >> 2, ti = lane & 3;
if constexpr (BM <= 128) {
#pragma unroll
for (int m = 0; m < WMT; ++m) {
int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
if (r0 < valid_m) {
float w0 = smem_wt[r0];
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
smem_f[r0 * EPI_STRIDE + c0] = acc[m][n][0] * w0;
smem_f[r0 * EPI_STRIDE + c0 + 1] = acc[m][n][1] * w0;
}
}
if (r1 < valid_m) {
float w1 = smem_wt[r1];
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
smem_f[r1 * EPI_STRIDE + c0] = acc[m][n][2] * w1;
smem_f[r1 * EPI_STRIDE + c0 + 1] = acc[m][n][3] * w1;
}
}
}
__syncthreads();
for (int r = wid; r < valid_m; r += 8) {
int tok = smem_tok[r];
float4 v = *reinterpret_cast<const float4*>(smem_f + r * EPI_STRIDE + lane * 4);
if constexpr (DIRECT_STORE) *reinterpret_cast<float4*>(out_fp32 + (size_t)tok * H + bn + lane * 4) = v;
else red_global_add_v4_f32(out_fp32 + (size_t)tok * H + bn + lane * 4, v);
}
} else {
// Two-pass epilogue for BM=192 (96 rows/pass -> 53KB <= 82KB!)
#pragma unroll
for (int pass = 0; pass < 2; ++pass) {
int row_base = pass * EPI_ROWS;
int pass_valid = min(EPI_ROWS, max(0, valid_m - row_base));
if (pass_valid <= 0) break;
if ((warp_m >> 1) == pass) {
int local_wm = warp_m & 1;
#pragma unroll
for (int m = 0; m < WMT; ++m) {
int lr0 = local_wm * WARP_M + m * 16 + gi, lr1 = lr0 + 8;
int gr0 = row_base + lr0, gr1 = row_base + lr1;
if (gr0 < valid_m) {
float w0 = smem_wt[gr0];
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
smem_f[lr0 * EPI_STRIDE + c0] = acc[m][n][0] * w0;
smem_f[lr0 * EPI_STRIDE + c0 + 1] = acc[m][n][1] * w0;
}
}
if (gr1 < valid_m) {
float w1 = smem_wt[gr1];
#pragma unroll
for (int n = 0; n < WNT; ++n) {
int c0 = warp_n * WARP_N + n * 8 + ti * 2;
smem_f[lr1 * EPI_STRIDE + c0] = acc[m][n][2] * w1;
smem_f[lr1 * EPI_STRIDE + c0 + 1] = acc[m][n][3] * w1;
}
}
}
}
__syncthreads();
for (int r = wid; r < pass_valid; r += 8) {
int tok = smem_tok[row_base + r];
float4 v = *reinterpret_cast<const float4*>(smem_f + r * EPI_STRIDE + lane * 4);
if constexpr (DIRECT_STORE) *reinterpret_cast<float4*>(out_fp32 + (size_t)tok * H + bn + lane * 4) = v;
else red_global_add_v4_f32(out_fp32 + (size_t)tok * H + bn + lane * 4, v);
}
if (pass == 0) __syncthreads();
}
}
}
// ============================================================================
// Vectorized FP32 -> BF16 Cast
// ============================================================================
__global__ void fp32_to_bf16_vec8_kernel(
const float* __restrict__ src,
__nv_bfloat16* __restrict__ dst,
size_t n_vec8
) {
size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
size_t stride = (size_t)gridDim.x * blockDim.x;
const float4* src4 = reinterpret_cast<const float4*>(src);
uint4* dst8 = reinterpret_cast<uint4*>(dst);
for (size_t i = idx; i < n_vec8; i += stride) {
float4 a = src4[i * 2];
float4 b = src4[i * 2 + 1];
__nv_bfloat162 r0 = __floats2bfloat162_rn(a.x, a.y);
__nv_bfloat162 r1 = __floats2bfloat162_rn(a.z, a.w);
__nv_bfloat162 r2 = __floats2bfloat162_rn(b.x, b.y);
__nv_bfloat162 r3 = __floats2bfloat162_rn(b.z, b.w);
uint4 packed;
packed.x = *reinterpret_cast<uint32_t*>(&r0);
packed.y = *reinterpret_cast<uint32_t*>(&r1);
packed.z = *reinterpret_cast<uint32_t*>(&r2);
packed.w = *reinterpret_cast<uint32_t*>(&r3);
dst8[i] = packed;
}
}
// ============================================================================
// TMA Helpers, Dual-Priority Streams, and O(1) Per-Device Workspace Cache
// ============================================================================
inline CUtensorMap make_tmap_2d(void* ptr, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols, bool swz64 = false) {
CUtensorMap tmap;
uint64_t dims[2] = {cols, rows};
uint64_t strides[1] = {cols * 2};
uint32_t box[2] = {box_cols, box_rows};
uint32_t es[2] = {1, 1};
CUresult res = cuTensorMapEncodeTiled(
&tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 2, ptr, dims, strides, box, es,
CU_TENSOR_MAP_INTERLEAVE_NONE,
swz64 ? CU_TENSOR_MAP_SWIZZLE_64B : CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
);
TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 2D failed: ", (int)res);
return tmap;
}
inline CUtensorMap make_tmap_3d(void* ptr, uint64_t E, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols, bool swz64 = false) {
CUtensorMap tmap;
uint64_t dims[3] = {cols, rows, E};
uint64_t strides[2] = {cols * 2, rows * cols * 2};
uint32_t box[3] = {box_cols, box_rows, 1};
uint32_t es[3] = {1, 1, 1};
CUresult res = cuTensorMapEncodeTiled(
&tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3, ptr, dims, strides, box, es,
CU_TENSOR_MAP_INTERLEAVE_NONE,
swz64 ? CU_TENSOR_MAP_SWIZZLE_64B : CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
);
TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 3D failed: ", (int)res);
return tmap;
}
struct DeviceWorkspace {
torch::Tensor out_fp32;
torch::Tensor h_routed;
torch::Tensor h_shared;
torch::Tensor h_decode;
torch::Tensor expert_counts;
torch::Tensor expert_offsets;
torch::Tensor expert_cursors;
torch::Tensor emto_a, m2e_a;
torch::Tensor emto_b, m2e_b;
torch::Tensor emto_c, m2e_c;
torch::Tensor sorted_token_ids;
torch::Tensor sorted_weights;
cudaStream_t shared_stream_high = nullptr;
cudaStream_t shared_stream_low = nullptr;
cudaEvent_t ev_in_ready = nullptr;
cudaEvent_t ev_sh_done = nullptr;
bool stream_init = false;
static void ensure(torch::Tensor& t, int64_t numel, c10::ScalarType dtype, const c10::Device& dev) {
if (!t.defined() || t.numel() < numel || t.scalar_type() != dtype) {
t = at::empty({numel}, at::TensorOptions().dtype(dtype).device(dev));
}
}
void init_streams() {
if (!stream_init) {
int low_p = 0, high_p = 0;
cudaDeviceGetStreamPriorityRange(&low_p, &high_p);
cudaStreamCreateWithPriority(&shared_stream_high, cudaStreamNonBlocking, high_p);
cudaStreamCreateWithPriority(&shared_stream_low, cudaStreamNonBlocking, low_p);
cudaEventCreateWithFlags(&ev_in_ready, cudaEventDisableTiming);
cudaEventCreateWithFlags(&ev_sh_done, cudaEventDisableTiming);
stream_init = true;
}
}
void prepare_decode(const c10::Device& dev, int64_t n_h) {
if (!h_decode.defined() || h_decode.numel() < n_h) {
h_decode = at::empty({n_h}, at::TensorOptions().dtype(at::kBFloat16).device(dev));
}
}
void prepare_general(const c10::Device& dev, int T, int E, int top_k, int H, int I) {
int64_t total_tokens = (int64_t)T * top_k;
int64_t max_mt = (total_tokens + 63) / 64 + E + 16;
ensure(out_fp32, (int64_t)T * H, at::kFloat, dev);
ensure(h_routed, total_tokens * I, at::kBFloat16, dev);
ensure(h_shared, (int64_t)T * I, at::kBFloat16, dev);
ensure(expert_counts, E, at::kInt, dev);
ensure(expert_offsets, E + 1, at::kInt, dev);
ensure(expert_cursors, E, at::kInt, dev);
ensure(emto_a, E + 1, at::kInt, dev);
ensure(m2e_a, max_mt, at::kInt, dev);
ensure(emto_b, E + 1, at::kInt, dev);
ensure(m2e_b, max_mt, at::kInt, dev);
ensure(emto_c, E + 1, at::kInt, dev);
ensure(m2e_c, max_mt, at::kInt, dev);
ensure(sorted_token_ids, total_tokens, at::kInt, dev);
ensure(sorted_weights, total_tokens, at::kFloat, dev);
}
};
static DeviceWorkspace g_ws[16];
static bool g_smem_init = false;
inline void init_smem_attrs() {
if (g_smem_init) return;
constexpr int S96T = 98432;
constexpr int S96 = 98304;
constexpr int S84T = 86144;
constexpr int S84 = 86016;
constexpr int S82T = 82048;
cudaFuncSetAttribute(shared_gemm1_tma_kernel<128, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(shared_gemm1_tma_kernel<64, 2, 4, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<64, 64, 2, 4, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<128, 64, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<192, 32, 4, 2, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T);
cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<128, 64, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96);
cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<96, 64, 2, 4, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S84);
cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<192, 32, 4, 2, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T);
cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, true, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, true, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(gemm2_tma_kernel<64, 64, 2, 4, 4, true, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(gemm2_tma_kernel<64, 64, 2, 4, 4, true, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(gemm2_tma_kernel<64, 64, 2, 4, 4, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
cudaFuncSetAttribute(gemm2_tma_kernel<96, 64, 2, 4, 3, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S84T);
cudaFuncSetAttribute(gemm2_tma_kernel<192, 32, 4, 2, 4, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T);
g_smem_init = true;
}
inline void launch_shared_experts(
cudaStream_t stream,
const torch::Tensor& x,
const torch::Tensor& w1_shared,
const torch::Tensor& w2_shared,
DeviceWorkspace& ws,
int T, int n_shared, int H, int I
) {
int sBM = (T >= 128) ? 128 : 64;
int s_m_tiles = (T + sBM - 1) / sBM;
int num_n1 = I / 64, num_n2 = H / 128;
constexpr int S96T = 98432;
CUtensorMap tmapX = make_tmap_2d(x.data_ptr(), T, H, sBM, 64);
CUtensorMap tmapHs = make_tmap_2d(ws.h_shared.data_ptr(), T, I, sBM, 64);
for (int s = 0; s < n_shared; ++s) {
void* w1s_ptr = reinterpret_cast<__nv_bfloat16*>(w1_shared.data_ptr()) + (size_t)s * (2 * I) * H;
void* w2s_ptr = reinterpret_cast<__nv_bfloat16*>(w2_shared.data_ptr()) + (size_t)s * H * I;
CUtensorMap tmapW1s = make_tmap_2d(w1s_ptr, 2 * I, H, 64, 64);
CUtensorMap tmapW2s = make_tmap_2d(w2s_ptr, H, I, 128, 64);
if (sBM == 128) {
shared_gemm1_tma_kernel<128, 4, 2, 3><<<s_m_tiles * num_n1, 256, S96T, stream>>>(
tmapX, tmapW1s, reinterpret_cast<__nv_bfloat16*>(ws.h_shared.data_ptr()), T, H, I);
if (s == 0) {
gemm2_tma_kernel<128, 64, 4, 2, 3, true, true><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
} else {
gemm2_tma_kernel<128, 64, 4, 2, 3, true, false><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
}
} else {
shared_gemm1_tma_kernel<64, 2, 4, 4><<<s_m_tiles * num_n1, 256, S96T, stream>>>(
tmapX, tmapW1s, reinterpret_cast<__nv_bfloat16*>(ws.h_shared.data_ptr()), T, H, I);
if (s == 0) {
gemm2_tma_kernel<64, 64, 2, 4, 4, true, true><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
} else {
gemm2_tma_kernel<64, 64, 2, 4, 4, true, false><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
}
}
}
}
} // anonymous namespace
torch::Tensor moe_forward_cuda(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared
) {
if (!x.is_contiguous()) x = x.contiguous();
if (!expert_ids.is_contiguous()) expert_ids = expert_ids.contiguous();
if (!expert_weights.is_contiguous()) expert_weights = expert_weights.contiguous();
int dev_idx = x.device().index();
c10::cuda::CUDAGuard device_guard(x.device());
cudaStream_t stream = at::cuda::getCurrentCUDAStream();
int T = x.size(0);
int H = x.size(1);
int E = w1_routed.size(0);
int I = w2_routed.size(2);
int top_k = expert_ids.size(1);
int n_shared = w1_shared.size(0);
auto out_bf16 = at::empty({T, H}, x.options());
DeviceWorkspace& ws = g_ws[dev_idx & 15];
// ------------------------------------------------------------------------
// Regime 1: Decode / Tiny-T (T <= 4)
// ------------------------------------------------------------------------
if (T <= 4) {
int S = n_shared + top_k;
ws.prepare_decode(x.device(), (int64_t)T * S * I);
decode_gemv1_splitwarp_128t_kernel<<<T * S * I, 128, 0, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
expert_ids.data_ptr<int64_t>(),
reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(ws.h_decode.data_ptr()),
T, top_k, n_shared, H, I
);
decode_gemv2_4w_1row_kernel<<<T * H, 128, 0, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(ws.h_decode.data_ptr()),
expert_ids.data_ptr<int64_t>(),
reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()),
T, top_k, n_shared, H, I
);
return out_bf16;
}
// ------------------------------------------------------------------------
// Regimes 2, 3, 4: General-T Grouped TMA MoE (Zero CPU Sync)
// ------------------------------------------------------------------------
init_smem_attrs();
ws.init_streams();
ws.prepare_general(x.device(), T, E, top_k, H, I);
int total_tokens = T * top_k;
int num_n1 = I / 64;
int num_n2 = H / 128;
// Regime selection based on average tokens per expert (general across all T, E, top_k)
bool is_small = (total_tokens <= E * 48); // Regime 2: BM=64 (e.g. T=512, 1000)
bool is_medium = (!is_small && total_tokens <= E * 180); // Regime 3: [BM=128, BM=192] (e.g. T=4096, 4127)
// else: Regime 4: Universal 3-bucket [BM=128, BM=96, BM=192] (e.g. T=8192)
bool overlap_shared = (n_shared > 0) && (is_small || is_medium);
if (n_shared == 0) {
cudaMemsetAsync(ws.out_fp32.data_ptr(), 0, (size_t)T * H * sizeof(float), stream);
} else if (overlap_shared) {
// Record entry event so shared_stream starts cleanly once inputs are ready,
// AFTER routing + routed_gemm1 are enqueued on main stream!
cudaEventRecord(ws.ev_in_ready, stream);
} else {
// Compute-bound Regime 4: run shared sequentially first (zero stream contention, zero memset)
launch_shared_experts(stream, x, w1_shared, w2_shared, ws, T, n_shared, H, I);
}
// 1. GPU Routing on main stream (Zero CPU Sync)
cudaMemsetAsync(ws.expert_counts.data_ptr(), 0, (size_t)E * sizeof(int32_t), stream);
int hist_blocks = min(256, (total_tokens + 255) / 256);
moe_histogram_kernel<<<hist_blocks, 256, E * sizeof(int32_t), stream>>>(
expert_ids.data_ptr<int64_t>(), ws.expert_counts.data_ptr<int32_t>(), total_tokens, E);
if (is_small) {
moe_prefix_and_schedule_kernel<0><<<1, 256, 0, stream>>>(
ws.expert_counts.data_ptr<int32_t>(), ws.expert_offsets.data_ptr<int32_t>(),
ws.expert_cursors.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), nullptr, nullptr, nullptr, nullptr, E);
} else if (is_medium) {
moe_prefix_and_schedule_kernel<1><<<1, 256, 0, stream>>>(
ws.expert_counts.data_ptr<int32_t>(), ws.expert_offsets.data_ptr<int32_t>(),
ws.expert_cursors.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), nullptr, nullptr,
ws.emto_c.data_ptr<int32_t>(), ws.m2e_c.data_ptr<int32_t>(), E);
} else {
moe_prefix_and_schedule_kernel<2><<<1, 256, 0, stream>>>(
ws.expert_counts.data_ptr<int32_t>(), ws.expert_offsets.data_ptr<int32_t>(),
ws.expert_cursors.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.emto_b.data_ptr<int32_t>(),
ws.m2e_b.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
ws.m2e_c.data_ptr<int32_t>(), E);
}
int scatter_blocks = min(512, (total_tokens + 255) / 256);
moe_scatter_kernel<<<scatter_blocks, 256, 0, stream>>>(
expert_ids.data_ptr<int64_t>(),
reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
ws.expert_cursors.data_ptr<int32_t>(),
ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(),
total_tokens, top_k);
// 2. Routed GEMM1 + Overlapped Shared + Routed GEMM2
constexpr int S96T = 98432, S96 = 98304, S84T = 86144, S84 = 86016, S82T = 82048;
if (is_small) {
int max_mt = (total_tokens + 63) / 64 + E;
CUtensorMap tmapW1 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 64, false);
CUtensorMap tmapW2 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false);
CUtensorMap tmapHr = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 64, 64, false);
routed_gemm1_hybrid_tma_kernel<64, 64, 2, 4, 4><<<max_mt * num_n1, 256, S96T, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
tmapW1, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);
if (overlap_shared) {
cudaStreamWaitEvent(ws.shared_stream_high, ws.ev_in_ready, 0);
launch_shared_experts(ws.shared_stream_high, x, w1_shared, w2_shared, ws, T, n_shared, H, I);
cudaEventRecord(ws.ev_sh_done, ws.shared_stream_high);
cudaStreamWaitEvent(stream, ws.ev_sh_done, 0);
}
gemm2_tma_kernel<64, 64, 2, 4, 4, false, false><<<max_mt * num_n2, 256, S96T, stream>>>(
tmapHr, tmapW2, ws.out_fp32.data_ptr<float>(),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(), E, H, I);
} else if (is_medium) {
// Regime 3: Disjoint [BM=128 (BK=64), BM=192 (BK=32)] with 1-barrier hybrid 3D-TMA (evict_first)
int max_mt128 = (total_tokens + 127) / 128 + E;
int max_mt192 = min(E, total_tokens / 129) + 1;
CUtensorMap tmapW1_64 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 64, false);
CUtensorMap tmapW1_32 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 32, true);
CUtensorMap tmapW2_64 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false);
CUtensorMap tmapW2_32 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 32, true);
CUtensorMap tmapH128 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 128, 64, false);
CUtensorMap tmapH192 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 192, 32, true);
routed_gemm1_hybrid_tma_kernel<128, 64, 4, 2, 3><<<max_mt128 * num_n1, 256, S96T, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
tmapW1_64, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);
routed_gemm1_hybrid_tma_kernel<192, 32, 4, 2, 4><<<max_mt192 * num_n1, 256, S82T, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
tmapW1_32, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);
if (overlap_shared) {
cudaStreamWaitEvent(ws.shared_stream_low, ws.ev_in_ready, 0);
launch_shared_experts(ws.shared_stream_low, x, w1_shared, w2_shared, ws, T, n_shared, H, I);
cudaEventRecord(ws.ev_sh_done, ws.shared_stream_low);
cudaStreamWaitEvent(stream, ws.ev_sh_done, 0);
}
gemm2_tma_kernel<128, 64, 4, 2, 3, false, false><<<max_mt128 * num_n2, 256, S96T, stream>>>(
tmapH128, tmapW2_64, ws.out_fp32.data_ptr<float>(),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(), E, H, I);
gemm2_tma_kernel<192, 32, 4, 2, 4, false, false><<<max_mt192 * num_n2, 256, S82T, stream>>>(
tmapH192, tmapW2_32, ws.out_fp32.data_ptr<float>(),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(), E, H, I);
} else {
// Regime 4: Universal 3-bucket [BM=128, BM=96, BM=192] (ZERO CPU sync)
int max_mt128 = (total_tokens + 127) / 128 + E;
int max_mt96 = (total_tokens + 95) / 96 + E;
int max_mt192 = total_tokens / 129 + 1;
CUtensorMap tmapW2_64 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false);
CUtensorMap tmapW2_32 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 32, true);
CUtensorMap tmapH128 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 128, 64, false);
CUtensorMap tmapH96 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 96, 64, false);
CUtensorMap tmapH192 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 192, 32, true);
routed_gemm1_cpasync_kernel<128, 64, 4, 2, 3><<<max_mt128 * num_n1, 256, S96, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);
routed_gemm1_cpasync_kernel<96, 64, 2, 4, 3><<<max_mt96 * num_n1, 256, S84, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_b.data_ptr<int32_t>(),
ws.m2e_b.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);
routed_gemm1_cpasync_kernel<192, 32, 4, 2, 4><<<max_mt192 * num_n1, 256, S82T, stream>>>(
reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);
gemm2_tma_kernel<128, 64, 4, 2, 3, false, false><<<max_mt128 * num_n2, 256, S96T, stream>>>(
tmapH128, tmapW2_64, ws.out_fp32.data_ptr<float>(),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(), E, H, I);
gemm2_tma_kernel<96, 64, 2, 4, 3, false, false><<<max_mt96 * num_n2, 256, S84T, stream>>>(
tmapH96, tmapW2_64, ws.out_fp32.data_ptr<float>(),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_b.data_ptr<int32_t>(),
ws.m2e_b.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(), E, H, I);
gemm2_tma_kernel<192, 32, 4, 2, 4, false, false><<<max_mt192 * num_n2, 256, S82T, stream>>>(
tmapH192, tmapW2_32, ws.out_fp32.data_ptr<float>(),
ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
ws.sorted_weights.data_ptr<float>(), E, H, I);
}
// 3. Vectorized FP32 -> BF16 cast
size_t n_vec8 = ((size_t)T * H) / 8;
int cast_blocks = (int)min((size_t)(188 * 8), (n_vec8 + 255) / 256);
fp32_to_bf16_vec8_kernel<<<cast_blocks, 256, 0, stream>>>(
ws.out_fp32.data_ptr<float>(),
reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()),
n_vec8);
return out_bf16;
}
'''
_CPP_SRC = r'''
torch::Tensor moe_forward_cuda(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared
);
'''
_EXT = None
def _get_ext():
global _EXT
if _EXT is None:
src_hash = hashlib.md5(_CUDA_SRC.encode()).hexdigest()[:10]
_EXT = load_inline(
name=f"glm52_fused_moe_sm120_v4_{src_hash}",
cpp_sources=_CPP_SRC,
cuda_sources=_CUDA_SRC,
functions=["moe_forward_cuda"],
extra_cuda_cflags=["-O3", "-arch=sm_120a"],
extra_ldflags=["-lcuda"],
verbose=False,
)
return _EXT
class Model(nn.Module):
def __init__(
self,
T: int,
E: int,
top_k: int,
n_shared: int,
H: int,
I: int,
):
super().__init__()
self.T, self.E, self.top_k = T, E, top_k
self.n_shared, self.H, self.I = n_shared, H, I
self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
for p in self.parameters():
nn.init.normal_(p, std=0.02)
self._fn = _get_ext().moe_forward_cuda
def forward(
self,
x: torch.Tensor,
expert_ids: torch.Tensor,
expert_weights: torch.Tensor,
) -> torch.Tensor:
return self._fn(
x,
expert_ids,
expert_weights,
self.w1_routed,
self.w2_routed,
self.w1_shared,
self.w2_shared,
)
20261119_123612_agy_gemini-4-neon-xhigh_01_glm52_fused_moe