KernelBench cuda · RTX PRO 6000
GLM-5.3 Fused MoE Gemini 4 Carbon (High)
manually audited: clean
Gemini 4 Carbon wrote the GLM-5.2 MoE layer as hand-written CUDA with inline PTX for SM120. One 512-thread CTA counting-sorts the (token, slot) pairs by expert and builds a tile table (tile height 32, 64 or 128 rows picked from the average tokens per expert); two grouped GEMMs then run on TMA-loaded weights with an mbarrier pipeline, cp.async-gathered activation rows, ldmatrix and bf16 mma.sync, with silu(gate)*up fused into the first GEMM's epilogue and the routing weight applied in the second before red.global.add.v4.f32 scatters rows into an fp32 buffer. The shared expert is a dense TMA GEMM pair that writes that buffer first, a cast kernel produces bf16, and T <= 2 takes a separate two-kernel persistent GEMV path.
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(36.7% · 36.0% · 0.3% · 45.5% · 5.7% · 11.1%) = 10.2%
Kernel source (redacted)
from __future__ import annotations
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline
OP_TYPE = "glm52_fused_moe"
SUPPORTED_PRECISIONS = ["bf16"]
HARDWARE_REQUIRED = ["RTX_PRO_6000"]
_CUDA_SRC = r"""
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <ATen/cuda/CUDAContext.h>
#include <torch/extension.h>
#include <stdint.h>
#include <algorithm>
namespace {
struct alignas(16) TileInfo {
int32_t expert_id;
int32_t row_offset;
int32_t valid_rows;
int32_t pad;
};
struct Workspace {
torch::Tensor h_routed;
torch::Tensor h_shared;
torch::Tensor out_fp32;
torch::Tensor sorted_token_ids;
torch::Tensor sorted_weights;
torch::Tensor tile_map;
torch::Tensor num_tiles;
torch::Tensor block_hist;
torch::Tensor expert_row_offsets;
__nv_bfloat16* h_routed_ptr = nullptr;
__nv_bfloat16* h_shared_ptr = nullptr;
float* out_fp32_ptr = nullptr;
int32_t* sorted_tok_ptr = nullptr;
float* sorted_wt_ptr = nullptr;
TileInfo* tile_map_ptr = nullptr;
int32_t* num_tiles_ptr = nullptr;
int32_t* block_hist_ptr = nullptr;
int32_t* expert_off_ptr = nullptr;
int64_t cap_hr = 0;
int64_t cap_hs = 0;
int64_t cap_out = 0;
int64_t cap_m = 0;
int64_t cap_tm = 0;
int64_t cap_bh = 0;
int64_t cap_eo = 0;
int dev_idx = -1;
inline void ensure(
int64_t M_tot,
int64_t T,
int64_t E,
int64_t H,
int64_t I,
int64_t max_tiles,
int64_t sort_blocks,
torch::Device dev
) {
int d = dev.index();
int64_t need_hr = std::max<int64_t>(M_tot, 32) * I;
int64_t need_hs = std::max<int64_t>(T, 1) * I;
int64_t need_out = std::max<int64_t>(T, 1) * H;
int64_t need_m = std::max<int64_t>(M_tot, 32);
int64_t need_tm = std::max<int64_t>(max_tiles, 32) * 4;
int64_t need_bh = std::max<int64_t>(sort_blocks * E, 1024);
int64_t need_eo = std::max<int64_t>(E + 4, 1028);
if (__builtin_expect(
d == dev_idx &&
need_hr <= cap_hr &&
need_hs <= cap_hs &&
need_out <= cap_out &&
need_m <= cap_m &&
need_tm <= cap_tm &&
need_bh <= cap_bh &&
need_eo <= cap_eo, 1)) {
return;
}
auto opt_bf16 = torch::TensorOptions().dtype(torch::kBFloat16).device(dev);
auto opt_f32 = torch::TensorOptions().dtype(torch::kFloat32).device(dev);
auto opt_i32 = torch::TensorOptions().dtype(torch::kInt32).device(dev);
if (d != dev_idx) {
cap_hr = cap_hs = cap_out = cap_m = cap_tm = cap_bh = cap_eo = 0;
dev_idx = d;
}
if (need_hr > cap_hr) {
h_routed = torch::empty({need_hr}, opt_bf16);
h_routed_ptr = reinterpret_cast<__nv_bfloat16*>(h_routed.data_ptr());
cap_hr = need_hr;
}
if (need_hs > cap_hs) {
h_shared = torch::empty({need_hs}, opt_bf16);
h_shared_ptr = reinterpret_cast<__nv_bfloat16*>(h_shared.data_ptr());
cap_hs = need_hs;
}
if (need_out > cap_out) {
out_fp32 = torch::empty({need_out}, opt_f32);
out_fp32_ptr = out_fp32.data_ptr<float>();
cap_out = need_out;
}
if (need_m > cap_m) {
sorted_token_ids = torch::empty({need_m}, opt_i32);
sorted_weights = torch::empty({need_m}, opt_f32);
sorted_tok_ptr = sorted_token_ids.data_ptr<int32_t>();
sorted_wt_ptr = sorted_weights.data_ptr<float>();
cap_m = need_m;
}
if (need_tm > cap_tm) {
tile_map = torch::empty({need_tm}, opt_i32);
tile_map_ptr = reinterpret_cast<TileInfo*>(tile_map.data_ptr());
cap_tm = need_tm;
}
if (!num_tiles.defined() || num_tiles.device() != dev) {
num_tiles = torch::empty({4}, opt_i32);
num_tiles_ptr = num_tiles.data_ptr<int32_t>();
}
if (need_bh > cap_bh) {
block_hist = torch::empty({need_bh}, opt_i32);
block_hist_ptr = block_hist.data_ptr<int32_t>();
cap_bh = need_bh;
}
if (need_eo > cap_eo) {
expert_row_offsets = torch::empty({need_eo}, opt_i32);
expert_off_ptr = expert_row_offsets.data_ptr<int32_t>();
cap_eo = need_eo;
}
}
};
static Workspace g_ws;
// ============================================================================
// Device PTX Helpers (SM120 Blackwell)
// ============================================================================
__device__ __forceinline__ uint32_t smem_swizzle_128b(uint32_t base_smem, int r, int c_bf16) {
int chunk = (c_bf16 >> 3) ^ (r & 7);
return base_smem + (r << 7) + (chunk << 4);
}
__device__ __forceinline__ uint64_t make_policy_evict_first() {
uint64_t p;
asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;\n" : "=l"(p));
return p;
}
__device__ __forceinline__ uint64_t make_policy_evict_last() {
uint64_t p;
asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;\n" : "=l"(p));
return p;
}
__device__ __forceinline__ void mbar_init(uint32_t mbar_smem, uint32_t count) {
asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(mbar_smem), "r"(count));
}
__device__ __forceinline__ void mbar_expect_tx(uint32_t mbar_smem, uint32_t bytes) {
asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" :: "r"(mbar_smem), "r"(bytes));
}
__device__ __forceinline__ void mbar_wait(uint32_t mbar_smem, uint32_t parity) {
asm volatile(
"{\n"
".reg .pred P_DONE;\n"
"MBAR_SPIN_%=:\n"
"mbarrier.try_wait.parity.shared::cta.b64 P_DONE, [%0], %1;\n"
"@!P_DONE bra MBAR_SPIN_%=;\n"
"}\n"
:: "r"(mbar_smem), "r"(parity)
);
}
__device__ __forceinline__ void tma_load_2d_hint(
uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem,
int32_t c0, int32_t c1, uint64_t cache_policy
) {
asm volatile(
"cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint"
" [%0], [%1, {%3, %4}], [%2], %5;\n"
:: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1), "l"(cache_policy)
: "memory"
);
}
__device__ __forceinline__ void tma_load_2d_lru(
uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem,
int32_t c0, int32_t c1
) {
asm volatile(
"cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes"
" [%0], [%1, {%3, %4}], [%2];\n"
:: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1)
: "memory"
);
}
__device__ __forceinline__ void tma_load_3d_hint(
uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem,
int32_t c0, int32_t c1, int32_t c2, uint64_t cache_policy
) {
asm volatile(
"cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint"
" [%0], [%1, {%3, %4, %5}], [%2], %6;\n"
:: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2), "l"(cache_policy)
: "memory"
);
}
__device__ __forceinline__ void tma_load_3d_lru(
uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem,
int32_t c0, int32_t c1, int32_t c2
) {
asm volatile(
"cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes"
" [%0], [%1, {%3, %4, %5}], [%2];\n"
:: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2)
: "memory"
);
}
__device__ __forceinline__ void cp_async_16b_pred(
uint32_t dst_smem, const void* src_gmem, int src_bytes
) {
asm volatile(
"cp.async.cg.shared.global [%0], [%1], 16, %2;\n"
:: "r"(dst_smem), "l"(src_gmem), "r"(src_bytes)
);
}
__device__ __forceinline__ void ldmatrix_x4(uint32_t (&r)[4], uint32_t smem_addr) {
asm volatile(
"ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
: "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3])
: "r"(smem_addr)
);
}
__device__ __forceinline__ void mma_m16n8k16_bf16(
float (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1
) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n"
: "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1)
);
}
__device__ __forceinline__ void red_add_v4_f32(float* dst, float v0, float v1, float v2, float v3) {
asm volatile(
"red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n"
:: "l"(dst), "f"(v0), "f"(v1), "f"(v2), "f"(v3)
: "memory"
);
}
__device__ __forceinline__ uint4 ld_global_cs_u4(const void* ptr) {
uint4 v;
asm volatile(
"ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr)
);
return v;
}
__device__ __forceinline__ uint4 ld_global_nc_u4(const void* ptr) {
uint4 v;
asm volatile(
"ld.global.nc.v4.u32 {%0, %1, %2, %3}, [%4];\n"
: "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
: "l"(ptr)
);
return v;
}
__device__ __forceinline__ float dot8_bf16_f32(uint4 a, uint4 b) {
float2 ax = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&a.x));
float2 ay = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&a.y));
float2 az = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&a.z));
float2 aw = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&a.w));
float2 bx = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&b.x));
float2 by = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&b.y));
float2 bz = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&b.z));
float2 bw = __bfloat1622float2(*reinterpret_cast<const __nv_bfloat162*>(&b.w));
float s0 = fmaf(ax.x, bx.x, ax.y * bx.y);
float s1 = fmaf(ay.x, by.x, ay.y * by.y);
float s2 = fmaf(az.x, bz.x, az.y * bz.y);
float s3 = fmaf(aw.x, bw.x, aw.y * bw.y);
return (s0 + s1) + (s2 + s3);
}
extern __shared__ __align__(128) uint8_t g_dyn_smem[];
// ============================================================================
// Wave-Balanced Persistent Lockless Decode Path (T <= 2): 98% SM Wave Balance!
// 940 CTAs = 5 CTAs/SM * 188 SMs, 128 threads/CTA = 3760 active warps, 0 SMEM!
// ============================================================================
__global__ void __launch_bounds__(128, 5) decode_stage1_persistent_kernel(
const __nv_bfloat16* __restrict__ x, // (T, H)
const int64_t* __restrict__ expert_ids, // (T, top_k)
const __nv_bfloat16* __restrict__ w1_routed, // (E, 2*I, H)
const __nv_bfloat16* __restrict__ w1_shared, // (n_shared, 2*I, H)
__nv_bfloat16* __restrict__ h_decode, // (T * slots_per_tok, I)
float* __restrict__ out_fp32, // (T, H) - zeroed here!
int T, int top_k, int n_shared, int H, int I
) {
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
// Zero out_fp32 using first few CTAs
const int zero_idx = blockIdx.x * 128 + tid;
const int total_f4 = (T * H) >> 2;
if (zero_idx < total_f4) {
reinterpret_cast<float4*>(out_fp32)[zero_idx] = make_float4(0.f, 0.f, 0.f, 0.f);
}
const int slots_per_tok = top_k + n_shared;
const int total_pairs = T * slots_per_tok;
const int total_jobs = total_pairs * I; // 1 (gate, up) col per warp -> 18432 jobs at T=1!
const int global_warp = blockIdx.x * 4 + warp_id;
const int num_warps = gridDim.x * 4;
const int H_u4 = H >> 3;
for (int job = global_warp; job < total_jobs; job += num_warps) {
const int pair_idx = job / I;
const int i_col = job - pair_idx * I;
const int t = pair_idx / slots_per_tok;
const int slot = pair_idx - t * slots_per_tok;
const __nv_bfloat16* w1_base;
if (slot < top_k) {
int64_t eid = expert_ids[(int64_t)t * top_k + slot];
w1_base = w1_routed + eid * (2LL * I * H);
} else {
int sid = slot - top_k;
w1_base = w1_shared + (int64_t)sid * (2LL * I * H);
}
const uint4* x_row_u4 = reinterpret_cast<const uint4*>(x + (int64_t)t * H);
const __nv_bfloat16* g_ptr = w1_base + (int64_t)i_col * H;
const __nv_bfloat16* u_ptr = w1_base + (int64_t)(I + i_col) * H;
float acc_g = 0.0f, acc_u = 0.0f;
#pragma unroll 8
for (int k_u4 = lane; k_u4 < H_u4; k_u4 += 32) {
uint4 xv = ld_global_nc_u4(x_row_u4 + k_u4);
int k_elem = k_u4 << 3;
uint4 gv = ld_global_cs_u4(g_ptr + k_elem);
uint4 uv = ld_global_cs_u4(u_ptr + k_elem);
acc_g += dot8_bf16_f32(xv, gv);
acc_u += dot8_bf16_f32(xv, uv);
}
#pragma unroll
for (int mask = 16; mask > 0; mask >>= 1) {
acc_g += __shfl_down_sync(0xffffffff, acc_g, mask);
acc_u += __shfl_down_sync(0xffffffff, acc_u, mask);
}
if (lane == 0) {
float h_val = (acc_g / (1.0f + __expf(-acc_g))) * acc_u;
h_decode[(int64_t)pair_idx * I + i_col] = __float2bfloat16_rn(h_val);
}
}
}
// 940 CTAs, 64 threads/CTA (2 warps) = 1880 warps; 4 rows of w2 per warp -> 9216 jobs (4.902 jobs/warp!)
__global__ void __launch_bounds__(64, 10) decode_stage2_persistent_kernel(
const __nv_bfloat16* __restrict__ h_decode, // (T * slots_per_tok, I)
const int64_t* __restrict__ expert_ids, // (T, top_k)
const __nv_bfloat16* __restrict__ expert_weights, // (T, top_k)
const __nv_bfloat16* __restrict__ w2_routed, // (E, H, I)
const __nv_bfloat16* __restrict__ w2_shared, // (n_shared, H, I)
float* __restrict__ out_fp32, // (T, H)
int T, int top_k, int n_shared, int H, int I
) {
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int slots_per_tok = top_k + n_shared;
const int total_pairs = T * slots_per_tok;
const int h_quads = (H + 3) >> 2;
const int total_jobs = total_pairs * h_quads; // 9216 jobs at T=1
const int global_warp = blockIdx.x * 2 + warp_id;
const int num_warps = gridDim.x * 2;
const int I_u4 = I >> 3;
for (int job = global_warp; job < total_jobs; job += num_warps) {
const int pair_idx = job / h_quads;
const int h_q = job - pair_idx * h_quads;
const int h0 = h_q << 2;
const int t = pair_idx / slots_per_tok;
const int slot = pair_idx - t * slots_per_tok;
float wt = 1.0f;
const __nv_bfloat16* w2_base;
if (slot < top_k) {
int64_t eid = expert_ids[(int64_t)t * top_k + slot];
wt = __bfloat162float(expert_weights[(int64_t)t * top_k + slot]);
w2_base = w2_routed + eid * ((int64_t)H * I);
} else {
int sid = slot - top_k;
w2_base = w2_shared + (int64_t)sid * ((int64_t)H * I);
}
const uint4* h_row_u4 = reinterpret_cast<const uint4*>(h_decode + (int64_t)pair_idx * I);
const __nv_bfloat16* r0 = w2_base + (int64_t)(h0 + 0) * I;
const __nv_bfloat16* r1 = (h0 + 1 < H) ? (r0 + I) : r0;
const __nv_bfloat16* r2 = (h0 + 2 < H) ? (r0 + 2LL * I) : r0;
const __nv_bfloat16* r3 = (h0 + 3 < H) ? (r0 + 3LL * I) : r0;
float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f;
#pragma unroll 8
for (int k_u4 = lane; k_u4 < I_u4; k_u4 += 32) {
uint4 hv = ld_global_nc_u4(h_row_u4 + k_u4);
int k_elem = k_u4 << 3;
uint4 wv0 = ld_global_cs_u4(r0 + k_elem);
uint4 wv1 = ld_global_cs_u4(r1 + k_elem);
uint4 wv2 = ld_global_cs_u4(r2 + k_elem);
uint4 wv3 = ld_global_cs_u4(r3 + k_elem);
a0 += dot8_bf16_f32(hv, wv0);
a1 += dot8_bf16_f32(hv, wv1);
a2 += dot8_bf16_f32(hv, wv2);
a3 += dot8_bf16_f32(hv, wv3);
}
#pragma unroll
for (int mask = 16; mask > 0; mask >>= 1) {
a0 += __shfl_down_sync(0xffffffff, a0, mask);
a1 += __shfl_down_sync(0xffffffff, a1, mask);
a2 += __shfl_down_sync(0xffffffff, a2, mask);
a3 += __shfl_down_sync(0xffffffff, a3, mask);
}
if (lane == 0) {
float* dst = out_fp32 + (int64_t)t * H + h0;
if (h0 + 3 < H) {
red_add_v4_f32(dst, a0 * wt, a1 * wt, a2 * wt, a3 * wt);
} else {
atomicAdd(dst + 0, a0 * wt);
if (h0 + 1 < H) atomicAdd(dst + 1, a1 * wt);
if (h0 + 2 < H) atomicAdd(dst + 2, a2 * wt);
}
}
}
}
// ============================================================================
// Token Counting-Sort & M-Tile Table Kernels
// ============================================================================
__global__ void __launch_bounds__(512) moe_fused_single_sort_kernel(
const int64_t* __restrict__ expert_ids,
const __nv_bfloat16* __restrict__ expert_weights,
int32_t* __restrict__ sorted_token_ids,
float* __restrict__ sorted_weights,
TileInfo* __restrict__ tile_map,
int32_t* __restrict__ d_num_tiles,
int M_tot, int top_k, int E, int BM
) {
__shared__ int32_t smem_hist[512];
__shared__ int32_t smem_row_off[513];
__shared__ int32_t smem_tile_off[513];
__shared__ int32_t smem_cur_off[512];
const int tid = threadIdx.x;
if (tid < E) smem_hist[tid] = 0;
__syncthreads();
int my_e[16];
int my_idx[16];
int n_local = 0;
for (int idx = tid; idx < M_tot; idx += 512) {
int e = static_cast<int>(expert_ids[idx]);
my_e[n_local] = e;
my_idx[n_local] = idx;
++n_local;
atomicAdd(&smem_hist[e], 1);
}
__syncthreads();
if (tid < 32) {
int r_run = 0;
int t_run = 0;
for (int base = 0; base < E; base += 32) {
int e = base + tid;
int c = (e < E) ? smem_hist[e] : 0;
int nt = (c + BM - 1) / BM;
int r_inc = c;
int t_inc = nt;
#pragma unroll
for (int offset = 1; offset < 32; offset <<= 1) {
int nr = __shfl_up_sync(0xffffffff, r_inc, offset);
int nti = __shfl_up_sync(0xffffffff, t_inc, offset);
if (tid >= offset) {
r_inc += nr;
t_inc += nti;
}
}
if (e < E) {
smem_row_off[e] = r_run + r_inc - c;
smem_cur_off[e] = r_run + r_inc - c;
smem_tile_off[e] = t_run + t_inc - nt;
}
r_run += __shfl_sync(0xffffffff, r_inc, 31);
t_run += __shfl_sync(0xffffffff, t_inc, 31);
}
if (tid == 0) {
smem_row_off[E] = r_run;
smem_tile_off[E] = t_run;
*d_num_tiles = t_run;
}
}
__syncthreads();
if (tid < E) {
int count = smem_hist[tid];
int r_off = smem_row_off[tid];
int t_off = smem_tile_off[tid];
int nt = smem_tile_off[tid + 1] - t_off;
for (int t = 0; t < nt; ++t) {
TileInfo ti;
ti.expert_id = tid;
ti.row_offset = r_off + t * BM;
ti.valid_rows = min(BM, count - t * BM);
ti.pad = 0;
tile_map[t_off + t] = ti;
}
}
for (int i = 0; i < n_local; ++i) {
int e = my_e[i];
int idx = my_idx[i];
int pos = atomicAdd(&smem_cur_off[e], 1);
sorted_token_ids[pos] = idx / top_k;
sorted_weights[pos] = __bfloat162float(expert_weights[idx]);
}
}
__global__ void __launch_bounds__(256) moe_hist_kernel(
const int64_t* __restrict__ expert_ids,
int32_t* __restrict__ block_hist,
int M_tot, int E
) {
int32_t* smem_h = reinterpret_cast<int32_t*>(g_dyn_smem);
const int tid = threadIdx.x;
for (int e = tid; e < E; e += 256) smem_h[e] = 0;
__syncthreads();
const int n_blks = gridDim.x;
const int chunk = (M_tot + n_blks - 1) / n_blks;
const int start = blockIdx.x * chunk;
const int end = min(M_tot, start + chunk);
for (int idx = start + tid; idx < end; idx += 256) {
int e = static_cast<int>(expert_ids[idx]);
atomicAdd(&smem_h[e], 1);
}
__syncthreads();
for (int e = tid; e < E; e += 256) {
block_hist[blockIdx.x * E + e] = smem_h[e];
}
}
__global__ void __launch_bounds__(256) moe_scan_tiles_kernel(
int32_t* __restrict__ block_hist,
int32_t* __restrict__ expert_row_offsets,
TileInfo* __restrict__ tile_map,
int32_t* __restrict__ d_num_tiles,
int sort_blocks, int E, int BM
) {
int32_t* dyn_s = reinterpret_cast<int32_t*>(g_dyn_smem);
int32_t* s_counts = dyn_s;
int32_t* s_row_off = dyn_s + E + 1;
int32_t* s_tile_off = dyn_s + 2 * (E + 1);
const int tid = threadIdx.x;
for (int e = tid; e < E; e += 256) {
int sum = 0;
for (int b = 0; b < sort_blocks; ++b) {
int c = block_hist[b * E + e];
block_hist[b * E + e] = sum;
sum += c;
}
s_counts[e] = sum;
}
__syncthreads();
if (tid < 32) {
int r_run = 0;
int t_run = 0;
for (int base = 0; base < E; base += 32) {
int e = base + tid;
int c = (e < E) ? s_counts[e] : 0;
int nt = (c + BM - 1) / BM;
int r_inc = c;
int t_inc = nt;
#pragma unroll
for (int offset = 1; offset < 32; offset <<= 1) {
int nr = __shfl_up_sync(0xffffffff, r_inc, offset);
int nti = __shfl_up_sync(0xffffffff, t_inc, offset);
if (tid >= offset) {
r_inc += nr;
t_inc += nti;
}
}
if (e < E) {
int ro = r_run + r_inc - c;
s_row_off[e] = ro;
expert_row_offsets[e] = ro;
s_tile_off[e] = t_run + t_inc - nt;
}
r_run += __shfl_sync(0xffffffff, r_inc, 31);
t_run += __shfl_sync(0xffffffff, t_inc, 31);
}
if (tid == 0) {
s_row_off[E] = r_run;
s_tile_off[E] = t_run;
*d_num_tiles = t_run;
}
}
__syncthreads();
for (int e = tid; e < E; e += 256) {
int count = s_counts[e];
int r_off = s_row_off[e];
int t_off = s_tile_off[e];
int nt = s_tile_off[e + 1] - t_off;
for (int t = 0; t < nt; ++t) {
TileInfo ti;
ti.expert_id = e;
ti.row_offset = r_off + t * BM;
ti.valid_rows = min(BM, count - t * BM);
ti.pad = 0;
tile_map[t_off + t] = ti;
}
}
}
__global__ void __launch_bounds__(256) moe_scatter_kernel(
const int64_t* __restrict__ expert_ids,
const __nv_bfloat16* __restrict__ expert_weights,
const int32_t* __restrict__ block_hist,
const int32_t* __restrict__ expert_row_offsets,
int32_t* __restrict__ sorted_token_ids,
float* __restrict__ sorted_weights,
int M_tot, int top_k, int E
) {
int32_t* smem_off = reinterpret_cast<int32_t*>(g_dyn_smem);
const int tid = threadIdx.x;
for (int e = tid; e < E; e += 256) {
smem_off[e] = expert_row_offsets[e] + block_hist[blockIdx.x * E + e];
}
__syncthreads();
const int n_blks = gridDim.x;
const int chunk = (M_tot + n_blks - 1) / n_blks;
const int start = blockIdx.x * chunk;
const int end = min(M_tot, start + chunk);
for (int idx = start + tid; idx < end; idx += 256) {
int e = static_cast<int>(expert_ids[idx]);
int pos = atomicAdd(&smem_off[e], 1);
sorted_token_ids[pos] = idx / top_k;
sorted_weights[pos] = __bfloat162float(expert_weights[idx]);
}
}
// ============================================================================
// Shared Expert Dense TMA + MMA Kernels (Register-Pipelined + L2 Weight Reuse!)
// ============================================================================
template <int BM, int STAGES, int NUM_WARPS>
__global__ void __launch_bounds__(NUM_WARPS * 32) dense_swiglu_stage1_kernel(
const __grid_constant__ CUtensorMap tmap_A, // x: (T, H)
const __grid_constant__ CUtensorMap tmap_B, // w1_shared[s]: (2*I, H) - 32 MB fits in 96 MB L2!
__nv_bfloat16* __restrict__ h_out, // (T, I)
int T, int H, int I
) {
constexpr int BN_OUT = 64;
constexpr int BK = 64;
constexpr int NUM_THREADS = NUM_WARPS * 32;
constexpr int WARPS_M = NUM_WARPS / 2;
constexpr int WM = BM / WARPS_M;
constexpr int M_TILES = WM / 16;
// Schedule M-tiles adjacent per N-tile so w1_shared slice is reused across all M-tiles in L2!
const int m_base = blockIdx.x * BM;
const int n_base = blockIdx.y * BN_OUT;
if (m_base >= T || n_base >= I) return;
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int wm_idx = warp_id >> 1;
const int wn_idx = warp_id & 1;
constexpr uint32_t A_STAGE_BYTES = BM * 128;
constexpr uint32_t B_HALF_BYTES = BN_OUT * 128;
constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_HALF_BYTES;
constexpr uint32_t EPI_BYTES = BM * 72 * sizeof(__nv_bfloat16);
constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u;
uint8_t* dyn_smem = g_dyn_smem;
uint64_t* mbar = reinterpret_cast<uint64_t*>(dyn_smem + MBAR_OFF);
uint32_t smem_base = static_cast<uint32_t>(__cvta_generic_to_shared(dyn_smem));
uint32_t sMbar[STAGES];
#pragma unroll
for (int s = 0; s < STAGES; ++s) {
sMbar[s] = static_cast<uint32_t>(__cvta_generic_to_shared(&mbar[s]));
}
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1);
}
__syncthreads();
const uint64_t pol_A = make_policy_evict_last();
const int num_k_tiles = (H + BK - 1) / BK;
#pragma unroll
for (int kt = 0; kt < STAGES - 1; ++kt) {
if (kt < num_k_tiles && tid == 0) {
uint32_t s_off = smem_base + kt * STAGE_BYTES;
mbar_expect_tx(sMbar[kt], STAGE_BYTES);
tma_load_2d_hint(s_off, &tmap_A, sMbar[kt], kt * BK, m_base, pol_A);
tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[kt], kt * BK, n_base);
tma_load_2d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_B, sMbar[kt], kt * BK, I + n_base);
}
}
float acc_gate[M_TILES][4][4];
float acc_up[M_TILES][4][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi)
#pragma unroll
for (int ni = 0; ni < 4; ++ni)
#pragma unroll
for (int r = 0; r < 4; ++r) {
acc_gate[mi][ni][r] = 0.0f;
acc_up[mi][ni][r] = 0.0f;
}
const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3);
const int a_lane_c = (lane >> 4) << 3;
const int b_lane_r = (lane & 7) + ((lane >> 4) << 3);
const int b_lane_c = ((lane >> 3) & 1) << 3;
const int bn0 = wn_idx * 32 + b_lane_r;
for (int kt = 0; kt < num_k_tiles; ++kt) {
int kt_next = kt + STAGES - 1;
if (kt_next < num_k_tiles && tid == 0) {
int p_next = kt_next % STAGES;
uint32_t s_off = smem_base + p_next * STAGE_BYTES;
mbar_expect_tx(sMbar[p_next], STAGE_BYTES);
tma_load_2d_hint(s_off, &tmap_A, sMbar[p_next], kt_next * BK, m_base, pol_A);
tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[p_next], kt_next * BK, n_base);
tma_load_2d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_B, sMbar[p_next], kt_next * BK, I + n_base);
}
int p_cur = kt % STAGES;
if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1);
__syncthreads();
uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES;
uint32_t sBg_cur = sA_cur + A_STAGE_BYTES;
uint32_t sBu_cur = sBg_cur + B_HALF_BYTES;
// Register-level double buffer across ks = 0, 16, 32, 48
uint32_t a_reg[2][M_TILES][4];
uint32_t bg0_reg[2][4], bg1_reg[2][4], bu0_reg[2][4], bu1_reg[2][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c));
}
ldmatrix_x4(bg0_reg[0], smem_swizzle_128b(sBg_cur, bn0, b_lane_c));
ldmatrix_x4(bg1_reg[0], smem_swizzle_128b(sBg_cur, bn0 + 16, b_lane_c));
ldmatrix_x4(bu0_reg[0], smem_swizzle_128b(sBu_cur, bn0, b_lane_c));
ldmatrix_x4(bu1_reg[0], smem_swizzle_128b(sBu_cur, bn0 + 16, b_lane_c));
#pragma unroll
for (int ks_idx = 0; ks_idx < 4; ++ks_idx) {
int cur_b = ks_idx & 1;
if (ks_idx < 3) {
int nxt_b = cur_b ^ 1;
int ks_nxt = (ks_idx + 1) * 16;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c));
}
ldmatrix_x4(bg0_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0, ks_nxt + b_lane_c));
ldmatrix_x4(bg1_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0 + 16, ks_nxt + b_lane_c));
ldmatrix_x4(bu0_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0, ks_nxt + b_lane_c));
ldmatrix_x4(bu1_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0 + 16, ks_nxt + b_lane_c));
}
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
mma_m16n8k16_bf16(acc_gate[mi][0], a_reg[cur_b][mi], bg0_reg[cur_b][0], bg0_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_gate[mi][1], a_reg[cur_b][mi], bg0_reg[cur_b][2], bg0_reg[cur_b][3]);
mma_m16n8k16_bf16(acc_gate[mi][2], a_reg[cur_b][mi], bg1_reg[cur_b][0], bg1_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_gate[mi][3], a_reg[cur_b][mi], bg1_reg[cur_b][2], bg1_reg[cur_b][3]);
mma_m16n8k16_bf16(acc_up[mi][0], a_reg[cur_b][mi], bu0_reg[cur_b][0], bu0_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_up[mi][1], a_reg[cur_b][mi], bu0_reg[cur_b][2], bu0_reg[cur_b][3]);
mma_m16n8k16_bf16(acc_up[mi][2], a_reg[cur_b][mi], bu1_reg[cur_b][0], bu1_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_up[mi][3], a_reg[cur_b][mi], bu1_reg[cur_b][2], bu1_reg[cur_b][3]);
}
}
__syncthreads();
}
constexpr int EPI_STRIDE = 72;
__nv_bfloat16* smem_epi = reinterpret_cast<__nv_bfloat16*>(dyn_smem);
const int gid = lane >> 2;
const int tid_g = lane & 3;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r0 = wm_idx * WM + mi * 16 + gid;
int r1 = r0 + 8;
#pragma unroll
for (int ni = 0; ni < 4; ++ni) {
int c = wn_idx * 32 + ni * 8 + tid_g * 2;
float g00 = acc_gate[mi][ni][0], g01 = acc_gate[mi][ni][1];
float g10 = acc_gate[mi][ni][2], g11 = acc_gate[mi][ni][3];
float u00 = acc_up[mi][ni][0], u01 = acc_up[mi][ni][1];
float u10 = acc_up[mi][ni][2], u11 = acc_up[mi][ni][3];
float h00 = (g00 / (1.0f + __expf(-g00))) * u00;
float h01 = (g01 / (1.0f + __expf(-g01))) * u01;
float h10 = (g10 / (1.0f + __expf(-g10))) * u10;
float h11 = (g11 / (1.0f + __expf(-g11))) * u11;
*reinterpret_cast<__nv_bfloat162*>(&smem_epi[r0 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h00, h01);
*reinterpret_cast<__nv_bfloat162*>(&smem_epi[r1 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h10, h11);
}
}
__syncthreads();
const int valid_m = min(BM, T - m_base);
for (int idx = tid; idx < valid_m * 8; idx += NUM_THREADS) {
int r = idx >> 3;
int c8 = (idx & 7) << 3;
if (n_base + c8 + 7 < I) {
uint4 v = *reinterpret_cast<const uint4*>(&smem_epi[r * EPI_STRIDE + c8]);
*reinterpret_cast<uint4*>(&h_out[(int64_t)(m_base + r) * I + n_base + c8]) = v;
} else {
for (int j = 0; j < 8 && n_base + c8 + j < I; ++j) {
h_out[(int64_t)(m_base + r) * I + n_base + c8 + j] = smem_epi[r * EPI_STRIDE + c8 + j];
}
}
}
}
template <int BM, int STAGES, int NUM_WARPS, bool ACCUMULATE>
__global__ void __launch_bounds__(NUM_WARPS * 32) dense_down_stage2_kernel(
const __grid_constant__ CUtensorMap tmap_A, // h_shared: (T, I)
const __grid_constant__ CUtensorMap tmap_B, // w2_shared[s]: (H, I) - 16 MB fits in 96 MB L2!
float* __restrict__ out_fp32, // (T, H)
int T, int H, int I
) {
constexpr int BN = 128;
constexpr int BK = 64;
constexpr int WARPS_M = NUM_WARPS / 2;
constexpr int WM = BM / WARPS_M;
constexpr int M_TILES = WM / 16;
const int m_base = blockIdx.x * BM;
const int n_base = blockIdx.y * BN;
if (m_base >= T || n_base >= H) return;
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int wm_idx = warp_id >> 1;
const int wn_idx = warp_id & 1;
constexpr uint32_t A_STAGE_BYTES = BM * 128;
constexpr uint32_t B_STAGE_BYTES = BN * 128;
constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES;
constexpr uint32_t EPI_BYTES = BM * 132 * sizeof(float);
constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u;
uint8_t* dyn_smem = g_dyn_smem;
uint64_t* mbar = reinterpret_cast<uint64_t*>(dyn_smem + MBAR_OFF);
uint32_t smem_base = static_cast<uint32_t>(__cvta_generic_to_shared(dyn_smem));
uint32_t sMbar[STAGES];
#pragma unroll
for (int s = 0; s < STAGES; ++s) {
sMbar[s] = static_cast<uint32_t>(__cvta_generic_to_shared(&mbar[s]));
}
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1);
}
__syncthreads();
const uint64_t pol_A = make_policy_evict_last();
const int num_k_tiles = (I + BK - 1) / BK;
#pragma unroll
for (int kt = 0; kt < STAGES - 1; ++kt) {
if (kt < num_k_tiles && tid == 0) {
uint32_t s_off = smem_base + kt * STAGE_BYTES;
mbar_expect_tx(sMbar[kt], STAGE_BYTES);
tma_load_2d_hint(s_off, &tmap_A, sMbar[kt], kt * BK, m_base, pol_A);
tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[kt], kt * BK, n_base);
}
}
float acc[M_TILES][8][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi)
#pragma unroll
for (int ni = 0; ni < 8; ++ni)
#pragma unroll
for (int r = 0; r < 4; ++r)
acc[mi][ni][r] = 0.0f;
const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3);
const int a_lane_c = (lane >> 4) << 3;
const int b_lane_r = (lane & 7) + ((lane >> 4) << 3);
const int b_lane_c = ((lane >> 3) & 1) << 3;
for (int kt = 0; kt < num_k_tiles; ++kt) {
int kt_next = kt + STAGES - 1;
if (kt_next < num_k_tiles && tid == 0) {
int p_next = kt_next % STAGES;
uint32_t s_off = smem_base + p_next * STAGE_BYTES;
mbar_expect_tx(sMbar[p_next], STAGE_BYTES);
tma_load_2d_hint(s_off, &tmap_A, sMbar[p_next], kt_next * BK, m_base, pol_A);
tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[p_next], kt_next * BK, n_base);
}
int p_cur = kt % STAGES;
if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1);
__syncthreads();
uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES;
uint32_t sB_cur = sA_cur + A_STAGE_BYTES;
uint32_t a_reg[2][M_TILES][4];
uint32_t b_reg[2][4][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
ldmatrix_x4(b_reg[0][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, b_lane_c));
}
#pragma unroll
for (int ks_idx = 0; ks_idx < 4; ++ks_idx) {
int cur_b = ks_idx & 1;
if (ks_idx < 3) {
int nxt_b = cur_b ^ 1;
int ks_nxt = (ks_idx + 1) * 16;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
ldmatrix_x4(b_reg[nxt_b][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, ks_nxt + b_lane_c));
}
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
mma_m16n8k16_bf16(acc[mi][np * 2 + 0], a_reg[cur_b][mi], b_reg[cur_b][np][0], b_reg[cur_b][np][1]);
mma_m16n8k16_bf16(acc[mi][np * 2 + 1], a_reg[cur_b][mi], b_reg[cur_b][np][2], b_reg[cur_b][np][3]);
}
}
}
__syncthreads();
}
constexpr int EPI_STRIDE = 132;
float* smem_epi = reinterpret_cast<float*>(dyn_smem);
const int gid = lane >> 2;
const int tid_g = lane & 3;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r0 = wm_idx * WM + mi * 16 + gid;
int r1 = r0 + 8;
#pragma unroll
for (int ni = 0; ni < 8; ++ni) {
int c = wn_idx * 64 + ni * 8 + tid_g * 2;
*reinterpret_cast<float2*>(&smem_epi[r0 * EPI_STRIDE + c]) = make_float2(acc[mi][ni][0], acc[mi][ni][1]);
*reinterpret_cast<float2*>(&smem_epi[r1 * EPI_STRIDE + c]) = make_float2(acc[mi][ni][2], acc[mi][ni][3]);
}
}
__syncthreads();
const int valid_m = min(BM, T - m_base);
for (int m = warp_id; m < valid_m; m += NUM_WARPS) {
float* dst_row = out_fp32 + (int64_t)(m_base + m) * H + n_base;
#pragma unroll
for (int c = lane * 4; c < BN; c += 128) {
if (n_base + c + 3 < H) {
float4 v = *reinterpret_cast<const float4*>(&smem_epi[m * EPI_STRIDE + c]);
if constexpr (ACCUMULATE) {
float4 prev = *reinterpret_cast<const float4*>(dst_row + c);
v.x += prev.x; v.y += prev.y; v.z += prev.z; v.w += prev.w;
}
*reinterpret_cast<float4*>(dst_row + c) = v;
} else {
for (int j = 0; j < 4 && n_base + c + j < H; ++j) {
if constexpr (ACCUMULATE) dst_row[c + j] += smem_epi[m * EPI_STRIDE + c + j];
else dst_row[c + j] = smem_epi[m * EPI_STRIDE + c + j];
}
}
}
}
}
// ============================================================================
// Routed Experts Grouped GEMM Stage 1 (Hole-Free Swizzled + Reg-Pipelined!)
// ============================================================================
template <int BM, int STAGES, int NUM_WARPS, bool STREAM_B, int GROUP_N>
__global__ void __launch_bounds__(NUM_WARPS * 32) routed_swiglu_stage1_kernel(
const __nv_bfloat16* __restrict__ x, // (T, H)
const __grid_constant__ CUtensorMap tmap_w1, // w1_routed: (E, 2*I, H)
const TileInfo* __restrict__ tile_map,
const int32_t* __restrict__ d_num_tiles,
const int32_t* __restrict__ sorted_token_ids,
__nv_bfloat16* __restrict__ h_interm, // (M_tot, I)
int H, int I, int n_tiles
) {
constexpr int BN_OUT = 64;
constexpr int BK = 64;
constexpr int NUM_THREADS = NUM_WARPS * 32;
constexpr int WARPS_M = NUM_WARPS / 2;
constexpr int WM = BM / WARPS_M;
constexpr int M_TILES = WM / 16;
const int total_m = *d_num_tiles;
const int job = blockIdx.x;
if (job >= total_m * n_tiles) return;
int m_tile, n_tile;
if constexpr (GROUP_N > 1) {
const int span = total_m * GROUP_N;
const int ng = job / span;
const int rem = job - ng * span;
const int g_eff = min(GROUP_N, n_tiles - ng * GROUP_N);
m_tile = rem / g_eff;
n_tile = ng * GROUP_N + (rem - m_tile * g_eff);
if (m_tile >= total_m) return;
} else {
n_tile = job / total_m;
m_tile = job - n_tile * total_m;
}
const int n_base = n_tile * BN_OUT;
const TileInfo ti = tile_map[m_tile];
const int expert_id = ti.expert_id;
const int row_offset = ti.row_offset;
const int valid_rows = ti.valid_rows;
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int wm_idx = warp_id >> 1;
const int wn_idx = warp_id & 1;
constexpr uint32_t A_STAGE_BYTES = BM * 128;
constexpr uint32_t B_HALF_BYTES = BN_OUT * 128;
constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_HALF_BYTES;
constexpr uint32_t EPI_BYTES = BM * 72 * sizeof(__nv_bfloat16);
constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u;
uint8_t* dyn_smem = g_dyn_smem;
uint64_t* mbar_B = reinterpret_cast<uint64_t*>(dyn_smem + MBAR_OFF);
uint32_t smem_base = static_cast<uint32_t>(__cvta_generic_to_shared(dyn_smem));
uint32_t sMbar[STAGES];
#pragma unroll
for (int s = 0; s < STAGES; ++s) {
sMbar[s] = static_cast<uint32_t>(__cvta_generic_to_shared(&mbar_B[s]));
}
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1);
}
constexpr int A_CHUNKS_PER_THREAD = (BM * 8) / NUM_THREADS;
const __nv_bfloat16* my_a_ptr[A_CHUNKS_PER_THREAD];
int my_a_bytes[A_CHUNKS_PER_THREAD];
uint32_t my_a_smem_off[A_CHUNKS_PER_THREAD];
#pragma unroll
for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) {
int idx = tid + c * NUM_THREADS;
int r = idx >> 3;
int col8 = (idx & 7) << 3;
my_a_smem_off[c] = smem_swizzle_128b(0, r, col8);
if (r < valid_rows) {
int32_t tok = sorted_token_ids[row_offset + r];
my_a_ptr[c] = x + (int64_t)tok * H + col8;
my_a_bytes[c] = 16;
} else {
my_a_ptr[c] = x;
my_a_bytes[c] = 0;
}
}
__syncthreads();
const uint64_t pol_B = STREAM_B ? make_policy_evict_first() : 0ULL;
const int num_k_tiles = (H + BK - 1) / BK;
#pragma unroll
for (int kt = 0; kt < STAGES - 1; ++kt) {
if (kt < num_k_tiles) {
uint32_t s_off = smem_base + kt * STAGE_BYTES;
if (tid == 0) {
mbar_expect_tx(sMbar[kt], 2 * B_HALF_BYTES);
if constexpr (STREAM_B) {
tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[kt], kt * BK, n_base, expert_id, pol_B);
tma_load_3d_hint(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[kt], kt * BK, I + n_base, expert_id, pol_B);
} else {
tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[kt], kt * BK, n_base, expert_id);
tma_load_3d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[kt], kt * BK, I + n_base, expert_id);
}
}
#pragma unroll
for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) {
cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt * BK, my_a_bytes[c]);
}
}
asm volatile("cp.async.commit_group;\n");
}
float acc_gate[M_TILES][4][4];
float acc_up[M_TILES][4][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi)
#pragma unroll
for (int ni = 0; ni < 4; ++ni)
#pragma unroll
for (int r = 0; r < 4; ++r) {
acc_gate[mi][ni][r] = 0.0f;
acc_up[mi][ni][r] = 0.0f;
}
const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3);
const int a_lane_c = (lane >> 4) << 3;
const int b_lane_r = (lane & 7) + ((lane >> 4) << 3);
const int b_lane_c = ((lane >> 3) & 1) << 3;
const int bn0 = wn_idx * 32 + b_lane_r;
for (int kt = 0; kt < num_k_tiles; ++kt) {
int kt_next = kt + STAGES - 1;
if (kt_next < num_k_tiles) {
int p_next = kt_next % STAGES;
uint32_t s_off = smem_base + p_next * STAGE_BYTES;
if (tid == 0) {
mbar_expect_tx(sMbar[p_next], 2 * B_HALF_BYTES);
if constexpr (STREAM_B) {
tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, n_base, expert_id, pol_B);
tma_load_3d_hint(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, I + n_base, expert_id, pol_B);
} else {
tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, n_base, expert_id);
tma_load_3d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, I + n_base, expert_id);
}
}
#pragma unroll
for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) {
cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt_next * BK, my_a_bytes[c]);
}
}
asm volatile("cp.async.commit_group;\n");
asm volatile("cp.async.wait_group %0;\n" :: "n"(STAGES - 1));
int p_cur = kt % STAGES;
if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1);
__syncthreads();
uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES;
uint32_t sBg_cur = sA_cur + A_STAGE_BYTES;
uint32_t sBu_cur = sBg_cur + B_HALF_BYTES;
uint32_t a_reg[2][M_TILES][4];
uint32_t bg0_reg[2][4], bg1_reg[2][4], bu0_reg[2][4], bu1_reg[2][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c));
}
ldmatrix_x4(bg0_reg[0], smem_swizzle_128b(sBg_cur, bn0, b_lane_c));
ldmatrix_x4(bg1_reg[0], smem_swizzle_128b(sBg_cur, bn0 + 16, b_lane_c));
ldmatrix_x4(bu0_reg[0], smem_swizzle_128b(sBu_cur, bn0, b_lane_c));
ldmatrix_x4(bu1_reg[0], smem_swizzle_128b(sBu_cur, bn0 + 16, b_lane_c));
#pragma unroll
for (int ks_idx = 0; ks_idx < 4; ++ks_idx) {
int cur_b = ks_idx & 1;
if (ks_idx < 3) {
int nxt_b = cur_b ^ 1;
int ks_nxt = (ks_idx + 1) * 16;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c));
}
ldmatrix_x4(bg0_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0, ks_nxt + b_lane_c));
ldmatrix_x4(bg1_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0 + 16, ks_nxt + b_lane_c));
ldmatrix_x4(bu0_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0, ks_nxt + b_lane_c));
ldmatrix_x4(bu1_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0 + 16, ks_nxt + b_lane_c));
}
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
mma_m16n8k16_bf16(acc_gate[mi][0], a_reg[cur_b][mi], bg0_reg[cur_b][0], bg0_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_gate[mi][1], a_reg[cur_b][mi], bg0_reg[cur_b][2], bg0_reg[cur_b][3]);
mma_m16n8k16_bf16(acc_gate[mi][2], a_reg[cur_b][mi], bg1_reg[cur_b][0], bg1_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_gate[mi][3], a_reg[cur_b][mi], bg1_reg[cur_b][2], bg1_reg[cur_b][3]);
mma_m16n8k16_bf16(acc_up[mi][0], a_reg[cur_b][mi], bu0_reg[cur_b][0], bu0_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_up[mi][1], a_reg[cur_b][mi], bu0_reg[cur_b][2], bu0_reg[cur_b][3]);
mma_m16n8k16_bf16(acc_up[mi][2], a_reg[cur_b][mi], bu1_reg[cur_b][0], bu1_reg[cur_b][1]);
mma_m16n8k16_bf16(acc_up[mi][3], a_reg[cur_b][mi], bu1_reg[cur_b][2], bu1_reg[cur_b][3]);
}
}
__syncthreads();
}
asm volatile("cp.async.wait_group 0;\n");
constexpr int EPI_STRIDE = 72;
__nv_bfloat16* smem_epi = reinterpret_cast<__nv_bfloat16*>(dyn_smem);
const int gid = lane >> 2;
const int tid_g = lane & 3;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r0 = wm_idx * WM + mi * 16 + gid;
int r1 = r0 + 8;
#pragma unroll
for (int ni = 0; ni < 4; ++ni) {
int c = wn_idx * 32 + ni * 8 + tid_g * 2;
float g00 = acc_gate[mi][ni][0], g01 = acc_gate[mi][ni][1];
float g10 = acc_gate[mi][ni][2], g11 = acc_gate[mi][ni][3];
float u00 = acc_up[mi][ni][0], u01 = acc_up[mi][ni][1];
float u10 = acc_up[mi][ni][2], u11 = acc_up[mi][ni][3];
float h00 = (g00 / (1.0f + __expf(-g00))) * u00;
float h01 = (g01 / (1.0f + __expf(-g01))) * u01;
float h10 = (g10 / (1.0f + __expf(-g10))) * u10;
float h11 = (g11 / (1.0f + __expf(-g11))) * u11;
*reinterpret_cast<__nv_bfloat162*>(&smem_epi[r0 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h00, h01);
*reinterpret_cast<__nv_bfloat162*>(&smem_epi[r1 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h10, h11);
}
}
__syncthreads();
for (int idx = tid; idx < valid_rows * 8; idx += NUM_THREADS) {
int r = idx >> 3;
int c8 = (idx & 7) << 3;
if (n_base + c8 + 7 < I) {
uint4 v = *reinterpret_cast<const uint4*>(&smem_epi[r * EPI_STRIDE + c8]);
*reinterpret_cast<uint4*>(&h_interm[(int64_t)(row_offset + r) * I + n_base + c8]) = v;
} else {
for (int j = 0; j < 8 && n_base + c8 + j < I; ++j) {
h_interm[(int64_t)(row_offset + r) * I + n_base + c8 + j] = smem_epi[r * EPI_STRIDE + c8 + j];
}
}
}
}
// ============================================================================
// Routed Experts Grouped GEMM Stage 2 (N-Banded L2-Resident AtomicAdd!)
// ============================================================================
template <int BM, int STAGES, int NUM_WARPS, bool STREAM_B, int GROUP_N>
__global__ void __launch_bounds__(NUM_WARPS * 32) routed_down_stage2_kernel(
const __nv_bfloat16* __restrict__ h_interm, // (M_tot, I)
const __grid_constant__ CUtensorMap tmap_w2, // w2_routed: (E, H, I)
const TileInfo* __restrict__ tile_map,
const int32_t* __restrict__ d_num_tiles,
const int32_t* __restrict__ sorted_token_ids,
const float* __restrict__ sorted_weights,
float* __restrict__ out_fp32, // (T, H)
int H, int I, int n_tiles
) {
constexpr int BN = 128;
constexpr int BK = 64;
constexpr int NUM_THREADS = NUM_WARPS * 32;
constexpr int WARPS_M = NUM_WARPS / 2;
constexpr int WM = BM / WARPS_M;
constexpr int M_TILES = WM / 16;
const int total_m = *d_num_tiles;
const int job = blockIdx.x;
if (job >= total_m * n_tiles) return;
int m_tile, n_tile;
if constexpr (GROUP_N > 1) {
const int span = total_m * GROUP_N;
const int ng = job / span;
const int rem = job - ng * span;
const int g_eff = min(GROUP_N, n_tiles - ng * GROUP_N);
m_tile = rem / g_eff;
n_tile = ng * GROUP_N + (rem - m_tile * g_eff);
if (m_tile >= total_m) return;
} else {
n_tile = job / total_m;
m_tile = job - n_tile * total_m;
}
const int n_base = n_tile * BN;
const TileInfo ti = tile_map[m_tile];
const int expert_id = ti.expert_id;
const int row_offset = ti.row_offset;
const int valid_rows = ti.valid_rows;
constexpr uint32_t A_STAGE_BYTES = BM * 128;
constexpr uint32_t B_STAGE_BYTES = BN * 128;
constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES;
constexpr uint32_t EPI_BYTES = BM * 132 * sizeof(float);
constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u;
uint8_t* dyn_smem = g_dyn_smem;
uint64_t* mbar_B = reinterpret_cast<uint64_t*>(dyn_smem + MBAR_OFF);
const int tid = threadIdx.x;
const int warp_id = tid >> 5;
const int lane = tid & 31;
const int wm_idx = warp_id >> 1;
const int wn_idx = warp_id & 1;
uint32_t smem_base = static_cast<uint32_t>(__cvta_generic_to_shared(dyn_smem));
uint32_t sMbar[STAGES];
#pragma unroll
for (int s = 0; s < STAGES; ++s) {
sMbar[s] = static_cast<uint32_t>(__cvta_generic_to_shared(&mbar_B[s]));
}
if (tid == 0) {
#pragma unroll
for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1);
}
constexpr int A_CHUNKS_PER_THREAD = (BM * 8) / NUM_THREADS;
const __nv_bfloat16* my_a_ptr[A_CHUNKS_PER_THREAD];
int my_a_bytes[A_CHUNKS_PER_THREAD];
uint32_t my_a_smem_off[A_CHUNKS_PER_THREAD];
#pragma unroll
for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) {
int idx = tid + c * NUM_THREADS;
int r = idx >> 3;
int col8 = (idx & 7) << 3;
my_a_smem_off[c] = smem_swizzle_128b(0, r, col8);
if (r < valid_rows) {
my_a_ptr[c] = h_interm + (int64_t)(row_offset + r) * I + col8;
my_a_bytes[c] = 16;
} else {
my_a_ptr[c] = h_interm;
my_a_bytes[c] = 0;
}
}
__syncthreads();
const uint64_t pol_B = STREAM_B ? make_policy_evict_first() : 0ULL;
const int num_k_tiles = (I + BK - 1) / BK;
#pragma unroll
for (int kt = 0; kt < STAGES - 1; ++kt) {
if (kt < num_k_tiles) {
uint32_t s_off = smem_base + kt * STAGE_BYTES;
if (tid == 0) {
mbar_expect_tx(sMbar[kt], B_STAGE_BYTES);
if constexpr (STREAM_B) {
tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[kt], kt * BK, n_base, expert_id, pol_B);
} else {
tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[kt], kt * BK, n_base, expert_id);
}
}
#pragma unroll
for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) {
cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt * BK, my_a_bytes[c]);
}
}
asm volatile("cp.async.commit_group;\n");
}
float acc[M_TILES][8][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi)
#pragma unroll
for (int ni = 0; ni < 8; ++ni)
#pragma unroll
for (int r = 0; r < 4; ++r)
acc[mi][ni][r] = 0.0f;
const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3);
const int a_lane_c = (lane >> 4) << 3;
const int b_lane_r = (lane & 7) + ((lane >> 4) << 3);
const int b_lane_c = ((lane >> 3) & 1) << 3;
for (int kt = 0; kt < num_k_tiles; ++kt) {
int kt_next = kt + STAGES - 1;
if (kt_next < num_k_tiles) {
int p_next = kt_next % STAGES;
uint32_t s_off = smem_base + p_next * STAGE_BYTES;
if (tid == 0) {
mbar_expect_tx(sMbar[p_next], B_STAGE_BYTES);
if constexpr (STREAM_B) {
tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[p_next], kt_next * BK, n_base, expert_id, pol_B);
} else {
tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[p_next], kt_next * BK, n_base, expert_id);
}
}
#pragma unroll
for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) {
cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt_next * BK, my_a_bytes[c]);
}
}
asm volatile("cp.async.commit_group;\n");
asm volatile("cp.async.wait_group %0;\n" :: "n"(STAGES - 1));
int p_cur = kt % STAGES;
if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1);
__syncthreads();
uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES;
uint32_t sB_cur = sA_cur + A_STAGE_BYTES;
uint32_t a_reg[2][M_TILES][4];
uint32_t b_reg[2][4][4];
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
ldmatrix_x4(b_reg[0][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, b_lane_c));
}
#pragma unroll
for (int ks_idx = 0; ks_idx < 4; ++ks_idx) {
int cur_b = ks_idx & 1;
if (ks_idx < 3) {
int nxt_b = cur_b ^ 1;
int ks_nxt = (ks_idx + 1) * 16;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c));
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
ldmatrix_x4(b_reg[nxt_b][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, ks_nxt + b_lane_c));
}
}
#pragma unroll
for (int np = 0; np < 4; ++np) {
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
mma_m16n8k16_bf16(acc[mi][np * 2 + 0], a_reg[cur_b][mi], b_reg[cur_b][np][0], b_reg[cur_b][np][1]);
mma_m16n8k16_bf16(acc[mi][np * 2 + 1], a_reg[cur_b][mi], b_reg[cur_b][np][2], b_reg[cur_b][np][3]);
}
}
}
__syncthreads();
}
asm volatile("cp.async.wait_group 0;\n");
// Store tok/wt right inside the already-free SMEM above EPI_BYTES!
constexpr int EPI_STRIDE = 132;
float* smem_epi = reinterpret_cast<float*>(dyn_smem);
int32_t* smem_tok = reinterpret_cast<int32_t*>(dyn_smem + EPI_BYTES);
float* smem_wt = reinterpret_cast<float*>(dyn_smem + EPI_BYTES + BM * sizeof(int32_t));
if (tid < BM) {
if (tid < valid_rows) {
smem_tok[tid] = sorted_token_ids[row_offset + tid];
smem_wt[tid] = sorted_weights[row_offset + tid];
} else {
smem_tok[tid] = 0;
smem_wt[tid] = 0.0f;
}
}
__syncthreads();
const int gid = lane >> 2;
const int tid_g = lane & 3;
#pragma unroll
for (int mi = 0; mi < M_TILES; ++mi) {
int r0 = wm_idx * WM + mi * 16 + gid;
int r1 = r0 + 8;
float wt0 = smem_wt[r0];
float wt1 = smem_wt[r1];
#pragma unroll
for (int ni = 0; ni < 8; ++ni) {
int c = wn_idx * 64 + ni * 8 + tid_g * 2;
*reinterpret_cast<float2*>(&smem_epi[r0 * EPI_STRIDE + c]) =
make_float2(acc[mi][ni][0] * wt0, acc[mi][ni][1] * wt0);
*reinterpret_cast<float2*>(&smem_epi[r1 * EPI_STRIDE + c]) =
make_float2(acc[mi][ni][2] * wt1, acc[mi][ni][3] * wt1);
}
}
__syncthreads();
for (int m = warp_id; m < valid_rows; m += NUM_WARPS) {
int32_t tok = smem_tok[m];
float* dst_row = out_fp32 + (int64_t)tok * H + n_base;
#pragma unroll
for (int c = lane * 4; c < BN; c += 128) {
if (n_base + c + 3 < H) {
float4 v = *reinterpret_cast<const float4*>(&smem_epi[m * EPI_STRIDE + c]);
red_add_v4_f32(dst_row + c, v.x, v.y, v.z, v.w);
} else {
for (int j = 0; j < 4 && n_base + c + j < H; ++j) {
atomicAdd(dst_row + c + j, smem_epi[m * EPI_STRIDE + c + j]);
}
}
}
}
}
// ============================================================================
// Vectorized FP32 -> BF16 Final Cast Kernel
// ============================================================================
__global__ void __launch_bounds__(256) cast_fp32_to_bf16_kernel(
const float* __restrict__ src,
__nv_bfloat16* __restrict__ dst,
int64_t n_elems
) {
int64_t idx8 = ((int64_t)blockIdx.x * 256 + threadIdx.x) << 3;
if (idx8 + 7 < n_elems) {
float4 v0 = *reinterpret_cast<const float4*>(src + idx8);
float4 v1 = *reinterpret_cast<const float4*>(src + idx8 + 4);
__nv_bfloat162 b0 = __floats2bfloat162_rn(v0.x, v0.y);
__nv_bfloat162 b1 = __floats2bfloat162_rn(v0.z, v0.w);
__nv_bfloat162 b2 = __floats2bfloat162_rn(v1.x, v1.y);
__nv_bfloat162 b3 = __floats2bfloat162_rn(v1.z, v1.w);
uint4 out_u4;
out_u4.x = *reinterpret_cast<uint32_t*>(&b0);
out_u4.y = *reinterpret_cast<uint32_t*>(&b1);
out_u4.z = *reinterpret_cast<uint32_t*>(&b2);
out_u4.w = *reinterpret_cast<uint32_t*>(&b3);
*reinterpret_cast<uint4*>(dst + idx8) = out_u4;
} else {
for (int64_t i = idx8; i < n_elems; ++i) {
dst[i] = __float2bfloat16_rn(src[i]);
}
}
}
// ============================================================================
// Host TMA Descriptor Builders & Kernel Launchers
// ============================================================================
inline CUtensorMap make_tmap_2d_bf16(
const void* ptr, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols
) {
CUtensorMap tmap{};
cuuint64_t globalDim[2] = {cols, rows};
cuuint64_t globalStrides[1] = {cols * sizeof(__nv_bfloat16)};
cuuint32_t boxDim[2] = {box_cols, box_rows};
cuuint32_t elemStrides[2] = {1, 1};
CUresult res = cuTensorMapEncodeTiled(
&tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 2, const_cast<void*>(ptr),
globalDim, globalStrides, boxDim, elemStrides,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
);
TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 2D failed: ", (int)res);
return tmap;
}
inline CUtensorMap make_tmap_3d_bf16(
const void* ptr, uint64_t batches, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols
) {
CUtensorMap tmap{};
cuuint64_t globalDim[3] = {cols, rows, batches};
cuuint64_t globalStrides[2] = {cols * sizeof(__nv_bfloat16), rows * cols * sizeof(__nv_bfloat16)};
cuuint32_t boxDim[3] = {box_cols, box_rows, 1};
cuuint32_t elemStrides[3] = {1, 1, 1};
CUresult res = cuTensorMapEncodeTiled(
&tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3, const_cast<void*>(ptr),
globalDim, globalStrides, boxDim, elemStrides,
CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B,
CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
);
TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 3D failed: ", (int)res);
return tmap;
}
template <typename KernelT>
inline void ensure_dyn_smem(KernelT kernel, int smem_bytes) {
if (smem_bytes > 48 * 1024) {
cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes);
}
}
template <int BM, int STAGES, int NUM_WARPS, bool STREAM_B, int GROUP_N_S1, int GROUP_N_S2>
void launch_routed_pipeline(
const __nv_bfloat16* x_ptr,
const __nv_bfloat16* w1_ptr,
const __nv_bfloat16* w2_ptr,
const TileInfo* tile_map_ptr,
const int32_t* d_num_tiles_ptr,
const int32_t* sorted_tok_ptr,
const float* sorted_wt_ptr,
__nv_bfloat16* h_routed_ptr,
float* out_fp32_ptr,
int E, int H, int I, int max_m_tiles,
cudaStream_t stream
) {
CUtensorMap tmap_w1 = make_tmap_3d_bf16(w1_ptr, E, 2 * I, H, 64, 64);
CUtensorMap tmap_w2 = make_tmap_3d_bf16(w2_ptr, E, H, I, 128, 64);
constexpr int s1_stage_bytes = STAGES * (BM * 128 + 2 * 64 * 128);
constexpr int s1_epi_bytes = BM * 72 * sizeof(__nv_bfloat16);
constexpr int s1_main = (((s1_stage_bytes > s1_epi_bytes) ? s1_stage_bytes : s1_epi_bytes) + 127) & ~127;
constexpr int s1_smem = s1_main + 64;
constexpr int s2_stage_bytes = STAGES * (BM * 128 + 128 * 128);
constexpr int s2_epi_bytes = BM * 132 * sizeof(float) + BM * 8;
constexpr int s2_main = (((s2_stage_bytes > s2_epi_bytes) ? s2_stage_bytes : s2_epi_bytes) + 127) & ~127;
constexpr int s2_smem = s2_main + 64;
auto k1 = routed_swiglu_stage1_kernel<BM, STAGES, NUM_WARPS, STREAM_B, GROUP_N_S1>;
auto k2 = routed_down_stage2_kernel<BM, STAGES, NUM_WARPS, STREAM_B, GROUP_N_S2>;
static bool configured = false;
if (!configured) {
ensure_dyn_smem(k1, s1_smem);
ensure_dyn_smem(k2, s2_smem);
configured = true;
}
int n_tiles_s1 = (I + 63) / 64;
k1<<<max_m_tiles * n_tiles_s1, NUM_WARPS * 32, s1_smem, stream>>>(
x_ptr, tmap_w1, tile_map_ptr, d_num_tiles_ptr, sorted_tok_ptr, h_routed_ptr, H, I, n_tiles_s1
);
int n_tiles_s2 = (H + 127) / 128;
k2<<<max_m_tiles * n_tiles_s2, NUM_WARPS * 32, s2_smem, stream>>>(
h_routed_ptr, tmap_w2, tile_map_ptr, d_num_tiles_ptr, sorted_tok_ptr, sorted_wt_ptr, out_fp32_ptr, H, I, n_tiles_s2
);
}
} // namespace
torch::Tensor glm52_fused_moe_forward(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared
) {
if (__builtin_expect(!x.is_contiguous(), 0)) x = x.contiguous();
if (__builtin_expect(!expert_ids.is_contiguous(), 0)) expert_ids = expert_ids.contiguous();
if (__builtin_expect(!expert_weights.is_contiguous(), 0)) expert_weights = expert_weights.contiguous();
const int T = static_cast<int>(x.size(0));
const int H = static_cast<int>(x.size(1));
const int top_k = static_cast<int>(expert_ids.size(1));
const int E = static_cast<int>(w1_routed.size(0));
const int I = static_cast<int>(w2_routed.size(2));
const int n_shared = static_cast<int>(w1_shared.size(0));
auto out = torch::empty({T, H}, x.options());
if (__builtin_expect(T == 0, 0)) return out;
cudaStream_t stream = at::cuda::getCurrentCUDAStream();
// Wave-Balanced Persistent Lockless Decode Path (T <= 2)
if (T <= 2 && (H & 7) == 0 && (I & 7) == 0 && (top_k + n_shared) > 0) {
const int slots_per_tok = top_k + n_shared;
const int total_pairs = T * slots_per_tok;
g_ws.ensure(total_pairs, T, E, H, I, 32, 1, x.device());
const auto* x_ptr = reinterpret_cast<const __nv_bfloat16*>(x.data_ptr());
const auto* ids_ptr = expert_ids.data_ptr<int64_t>();
const auto* wts_ptr = reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr());
const auto* w1r_ptr = reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr());
const auto* w2r_ptr = reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr());
const auto* w1s_ptr = n_shared > 0 ? reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr()) : nullptr;
const auto* w2s_ptr = n_shared > 0 ? reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr()) : nullptr;
auto* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr());
// 940 CTAs = 5 * 188 SMs (exact uniform wave balance across all 188 SMs!)
decode_stage1_persistent_kernel<<<940, 128, 0, stream>>>(
x_ptr, ids_ptr, w1r_ptr, w1s_ptr, g_ws.h_routed_ptr, g_ws.out_fp32_ptr,
T, top_k, n_shared, H, I
);
decode_stage2_persistent_kernel<<<940, 64, 0, stream>>>(
g_ws.h_routed_ptr, ids_ptr, wts_ptr, w2r_ptr, w2s_ptr, g_ws.out_fp32_ptr,
T, top_k, n_shared, H, I
);
int64_t total_elems = (int64_t)T * H;
int cast_blocks = static_cast<int>((total_elems + 2047) / 2048);
cast_fp32_to_bf16_kernel<<<cast_blocks, 256, 0, stream>>>(g_ws.out_fp32_ptr, out_bf16_ptr, total_elems);
return out;
}
// General Tensor Core TMA + MMA Path (all T)
const int M_tot = T * top_k;
const int avg_m = M_tot / std::max(E, 1);
const int BM_routed = (avg_m <= 20) ? 32 : ((avg_m <= 56) ? 64 : 128);
const int max_m_tiles = (M_tot + BM_routed - 1) / BM_routed + E;
const int sort_blocks = std::min(64, std::max(1, (M_tot + 255) / 256));
g_ws.ensure(M_tot, T, E, H, I, max_m_tiles, sort_blocks, x.device());
const auto* x_ptr = reinterpret_cast<const __nv_bfloat16*>(x.data_ptr());
const auto* ids_ptr = expert_ids.data_ptr<int64_t>();
const auto* wts_ptr = reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr());
const auto* w1r_ptr = reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr());
const auto* w2r_ptr = reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr());
auto* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr());
// 1. Sort routed tokens & construct tile map
if (M_tot > 0) {
if (M_tot <= 8192 && E <= 512) {
moe_fused_single_sort_kernel<<<1, 512, 0, stream>>>(
ids_ptr, wts_ptr, g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr,
g_ws.tile_map_ptr, g_ws.num_tiles_ptr, M_tot, top_k, E, BM_routed
);
} else {
int hist_smem = E * sizeof(int32_t);
moe_hist_kernel<<<sort_blocks, 256, hist_smem, stream>>>(ids_ptr, g_ws.block_hist_ptr, M_tot, E);
int scan_smem = 3 * (E + 1) * sizeof(int32_t);
moe_scan_tiles_kernel<<<1, 256, scan_smem, stream>>>(
g_ws.block_hist_ptr, g_ws.expert_off_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr,
sort_blocks, E, BM_routed
);
moe_scatter_kernel<<<sort_blocks, 256, hist_smem, stream>>>(
ids_ptr, wts_ptr, g_ws.block_hist_ptr, g_ws.expert_off_ptr,
g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr, M_tot, top_k, E
);
}
}
// 2. Shared experts (writes out_fp32 directly for s=0, eliminating memset!)
if (n_shared == 0) {
cudaMemsetAsync(g_ws.out_fp32_ptr, 0, (size_t)T * H * sizeof(float), stream);
} else {
const auto* w1s_all = reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr());
const auto* w2s_all = reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr());
constexpr int BM_S = 128;
constexpr int ST_S = 3;
constexpr int NW_S = 8;
constexpr int smem_ds1 = ST_S * (BM_S * 128 + 2 * 64 * 128) + 128;
constexpr int smem_ds2 = ST_S * (BM_S * 128 + 128 * 128) + 128;
static bool shared_cfg = false;
if (!shared_cfg) {
ensure_dyn_smem(dense_swiglu_stage1_kernel<BM_S, ST_S, NW_S>, smem_ds1);
ensure_dyn_smem(dense_down_stage2_kernel<BM_S, ST_S, NW_S, false>, smem_ds2);
ensure_dyn_smem(dense_down_stage2_kernel<BM_S, ST_S, NW_S, true>, smem_ds2);
shared_cfg = true;
}
CUtensorMap tmap_x = make_tmap_2d_bf16(x_ptr, T, H, BM_S, 64);
CUtensorMap tmap_hs = make_tmap_2d_bf16(g_ws.h_shared_ptr, T, I, BM_S, 64);
// blockIdx.x = M-tile (varies fastest -> w1_shared/w2_shared N-slice stays hot in L2 across all M-tiles!)
dim3 grid_ds1((T + BM_S - 1) / BM_S, (I + 63) / 64);
dim3 grid_ds2((T + BM_S - 1) / BM_S, (H + 127) / 128);
for (int s = 0; s < n_shared; ++s) {
const __nv_bfloat16* w1s = w1s_all + (int64_t)s * (2LL * I * H);
const __nv_bfloat16* w2s = w2s_all + (int64_t)s * ((int64_t)H * I);
CUtensorMap tmap_w1s = make_tmap_2d_bf16(w1s, 2 * I, H, 64, 64);
CUtensorMap tmap_w2s = make_tmap_2d_bf16(w2s, H, I, 128, 64);
dense_swiglu_stage1_kernel<BM_S, ST_S, NW_S><<<grid_ds1, NW_S * 32, smem_ds1, stream>>>(
tmap_x, tmap_w1s, g_ws.h_shared_ptr, T, H, I
);
if (s == 0) {
dense_down_stage2_kernel<BM_S, ST_S, NW_S, false><<<grid_ds2, NW_S * 32, smem_ds2, stream>>>(
tmap_hs, tmap_w2s, g_ws.out_fp32_ptr, T, H, I
);
} else {
dense_down_stage2_kernel<BM_S, ST_S, NW_S, true><<<grid_ds2, NW_S * 32, smem_ds2, stream>>>(
tmap_hs, tmap_w2s, g_ws.out_fp32_ptr, T, H, I
);
}
}
}
// 3. Routed experts Grouped GEMM (Stage 1 + Stage 2)
if (M_tot > 0) {
if (BM_routed == 32) {
// Single M-tile per expert -> STREAM_B=true protects x & out_fp32 in L2!
launch_routed_pipeline<32, 2, 4, true, 4, 8>(
x_ptr, w1r_ptr, w2r_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr,
g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr,
g_ws.h_routed_ptr, g_ws.out_fp32_ptr, E, H, I, max_m_tiles, stream
);
} else if (BM_routed == 64) {
launch_routed_pipeline<64, 2, 4, true, 4, 8>(
x_ptr, w1r_ptr, w2r_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr,
g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr,
g_ws.h_routed_ptr, g_ws.out_fp32_ptr, E, H, I, max_m_tiles, stream
);
} else {
// Multi M-tile per expert -> STREAM_B=false keeps weights in L2 across adjacent M-tiles!
// GROUP_N_S1=2 maximizes M-tile weight L2 reuse; GROUP_N_S2=8 keeps out_fp32 band (<=33.5 MB) in L2!
launch_routed_pipeline<128, 3, 8, false, 2, 8>(
x_ptr, w1r_ptr, w2r_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr,
g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr,
g_ws.h_routed_ptr, g_ws.out_fp32_ptr, E, H, I, max_m_tiles, stream
);
}
}
// 4. Cast out_fp32 -> out_bf16
int64_t total_elems = (int64_t)T * H;
int cast_blocks = static_cast<int>((total_elems + 2047) / 2048);
cast_fp32_to_bf16_kernel<<<cast_blocks, 256, 0, stream>>>(g_ws.out_fp32_ptr, out_bf16_ptr, total_elems);
return out;
}
"""
_CPP_SRC = r"""
#include <torch/extension.h>
torch::Tensor glm52_fused_moe_forward(
torch::Tensor x,
torch::Tensor expert_ids,
torch::Tensor expert_weights,
torch::Tensor w1_routed,
torch::Tensor w2_routed,
torch::Tensor w1_shared,
torch::Tensor w2_shared
);
"""
_EXT = load_inline(
name="glm52_fused_moe_sm120_ext_v4",
cpp_sources=_CPP_SRC,
cuda_sources=_CUDA_SRC,
functions=["glm52_fused_moe_forward"],
extra_cuda_cflags=["-O3", "-arch=sm_120a"],
extra_ldflags=["-lcuda"],
verbose=False,
)
class Model(nn.Module):
def __init__(
self,
T: int,
E: int,
top_k: int,
n_shared: int,
H: int,
I: int,
):
super().__init__()
self.T, self.E, self.top_k = T, E, top_k
self.n_shared, self.H, self.I = n_shared, H, I
self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
for p in self.parameters():
nn.init.normal_(p, std=0.02)
def forward(
self,
x: torch.Tensor,
expert_ids: torch.Tensor,
expert_weights: torch.Tensor,
) -> torch.Tensor:
return _EXT.glm52_fused_moe_forward(
x,
expert_ids,
expert_weights,
self.w1_routed,
self.w2_routed,
self.w1_shared,
self.w2_shared,
)
20261110_113646_agy_gemini-4-carbon-high_01_glm52_fused_moe