"""Fused GLM-5.2 MoE layer in CUDA C++/PTX for RTX PRO 6000 (SM120 Blackwell). Architecture: - Small-batch decode path (T <= 4): 2-kernel fused GEMV covering all active routed + shared experts in one shot: 1) moe_decode_gemv1_silu_mul: vectorized 128-bit streaming gate+up+SiLU*mul -> h_buf (fp32) 2) moe_decode_gemv2_reduce: vectorized 128-bit down-proj + weighted sum -> out (bf16) - Grouped prefill/batch path (T > 4): 0) Zero-sync GPU routing histogram + prefix sum + scatter + M-tile lookup table 1) Fused Grouped GEMM1 (SM120 PTX mma.sync.aligned.m16n8k16 + cp.async.cg + 128B XOR swizzle + ldmatrix.x4): gathers x[token_id] on-the-fly from L2, computes gate & up simultaneously, applies SiLU(gate)*up in FP32 registers, writes packed h (bf16) via SMEM coalescing. 2) Fused Grouped GEMM2: computes h @ w2^T, scales by routing weight in FP32, and scatters directly into out_fp32 via 128-bit vectorized red.relaxed.gpu.global.add.v4.f32. 3) Fused Shared Expert GEMM1 + GEMM2: computes shared expert(s), fusing the final addition with out_fp32 and FP32->BF16 conversion into out (bf16). """ from __future__ import annotations import os import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline _CUDA_SRC = r""" #include #include #include #include #include #include #include // ============================================================================ // PTX & Low-Level Device Primitives (SM120 Blackwell) // ============================================================================ __device__ __forceinline__ uint32_t smem_u32(const void* ptr) { return static_cast(__cvta_generic_to_shared(ptr)); } __device__ __forceinline__ void cp_async_cg_16(uint32_t dst_smem, const void* src_gmem) { asm volatile( "cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\n" :: "r"(dst_smem), "l"(src_gmem) ); } __device__ __forceinline__ void cp_async_cg_16_zfill(uint32_t dst_smem, const void* src_gmem, int src_bytes) { asm volatile( "cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n" :: "r"(dst_smem), "l"(src_gmem), "r"(src_bytes) ); } __device__ __forceinline__ void cp_async_commit() { asm volatile("cp.async.commit_group;\n" ::: "memory"); } template __device__ __forceinline__ void cp_async_wait() { asm volatile("cp.async.wait_group %0;\n" :: "n"(N) : "memory"); } __device__ __forceinline__ void ldmatrix_x4( uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t smem_addr ) { asm volatile( "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(smem_addr) ); } __device__ __forceinline__ void mma_m16n8k16_bf16( float& d0, float& d1, float& d2, float& d3, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1 ) { asm volatile( "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n" : "+f"(d0), "+f"(d1), "+f"(d2), "+f"(d3) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1) ); } __device__ __forceinline__ void red_global_add_v4_f32(float* addr, float4 v) { asm volatile( "red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n" :: "l"(addr), "f"(v.x), "f"(v.y), "f"(v.z), "f"(v.w) ); } __device__ __forceinline__ float silu_f32(float x) { return x / (1.0f + expf(-x)); } __device__ __forceinline__ float warp_reduce_sum(float val) { #pragma unroll for (int offset = 16; offset > 0; offset >>= 1) { val += __shfl_down_sync(0xffffffff, val, offset); } return val; } __device__ __forceinline__ float dot8_bf16_f32(uint4 a, uint4 b) { const __nv_bfloat162* a2 = reinterpret_cast(&a); const __nv_bfloat162* b2 = reinterpret_cast(&b); float acc = 0.0f; #pragma unroll for (int i = 0; i < 4; ++i) { float2 fa = __bfloat1622float2(a2[i]); float2 fb = __bfloat1622float2(b2[i]); acc = fmaf(fa.x, fb.x, acc); acc = fmaf(fa.y, fb.y, acc); } return acc; } // ============================================================================ // REGIME A: Ultra-Fast Fused Decode GEMV (T <= 4, e.g. T = 1) // ============================================================================ // Kernel D1: Computes h_buf[t, slot, col] = silu(dot(x[t], gate)) * dot(x[t], up) // Grid: x = I / WARPS_PER_BLOCK, y = total_slots (top_k + n_shared), z = T // Block: WARPS_PER_BLOCK * 32 threads (default 4 warps = 128 threads) template __global__ void moe_decode_gemv1_silu_mul_kernel( const __nv_bfloat16* __restrict__ x, // (T, H) const int64_t* __restrict__ expert_ids, // (T, top_k) const __nv_bfloat16* __restrict__ w1_routed, // (E, 2*I, H) const __nv_bfloat16* __restrict__ w1_shared, // (n_shared, 2*I, H) __nv_bfloat16* __restrict__ h_buf, // (T, top_k + n_shared, I) int T, int E, int top_k, int n_shared, int H, int I ) { extern __shared__ __nv_bfloat16 s_x[]; // H bf16 (8 KB for H=4096) const int t = blockIdx.z; const int slot = blockIdx.y; const int warp_id = threadIdx.x >> 5; const int lane = threadIdx.x & 31; const int col = blockIdx.x * WARPS_PER_BLOCK + warp_id; // Cooperatively load x[t, 0..H-1] into SMEM using 128-bit loads const int H_u4 = H >> 3; const uint4* x_u4 = reinterpret_cast(x + (size_t)t * H); uint4* sx_u4 = reinterpret_cast(s_x); for (int i = threadIdx.x; i < H_u4; i += WARPS_PER_BLOCK * 32) { sx_u4[i] = x_u4[i]; } __syncthreads(); if (col >= I) return; const __nv_bfloat16* w1_base; if (slot < top_k) { int64_t e = expert_ids[(size_t)t * top_k + slot]; w1_base = w1_routed + (size_t)e * (size_t)(2 * I) * (size_t)H; } else { int s = slot - top_k; w1_base = w1_shared + (size_t)s * (size_t)(2 * I) * (size_t)H; } const uint4* wg_u4 = reinterpret_cast(w1_base + (size_t)col * H); const uint4* wu_u4 = reinterpret_cast(w1_base + (size_t)(I + col) * H); float acc_g0 = 0.0f, acc_g1 = 0.0f; float acc_u0 = 0.0f, acc_u1 = 0.0f; // 2x unrolled streaming 128-bit loads along H int i = lane; for (; i + 32 < H_u4; i += 64) { uint4 x0 = sx_u4[i]; uint4 x1 = sx_u4[i + 32]; uint4 g0 = __ldcs(&wg_u4[i]); uint4 u0 = __ldcs(&wu_u4[i]); uint4 g1 = __ldcs(&wg_u4[i + 32]); uint4 u1 = __ldcs(&wu_u4[i + 32]); acc_g0 += dot8_bf16_f32(x0, g0); acc_u0 += dot8_bf16_f32(x0, u0); acc_g1 += dot8_bf16_f32(x1, g1); acc_u1 += dot8_bf16_f32(x1, u1); } for (; i < H_u4; i += 32) { uint4 x0 = sx_u4[i]; uint4 g0 = __ldcs(&wg_u4[i]); uint4 u0 = __ldcs(&wu_u4[i]); acc_g0 += dot8_bf16_f32(x0, g0); acc_u0 += dot8_bf16_f32(x0, u0); } float g_sum = warp_reduce_sum(acc_g0 + acc_g1); float u_sum = warp_reduce_sum(acc_u0 + acc_u1); if (lane == 0) { int total_slots = top_k + n_shared; float h = silu_f32(g_sum) * u_sum; h_buf[((size_t)t * total_slots + slot) * I + col] = __float2bfloat16(h); } } // Kernel D2: Computes out[t, col] = sum_slot w_slot * dot(h_buf[t, slot, :], w2_slot[col, :]) // Grid: x = H / WARPS_PER_BLOCK, z = T // Block: WARPS_PER_BLOCK * 32 threads (2 warps = 64 threads -> 2048 CTAs for H=4096 -> 10.9 waves on 188 SMs!) template __global__ void moe_decode_gemv2_reduce_kernel( const __nv_bfloat16* __restrict__ h_buf, // (T, total_slots, I) const int64_t* __restrict__ expert_ids, // (T, top_k) const __nv_bfloat16* __restrict__ expert_weights, // (T, top_k) const __nv_bfloat16* __restrict__ w2_routed, // (E, H, I) const __nv_bfloat16* __restrict__ w2_shared, // (n_shared, H, I) __nv_bfloat16* __restrict__ out, // (T, H) int T, int E, int top_k, int n_shared, int H, int I ) { const int t = blockIdx.z; const int warp_id = threadIdx.x >> 5; const int lane = threadIdx.x & 31; const int col = blockIdx.x * WARPS_PER_BLOCK + warp_id; if (col >= H) return; const int total_slots = top_k + n_shared; const int I_u4 = I >> 3; float total_acc = 0.0f; for (int slot = 0; slot < total_slots; ++slot) { const __nv_bfloat16* w2_row; float weight; if (slot < top_k) { int64_t e = expert_ids[(size_t)t * top_k + slot]; weight = __bfloat162float(expert_weights[(size_t)t * top_k + slot]); w2_row = w2_routed + ((size_t)e * (size_t)H + (size_t)col) * (size_t)I; } else { int s = slot - top_k; weight = 1.0f; w2_row = w2_shared + ((size_t)s * (size_t)H + (size_t)col) * (size_t)I; } const uint4* h_u4 = reinterpret_cast( h_buf + ((size_t)t * total_slots + slot) * I ); const uint4* w_u4 = reinterpret_cast(w2_row); float acc0 = 0.0f, acc1 = 0.0f; int i = lane; #pragma unroll 4 for (; i + 32 < I_u4; i += 64) { uint4 hv0 = __ldg(&h_u4[i]); uint4 wv0 = __ldcs(&w_u4[i]); uint4 hv1 = __ldg(&h_u4[i + 32]); uint4 wv1 = __ldcs(&w_u4[i + 32]); acc0 += dot8_bf16_f32(hv0, wv0); acc1 += dot8_bf16_f32(hv1, wv1); } for (; i < I_u4; i += 32) { uint4 hv0 = __ldg(&h_u4[i]); uint4 wv0 = __ldcs(&w_u4[i]); acc0 += dot8_bf16_f32(hv0, wv0); } total_acc += weight * (acc0 + acc1); } float final_sum = warp_reduce_sum(total_acc); if (lane == 0) { out[(size_t)t * H + col] = __float2bfloat16(final_sum); } } // ============================================================================ // REGIME B: Zero-Sync Routing Sort + M-Tile Table Construction // ============================================================================ __global__ void moe_count_experts_kernel( const int64_t* __restrict__ expert_ids, int32_t* __restrict__ expert_counts, int M_total, int E ) { extern __shared__ int32_t s_counts[]; for (int i = threadIdx.x; i < E; i += blockDim.x) { s_counts[i] = 0; } __syncthreads(); int idx = blockIdx.x * blockDim.x + threadIdx.x; int stride = gridDim.x * blockDim.x; for (; idx < M_total; idx += stride) { int e = static_cast(expert_ids[idx]); atomicAdd(&s_counts[e], 1); } __syncthreads(); for (int i = threadIdx.x; i < E; i += blockDim.x) { int c = s_counts[i]; if (c > 0) { atomicAdd(&expert_counts[i], c); } } } // Single-block kernel (up to E <= 1024) computing prefix sums & tile table __global__ void moe_build_offsets_and_tiles_kernel( const int32_t* __restrict__ expert_counts, int32_t* __restrict__ expert_offsets, // (E + 1) int32_t* __restrict__ write_cursors, // (E) int32_t* __restrict__ expert_tile_prefix, // (E + 1) int32_t* __restrict__ m_tile_to_expert, // (max_m_tiles) int E, int BM ) { // Simple warp/thread scan (E=256 takes < 2 microseconds) if (threadIdx.x == 0) { int off = 0; int tile_off = 0; for (int e = 0; e < E; ++e) { int c = expert_counts[e]; expert_offsets[e] = off; write_cursors[e] = off; expert_tile_prefix[e] = tile_off; int mt = (c + BM - 1) / BM; for (int m = 0; m < mt; ++m) { m_tile_to_expert[tile_off + m] = e; } off += c; tile_off += mt; } expert_offsets[E] = off; expert_tile_prefix[E] = tile_off; } } __global__ void moe_scatter_tokens_kernel( const int64_t* __restrict__ expert_ids, const __nv_bfloat16* __restrict__ expert_weights, int32_t* __restrict__ write_cursors, int32_t* __restrict__ sorted_token_ids, float* __restrict__ sorted_weights, int M_total, int top_k ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int stride = gridDim.x * blockDim.x; for (; idx < M_total; idx += stride) { int e = static_cast(expert_ids[idx]); int pos = atomicAdd(&write_cursors[e], 1); sorted_token_ids[pos] = idx / top_k; sorted_weights[pos] = __bfloat162float(expert_weights[idx]); } } // ============================================================================ // REGIME B: Fused Grouped GEMM1 (Gather X + Gate + Up + SiLU*Mul -> h) // ============================================================================ // Tile: BM x BN_OUT (BN_OUT = 64 -> 64 gate + 64 up = 128 weight rows), BK = 64 // 8 warps (256 threads): // - BM = 32: WARPS_M = 2 (WM = 16, 1 tile), WARPS_N = 4 (WN = 16, 1 tile of 16) // - BM = 64: WARPS_M = 2 (WM = 32, 2 tiles), WARPS_N = 4 (WN = 16, 1 tile of 16) // - BM = 128: WARPS_M = 4 (WM = 32, 2 tiles), WARPS_N = 2 (WN = 32, 2 tiles of 16) template __global__ __launch_bounds__(WARPS_M * WARPS_N * 32, 1) void moe_gemm1_silu_mul_kernel( const __nv_bfloat16* __restrict__ x, // (T, H) const __nv_bfloat16* __restrict__ w1, // (E or n_shared, 2*I, H) __nv_bfloat16* __restrict__ h_out, // (M_total or T, I) const int32_t* __restrict__ expert_offsets, // (E + 1) [routed] const int32_t* __restrict__ expert_tile_prefix, // (E + 1) [routed] const int32_t* __restrict__ m_tile_to_expert, // (max_m_tiles) [routed] const int32_t* __restrict__ sorted_token_ids, // (M_total) [routed] int T, int E, int H, int I, int shared_idx ) { static_assert(BK == 64, "BK must be 64 for 128B swizzle"); static_assert(BN_OUT == 64, "BN_OUT must be 64"); constexpr int NWARPS = WARPS_M * WARPS_N; constexpr int NTHREADS = NWARPS * 32; constexpr int WM = BM / WARPS_M; constexpr int WN = BN_OUT / WARPS_N; constexpr int M_TILES = WM / 16; constexpr int N_TILES = WN / 16; // each N_TILE is 16 cols = two m16n8k16 MMAs const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int warp_m = warp_id / WARPS_N; const int warp_n = warp_id % WARPS_N; const int n_tiles_total = I / BN_OUT; // 32 for I=2048 int e, row_base, m_valid, n_blk; if constexpr (IS_SHARED) { const int work_id = blockIdx.x; const int m_tiles_shared = (T + BM - 1) / BM; if (work_id >= m_tiles_shared * n_tiles_total) return; e = shared_idx; n_blk = work_id / m_tiles_shared; int m_blk = work_id - n_blk * m_tiles_shared; row_base = m_blk * BM; m_valid = min(BM, T - row_base); } else { const int work_id = blockIdx.x; const int total_m_tiles = expert_tile_prefix[E]; if (work_id >= total_m_tiles * n_tiles_total) return; int m_tile_approx = work_id / n_tiles_total; e = m_tile_to_expert[m_tile_approx]; int p0 = expert_tile_prefix[e]; int p1 = expert_tile_prefix[e + 1]; int m_tiles_e = p1 - p0; int local = work_id - p0 * n_tiles_total; n_blk = local / m_tiles_e; int m_blk = local - n_blk * m_tiles_e; int off0 = expert_offsets[e]; int off1 = expert_offsets[e + 1]; row_base = off0 + m_blk * BM; m_valid = min(BM, off1 - row_base); } const int col_base = n_blk * BN_OUT; // Shared memory layout: // int32 s_tok[BM] (at the end or separate? Put s_tok in small static smem!) // Dynamic SMEM (16B aligned): // For each stage s in [0, STAGES): // A[s]: BM * 64 bf16 = BM * 128 B // B_gate[s]: BN_OUT * 64 bf16 = 8192 B // B_up[s]: BN_OUT * 64 bf16 = 8192 B // Epilogue reuses dynamic SMEM: s_h[BM][BN_OUT + 8] bf16 __shared__ int32_t s_tok[BM]; if constexpr (!IS_SHARED) { if (tid < BM) { s_tok[tid] = (tid < m_valid) ? sorted_token_ids[row_base + tid] : 0; } __syncthreads(); } extern __shared__ char smem_raw[]; constexpr uint32_t A_STAGE_BYTES = BM * 128; constexpr uint32_t B_STAGE_BYTES = BN_OUT * 128; // 8192 B constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_STAGE_BYTES; const uint32_t smem_base = smem_u32(smem_raw); const __nv_bfloat16* w1_e = w1 + (size_t)e * (size_t)(2 * I) * (size_t)H; const __nv_bfloat16* wg_base = w1_e + (size_t)col_base * (size_t)H; const __nv_bfloat16* wu_base = w1_e + (size_t)(I + col_base) * (size_t)H; // Accumulators in FP32 float acc_g[M_TILES][N_TILES * 2][4]; float acc_u[M_TILES][N_TILES * 2][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { #pragma unroll for (int n8 = 0; n8 < N_TILES * 2; ++n8) { acc_g[mi][n8][0] = 0.0f; acc_g[mi][n8][1] = 0.0f; acc_g[mi][n8][2] = 0.0f; acc_g[mi][n8][3] = 0.0f; acc_u[mi][n8][0] = 0.0f; acc_u[mi][n8][1] = 0.0f; acc_u[mi][n8][2] = 0.0f; acc_u[mi][n8][3] = 0.0f; } } // Helper lambda to issue async loads for a K-tile `kt` into `stage` auto load_stage = [&](int kt, int stage) { const int k_off = kt * BK; const uint32_t st_base = smem_base + stage * STAGE_BYTES; const uint32_t a_base = st_base; const uint32_t bg_base = st_base + A_STAGE_BYTES; const uint32_t bu_base = bg_base + B_STAGE_BYTES; // Load A: BM * 8 chunks of 16B constexpr int A_CHUNKS = BM * 8; #pragma unroll for (int idx = tid; idx < A_CHUNKS; idx += NTHREADS) { int r = idx >> 3; int c8 = idx & 7; int swiz = c8 ^ (r & 7); uint32_t dst = a_base + (r * 8 + swiz) * 16; int valid_bytes = (r < m_valid) ? 16 : 0; const __nv_bfloat16* src; if constexpr (IS_SHARED) { int r_clamped = (r < m_valid) ? (row_base + r) : row_base; src = x + (size_t)r_clamped * H + k_off + c8 * 8; } else { int tok = s_tok[r]; src = x + (size_t)tok * H + k_off + c8 * 8; } cp_async_cg_16_zfill(dst, src, valid_bytes); } // Load B_gate & B_up: BN_OUT * 8 = 512 chunks each constexpr int B_CHUNKS = BN_OUT * 8; #pragma unroll for (int idx = tid; idx < B_CHUNKS; idx += NTHREADS) { int r = idx >> 3; int c8 = idx & 7; int swiz = c8 ^ (r & 7); uint32_t off = (r * 8 + swiz) * 16; cp_async_cg_16(bg_base + off, wg_base + (size_t)r * H + k_off + c8 * 8); cp_async_cg_16(bu_base + off, wu_base + (size_t)r * H + k_off + c8 * 8); } }; const int num_k_tiles = H / BK; // Prologue: fill STAGES - 1 buffers #pragma unroll for (int s = 0; s < STAGES - 1; ++s) { if (s < num_k_tiles) { load_stage(s, s); } cp_async_commit(); } const int lane_r16 = lane & 15; const int lane_r7 = lane & 7; const int lane_c_bit = lane >> 4; // 0 or 1 const int warp_m_base = warp_m * WM; const int warp_n_base = warp_n * WN; // Main K-loop for (int kt = 0; kt < num_k_tiles; ++kt) { const int read_stage = kt % STAGES; const int write_stage = (kt + STAGES - 1) % STAGES; const int next_kt = kt + STAGES - 1; cp_async_wait(); __syncthreads(); // Issue next async stage immediately if (next_kt < num_k_tiles) { load_stage(next_kt, write_stage); } cp_async_commit(); const uint32_t st_base = smem_base + read_stage * STAGE_BYTES; const uint32_t a_base = st_base; const uint32_t bg_base = st_base + A_STAGE_BYTES; const uint32_t bu_base = bg_base + B_STAGE_BYTES; // Compute 4 K-steps of 16 with register double-buffering on A uint32_t ra[2][M_TILES][4]; // Prefetch A for ks = 0 #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r = warp_m_base + mi * 16 + lane_r16; int c8 = lane_c_bit; uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16; ldmatrix_x4(ra[0][mi][0], ra[0][mi][1], ra[0][mi][2], ra[0][mi][3], addr); } #pragma unroll for (int ks = 0; ks < 4; ++ks) { const int cur = ks & 1; const int nxt = cur ^ 1; // Prefetch A for ks + 1 if (ks < 3) { #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r = warp_m_base + mi * 16 + lane_r16; int c8 = (ks + 1) * 2 + lane_c_bit; uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16; ldmatrix_x4(ra[nxt][mi][0], ra[nxt][mi][1], ra[nxt][mi][2], ra[nxt][mi][3], addr); } } // Stream B_gate and B_up per N-tile and run MMA #pragma unroll for (int ni = 0; ni < N_TILES; ++ni) { int nb = warp_n_base + ni * 16 + lane_r16; int c8 = ks * 2 + lane_c_bit; uint32_t off = (nb * 8 + (c8 ^ lane_r7)) * 16; uint32_t bg0, bg1, bg2, bg3; uint32_t bu0, bu1, bu2, bu3; ldmatrix_x4(bg0, bg1, bg2, bg3, bg_base + off); ldmatrix_x4(bu0, bu1, bu2, bu3, bu_base + off); #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { mma_m16n8k16_bf16( acc_g[mi][2 * ni + 0][0], acc_g[mi][2 * ni + 0][1], acc_g[mi][2 * ni + 0][2], acc_g[mi][2 * ni + 0][3], ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3], bg0, bg2 ); mma_m16n8k16_bf16( acc_g[mi][2 * ni + 1][0], acc_g[mi][2 * ni + 1][1], acc_g[mi][2 * ni + 1][2], acc_g[mi][2 * ni + 1][3], ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3], bg1, bg3 ); mma_m16n8k16_bf16( acc_u[mi][2 * ni + 0][0], acc_u[mi][2 * ni + 0][1], acc_u[mi][2 * ni + 0][2], acc_u[mi][2 * ni + 0][3], ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3], bu0, bu2 ); mma_m16n8k16_bf16( acc_u[mi][2 * ni + 1][0], acc_u[mi][2 * ni + 1][1], acc_u[mi][2 * ni + 1][2], acc_u[mi][2 * ni + 1][3], ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3], bu1, bu3 ); } } } } // Drain any remaining async ops before reusing SMEM for epilogue cp_async_wait<0>(); __syncthreads(); // Epilogue: compute SiLU(gate) * up in FP32, stage bf16 to SMEM with +8 padding constexpr int S_H_STRIDE = BN_OUT + 8; // 72 bf16 = 144B (16B aligned, bank-conflict-free!) __nv_bfloat16* s_h = reinterpret_cast<__nv_bfloat16*>(smem_raw); const int g = lane >> 2; const int t = lane & 3; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r0 = warp_m_base + mi * 16 + g; int r1 = r0 + 8; #pragma unroll for (int n8 = 0; n8 < N_TILES * 2; ++n8) { int c0 = warp_n_base + n8 * 8 + t * 2; float h0 = silu_f32(acc_g[mi][n8][0]) * acc_u[mi][n8][0]; float h1 = silu_f32(acc_g[mi][n8][1]) * acc_u[mi][n8][1]; float h2 = silu_f32(acc_g[mi][n8][2]) * acc_u[mi][n8][2]; float h3 = silu_f32(acc_g[mi][n8][3]) * acc_u[mi][n8][3]; *reinterpret_cast<__nv_bfloat162*>(&s_h[r0 * S_H_STRIDE + c0]) = __floats2bfloat162_rn(h0, h1); *reinterpret_cast<__nv_bfloat162*>(&s_h[r1 * S_H_STRIDE + c0]) = __floats2bfloat162_rn(h2, h3); } } __syncthreads(); // Coalesced 128-bit stores to global h_out[row_base + r, col_base + c] constexpr int H_CHUNKS = BM * (BN_OUT / 8); // BM * 8 #pragma unroll for (int idx = tid; idx < H_CHUNKS; idx += NTHREADS) { int r = idx >> 3; int c8 = idx & 7; if (r < m_valid) { uint4 v = reinterpret_cast(&s_h[r * S_H_STRIDE])[c8]; reinterpret_cast(h_out + (size_t)(row_base + r) * I + col_base)[c8] = v; } } } // ============================================================================ // REGIME B: Fused Grouped GEMM2 (Down Proj + Weight Scale + v4.f32 Scatter-Add) // Also supports Shared Expert (FUSE_ADD_CAST -> direct bf16 output!) // ============================================================================ // Tile: BM x BN (BN = 128), BK = 64 // 8 warps (256 threads): // - BM = 32: WARPS_M = 2 (WM = 16, 1 tile), WARPS_N = 4 (WN = 32, 2 tiles of 16) // - BM = 64: WARPS_M = 2 (WM = 32, 2 tiles), WARPS_N = 4 (WN = 32, 2 tiles of 16) // - BM = 128: WARPS_M = 4 (WM = 32, 2 tiles), WARPS_N = 2 (WN = 64, 4 tiles of 16) // EPILOGUE_MODE: // 0 = Routed scatter-add: scale by sorted_weights[r], red.global.add.v4.f32 -> out_fp32[tok] // 1 = Shared add to out_fp32: red.global.add.v4.f32 -> out_fp32[row] (for multi-shared s < n_shared-1) // 2 = Shared final fused add+cast: out_bf16[row] = bf16(acc + out_fp32[row]) (for last shared expert!) template __global__ __launch_bounds__(WARPS_M * WARPS_N * 32, 1) void moe_gemm2_kernel( const __nv_bfloat16* __restrict__ h_in, // (M_total or T, I) const __nv_bfloat16* __restrict__ w2, // (E or n_shared, H, I) float* __restrict__ out_fp32, // (T, H) __nv_bfloat16* __restrict__ out_bf16, // (T, H) [mode 2] const int32_t* __restrict__ expert_offsets, // (E + 1) [mode 0] const int32_t* __restrict__ expert_tile_prefix, // (E + 1) [mode 0] const int32_t* __restrict__ m_tile_to_expert, // (max_m_tiles) [mode 0] const int32_t* __restrict__ sorted_token_ids, // (M_total) [mode 0] const float* __restrict__ sorted_weights, // (M_total) [mode 0] int T, int E, int H, int I, int shared_idx ) { static_assert(BK == 64, "BK must be 64"); static_assert(BN == 128, "BN must be 128"); constexpr bool IS_SHARED = (EPILOGUE_MODE != 0); constexpr int NWARPS = WARPS_M * WARPS_N; constexpr int NTHREADS = NWARPS * 32; constexpr int WM = BM / WARPS_M; constexpr int WN = BN / WARPS_N; constexpr int M_TILES = WM / 16; constexpr int N_TILES = WN / 16; const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int warp_m = warp_id / WARPS_N; const int warp_n = warp_id % WARPS_N; const int n_tiles_total = H / BN; // 32 for H=4096 int e, row_base, m_valid, n_blk; if constexpr (IS_SHARED) { const int work_id = blockIdx.x; const int m_tiles_shared = (T + BM - 1) / BM; if (work_id >= m_tiles_shared * n_tiles_total) return; e = shared_idx; n_blk = work_id / m_tiles_shared; int m_blk = work_id - n_blk * m_tiles_shared; row_base = m_blk * BM; m_valid = min(BM, T - row_base); } else { const int work_id = blockIdx.x; const int total_m_tiles = expert_tile_prefix[E]; if (work_id >= total_m_tiles * n_tiles_total) return; int m_tile_approx = work_id / n_tiles_total; e = m_tile_to_expert[m_tile_approx]; int p0 = expert_tile_prefix[e]; int p1 = expert_tile_prefix[e + 1]; int m_tiles_e = p1 - p0; int local = work_id - p0 * n_tiles_total; n_blk = local / m_tiles_e; int m_blk = local - n_blk * m_tiles_e; int off0 = expert_offsets[e]; int off1 = expert_offsets[e + 1]; row_base = off0 + m_blk * BM; m_valid = min(BM, off1 - row_base); } const int col_base = n_blk * BN; __shared__ int32_t s_tok[BM]; __shared__ float s_wt[BM]; if constexpr (!IS_SHARED) { if (tid < BM) { if (tid < m_valid) { s_tok[tid] = sorted_token_ids[row_base + tid]; s_wt[tid] = sorted_weights[row_base + tid]; } else { s_tok[tid] = 0; s_wt[tid] = 0.0f; } } __syncthreads(); } extern __shared__ char smem_raw[]; constexpr uint32_t A_STAGE_BYTES = BM * 128; constexpr uint32_t B_STAGE_BYTES = BN * 128; // 16384 B constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES; const uint32_t smem_base = smem_u32(smem_raw); const __nv_bfloat16* w2_base = w2 + ((size_t)e * (size_t)H + (size_t)col_base) * (size_t)I; float acc[M_TILES][N_TILES * 2][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { #pragma unroll for (int n8 = 0; n8 < N_TILES * 2; ++n8) { acc[mi][n8][0] = 0.0f; acc[mi][n8][1] = 0.0f; acc[mi][n8][2] = 0.0f; acc[mi][n8][3] = 0.0f; } } auto load_stage = [&](int kt, int stage) { const int k_off = kt * BK; const uint32_t st_base = smem_base + stage * STAGE_BYTES; const uint32_t a_base = st_base; const uint32_t b_base = st_base + A_STAGE_BYTES; // Load A (h_in is contiguous!) constexpr int A_CHUNKS = BM * 8; #pragma unroll for (int idx = tid; idx < A_CHUNKS; idx += NTHREADS) { int r = idx >> 3; int c8 = idx & 7; int swiz = c8 ^ (r & 7); uint32_t dst = a_base + (r * 8 + swiz) * 16; int valid_bytes = (r < m_valid) ? 16 : 0; int r_clamped = (r < m_valid) ? (row_base + r) : row_base; const __nv_bfloat16* src = h_in + (size_t)r_clamped * I + k_off + c8 * 8; cp_async_cg_16_zfill(dst, src, valid_bytes); } // Load B (w2) constexpr int B_CHUNKS = BN * 8; // 1024 chunks #pragma unroll for (int idx = tid; idx < B_CHUNKS; idx += NTHREADS) { int r = idx >> 3; int c8 = idx & 7; int swiz = c8 ^ (r & 7); uint32_t dst = b_base + (r * 8 + swiz) * 16; cp_async_cg_16(dst, w2_base + (size_t)r * I + k_off + c8 * 8); } }; const int num_k_tiles = I / BK; #pragma unroll for (int s = 0; s < STAGES - 1; ++s) { if (s < num_k_tiles) { load_stage(s, s); } cp_async_commit(); } const int lane_r16 = lane & 15; const int lane_r7 = lane & 7; const int lane_c_bit = lane >> 4; const int warp_m_base = warp_m * WM; const int warp_n_base = warp_n * WN; for (int kt = 0; kt < num_k_tiles; ++kt) { const int read_stage = kt % STAGES; const int write_stage = (kt + STAGES - 1) % STAGES; const int next_kt = kt + STAGES - 1; cp_async_wait(); __syncthreads(); if (next_kt < num_k_tiles) { load_stage(next_kt, write_stage); } cp_async_commit(); const uint32_t st_base = smem_base + read_stage * STAGE_BYTES; const uint32_t a_base = st_base; const uint32_t b_base = st_base + A_STAGE_BYTES; uint32_t ra[2][M_TILES][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r = warp_m_base + mi * 16 + lane_r16; int c8 = lane_c_bit; uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16; ldmatrix_x4(ra[0][mi][0], ra[0][mi][1], ra[0][mi][2], ra[0][mi][3], addr); } #pragma unroll for (int ks = 0; ks < 4; ++ks) { const int cur = ks & 1; const int nxt = cur ^ 1; if (ks < 3) { #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r = warp_m_base + mi * 16 + lane_r16; int c8 = (ks + 1) * 2 + lane_c_bit; uint32_t addr = a_base + (r * 8 + (c8 ^ lane_r7)) * 16; ldmatrix_x4(ra[nxt][mi][0], ra[nxt][mi][1], ra[nxt][mi][2], ra[nxt][mi][3], addr); } } #pragma unroll for (int ni = 0; ni < N_TILES; ++ni) { int nb = warp_n_base + ni * 16 + lane_r16; int c8 = ks * 2 + lane_c_bit; uint32_t b0, b1, b2, b3; ldmatrix_x4(b0, b1, b2, b3, b_base + (nb * 8 + (c8 ^ lane_r7)) * 16); #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { mma_m16n8k16_bf16( acc[mi][2 * ni + 0][0], acc[mi][2 * ni + 0][1], acc[mi][2 * ni + 0][2], acc[mi][2 * ni + 0][3], ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3], b0, b2 ); mma_m16n8k16_bf16( acc[mi][2 * ni + 1][0], acc[mi][2 * ni + 1][1], acc[mi][2 * ni + 1][2], acc[mi][2 * ni + 1][3], ra[cur][mi][0], ra[cur][mi][1], ra[cur][mi][2], ra[cur][mi][3], b1, b3 ); } } } } cp_async_wait<0>(); __syncthreads(); // Epilogue: stage FP32 results to SMEM (padded stride 136 floats = 544B, 16B aligned) // Max size: BM=128 -> 128 * 136 * 4 = 69,632 B (fits in 96KB SMEM!) // BM=64 -> 64 * 136 * 4 = 34,816 B (fits in 96KB SMEM!) // BM=32 -> 32 * 136 * 4 = 17,408 B (fits in 80KB SMEM!) constexpr int S_OUT_STRIDE = BN + 8; // 136 floats float* s_out = reinterpret_cast(smem_raw); const int g = lane >> 2; const int t = lane & 3; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r0 = warp_m_base + mi * 16 + g; int r1 = r0 + 8; float w0 = IS_SHARED ? 1.0f : s_wt[r0]; float w1 = IS_SHARED ? 1.0f : s_wt[r1]; #pragma unroll for (int n8 = 0; n8 < N_TILES * 2; ++n8) { int c0 = warp_n_base + n8 * 8 + t * 2; float2 v0 = make_float2(acc[mi][n8][0] * w0, acc[mi][n8][1] * w0); float2 v1 = make_float2(acc[mi][n8][2] * w1, acc[mi][n8][3] * w1); *reinterpret_cast(&s_out[r0 * S_OUT_STRIDE + c0]) = v0; *reinterpret_cast(&s_out[r1 * S_OUT_STRIDE + c0]) = v1; } } __syncthreads(); // 1 warp writes 1 full row of 128 floats (32 float4s = 512B) in a single coalesced step! for (int r = warp_id; r < m_valid; r += NWARPS) { float4 v = reinterpret_cast(&s_out[r * S_OUT_STRIDE])[lane]; if constexpr (EPILOGUE_MODE == 0) { int tok = s_tok[r]; float* dst = out_fp32 + (size_t)tok * H + col_base + lane * 4; red_global_add_v4_f32(dst, v); } else if constexpr (EPILOGUE_MODE == 1) { int row = row_base + r; float* dst = out_fp32 + (size_t)row * H + col_base + lane * 4; red_global_add_v4_f32(dst, v); } else { int row = row_base + r; const float4 prev = reinterpret_cast( out_fp32 + (size_t)row * H + col_base )[lane]; __nv_bfloat162 b0 = __floats2bfloat162_rn(v.x + prev.x, v.y + prev.y); __nv_bfloat162 b1 = __floats2bfloat162_rn(v.z + prev.z, v.w + prev.w); uint2 packed; packed.x = *reinterpret_cast(&b0); packed.y = *reinterpret_cast(&b1); reinterpret_cast(out_bf16 + (size_t)row * H + col_base)[lane] = packed; } } } __global__ void cast_f32_to_bf16_kernel( const float* __restrict__ src, __nv_bfloat16* __restrict__ dst, size_t n_vec4 ) { size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; size_t stride = (size_t)gridDim.x * blockDim.x; const float4* src4 = reinterpret_cast(src); uint2* dst2 = reinterpret_cast(dst); for (; idx < n_vec4; idx += stride) { float4 v = src4[idx]; __nv_bfloat162 b0 = __floats2bfloat162_rn(v.x, v.y); __nv_bfloat162 b1 = __floats2bfloat162_rn(v.z, v.w); uint2 packed; packed.x = *reinterpret_cast(&b0); packed.y = *reinterpret_cast(&b1); dst2[idx] = packed; } } // ============================================================================ // C++ Launch Wrappers // ============================================================================ template static inline void ensure_smem_once(bool& configured, KernelT kernel, size_t smem_bytes) { if (!configured && smem_bytes > 48 * 1024) { C10_CUDA_CHECK(cudaFuncSetAttribute( kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, static_cast(smem_bytes) )); configured = true; } } void launch_decode_moe( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared, torch::Tensor h_buf, torch::Tensor out ) { const int T = static_cast(x.size(0)); const int H = static_cast(x.size(1)); const int E = static_cast(w1_routed.size(0)); const int I = static_cast(w2_routed.size(2)); const int top_k = static_cast(expert_ids.size(1)); const int n_shared = static_cast(w1_shared.size(0)); const int total_slots = top_k + n_shared; cudaStream_t stream = c10::cuda::getCurrentCUDAStream(); // Kernel D1: 4 warps/block -> 4608 blocks for T=1, 9 slots, I=2048 { constexpr int WPB = 4; dim3 grid((I + WPB - 1) / WPB, total_slots, T); dim3 block(WPB * 32); size_t smem = (size_t)H * sizeof(__nv_bfloat16); moe_decode_gemv1_silu_mul_kernel<<>>( reinterpret_cast(x.data_ptr()), expert_ids.data_ptr(), reinterpret_cast(w1_routed.data_ptr()), reinterpret_cast(w1_shared.data_ptr()), reinterpret_cast<__nv_bfloat16*>(h_buf.data_ptr()), T, E, top_k, n_shared, H, I ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } // Kernel D2: 2 warps/block -> 2048 blocks for T=1, H=4096 (10.9 waves on 188 SMs) { constexpr int WPB = 2; dim3 grid((H + WPB - 1) / WPB, 1, T); dim3 block(WPB * 32); moe_decode_gemv2_reduce_kernel<<>>( reinterpret_cast(h_buf.data_ptr()), expert_ids.data_ptr(), reinterpret_cast(expert_weights.data_ptr()), reinterpret_cast(w2_routed.data_ptr()), reinterpret_cast(w2_shared.data_ptr()), reinterpret_cast<__nv_bfloat16*>(out.data_ptr()), T, E, top_k, n_shared, H, I ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } } template static void run_routed_grouped_impl( const __nv_bfloat16* x_ptr, const __nv_bfloat16* w1_ptr, const __nv_bfloat16* w2_ptr, __nv_bfloat16* h_ptr, float* out_fp32_ptr, const int32_t* offsets_ptr, const int32_t* tile_prefix_ptr, const int32_t* m_tile_to_expert_ptr, const int32_t* sorted_tok_ptr, const float* sorted_wt_ptr, int T, int E, int H, int I, int max_m_tiles, cudaStream_t stream ) { constexpr int BN1_OUT = 64; constexpr int BN2 = 128; constexpr int BK = 64; constexpr int NTHREADS = WARPS_M * WARPS_N * 32; // GEMM1 { static bool cfg1 = false; constexpr size_t smem1 = STAGES * (BM * 128 + 2 * BN1_OUT * 128); auto k1 = moe_gemm1_silu_mul_kernel; ensure_smem_once(cfg1, k1, smem1); int grid1 = max_m_tiles * (I / BN1_OUT); k1<<>>( x_ptr, w1_ptr, h_ptr, offsets_ptr, tile_prefix_ptr, m_tile_to_expert_ptr, sorted_tok_ptr, T, E, H, I, 0 ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } // GEMM2 { static bool cfg2 = false; constexpr size_t smem2_pipe = STAGES * (BM * 128 + BN2 * 128); constexpr size_t smem2_epi = BM * (BN2 + 8) * sizeof(float); constexpr size_t smem2 = smem2_pipe > smem2_epi ? smem2_pipe : smem2_epi; auto k2 = moe_gemm2_kernel; ensure_smem_once(cfg2, k2, smem2); int grid2 = max_m_tiles * (H / BN2); k2<<>>( h_ptr, w2_ptr, out_fp32_ptr, nullptr, offsets_ptr, tile_prefix_ptr, m_tile_to_expert_ptr, sorted_tok_ptr, sorted_wt_ptr, T, E, H, I, 0 ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } } template static void run_shared_impl( const __nv_bfloat16* x_ptr, const __nv_bfloat16* w1_s_ptr, const __nv_bfloat16* w2_s_ptr, __nv_bfloat16* h_s_ptr, float* out_fp32_ptr, __nv_bfloat16* out_bf16_ptr, int T, int H, int I, int s_idx, bool is_last, cudaStream_t stream ) { constexpr int BN1_OUT = 64; constexpr int BN2 = 128; constexpr int BK = 64; constexpr int NTHREADS = WARPS_M * WARPS_N * 32; const int m_tiles = (T + BM - 1) / BM; // Shared GEMM1 { static bool cfg_s1 = false; constexpr size_t smem1 = STAGES * (BM * 128 + 2 * BN1_OUT * 128); auto k1 = moe_gemm1_silu_mul_kernel; ensure_smem_once(cfg_s1, k1, smem1); int grid1 = m_tiles * (I / BN1_OUT); k1<<>>( x_ptr, w1_s_ptr, h_s_ptr, nullptr, nullptr, nullptr, nullptr, T, 1, H, I, s_idx ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } // Shared GEMM2 { static bool cfg_s2_last = false; static bool cfg_s2_mid = false; constexpr size_t smem2_pipe = STAGES * (BM * 128 + BN2 * 128); constexpr size_t smem2_epi = BM * (BN2 + 8) * sizeof(float); constexpr size_t smem2 = smem2_pipe > smem2_epi ? smem2_pipe : smem2_epi; int grid2 = m_tiles * (H / BN2); if (is_last) { auto k2 = moe_gemm2_kernel; ensure_smem_once(cfg_s2_last, k2, smem2); k2<<>>( h_s_ptr, w2_s_ptr, out_fp32_ptr, out_bf16_ptr, nullptr, nullptr, nullptr, nullptr, nullptr, T, 1, H, I, s_idx ); } else { auto k2 = moe_gemm2_kernel; ensure_smem_once(cfg_s2_mid, k2, smem2); k2<<>>( h_s_ptr, w2_s_ptr, out_fp32_ptr, nullptr, nullptr, nullptr, nullptr, nullptr, nullptr, T, 1, H, I, s_idx ); } C10_CUDA_KERNEL_LAUNCH_CHECK(); } } void launch_grouped_moe( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared, torch::Tensor h_routed, torch::Tensor h_shared, torch::Tensor out_fp32, torch::Tensor out_bf16, torch::Tensor expert_counts, torch::Tensor expert_offsets, torch::Tensor write_cursors, torch::Tensor expert_tile_prefix, torch::Tensor m_tile_to_expert, torch::Tensor sorted_token_ids, torch::Tensor sorted_weights, int bm_routed ) { const int T = static_cast(x.size(0)); const int H = static_cast(x.size(1)); const int E = static_cast(w1_routed.size(0)); const int I = static_cast(w2_routed.size(2)); const int top_k = static_cast(expert_ids.size(1)); const int n_shared = static_cast(w1_shared.size(0)); const int M_total = T * top_k; const int max_m_tiles = (M_total + bm_routed - 1) / bm_routed + E; cudaStream_t stream = c10::cuda::getCurrentCUDAStream(); // 1. Zero expert_counts and out_fp32 C10_CUDA_CHECK(cudaMemsetAsync(expert_counts.data_ptr(), 0, (size_t)E * sizeof(int32_t), stream)); C10_CUDA_CHECK(cudaMemsetAsync(out_fp32.data_ptr(), 0, (size_t)T * (size_t)H * sizeof(float), stream)); // 2. Histogram expert counts { int blocks = min(256, (M_total + 255) / 256); moe_count_experts_kernel<<>>( expert_ids.data_ptr(), expert_counts.data_ptr(), M_total, E ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } // 3. Prefix sum + tile lookup table moe_build_offsets_and_tiles_kernel<<<1, 1, 0, stream>>>( expert_counts.data_ptr(), expert_offsets.data_ptr(), write_cursors.data_ptr(), expert_tile_prefix.data_ptr(), m_tile_to_expert.data_ptr(), E, bm_routed ); C10_CUDA_KERNEL_LAUNCH_CHECK(); // 4. Scatter sorted token_ids and weights { int blocks = min(256, (M_total + 255) / 256); moe_scatter_tokens_kernel<<>>( expert_ids.data_ptr(), reinterpret_cast(expert_weights.data_ptr()), write_cursors.data_ptr(), sorted_token_ids.data_ptr(), sorted_weights.data_ptr(), M_total, top_k ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } const __nv_bfloat16* x_ptr = reinterpret_cast(x.data_ptr()); const __nv_bfloat16* w1_r_ptr = reinterpret_cast(w1_routed.data_ptr()); const __nv_bfloat16* w2_r_ptr = reinterpret_cast(w2_routed.data_ptr()); __nv_bfloat16* h_r_ptr = reinterpret_cast<__nv_bfloat16*>(h_routed.data_ptr()); float* out_f32_ptr = out_fp32.data_ptr(); __nv_bfloat16* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()); // 5. Routed Experts (GEMM1 fused SiLU*mul -> GEMM2 fused weight-scale + v4.f32 scatter-add) if (bm_routed == 32) { run_routed_grouped_impl<32, 4, 2, 4>( x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr, expert_offsets.data_ptr(), expert_tile_prefix.data_ptr(), m_tile_to_expert.data_ptr(), sorted_token_ids.data_ptr(), sorted_weights.data_ptr(), T, E, H, I, max_m_tiles, stream ); } else if (bm_routed == 64) { run_routed_grouped_impl<64, 4, 2, 4>( x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr, expert_offsets.data_ptr(), expert_tile_prefix.data_ptr(), m_tile_to_expert.data_ptr(), sorted_token_ids.data_ptr(), sorted_weights.data_ptr(), T, E, H, I, max_m_tiles, stream ); } else { run_routed_grouped_impl<128, 3, 4, 2>( x_ptr, w1_r_ptr, w2_r_ptr, h_r_ptr, out_f32_ptr, expert_offsets.data_ptr(), expert_tile_prefix.data_ptr(), m_tile_to_expert.data_ptr(), sorted_token_ids.data_ptr(), sorted_weights.data_ptr(), T, E, H, I, max_m_tiles, stream ); } // 6. Shared Expert(s) — last shared expert fuses add(out_fp32) + cast(bf16) -> out_bf16! if (n_shared > 0) { const __nv_bfloat16* w1_s_ptr = reinterpret_cast(w1_shared.data_ptr()); const __nv_bfloat16* w2_s_ptr = reinterpret_cast(w2_shared.data_ptr()); __nv_bfloat16* h_s_ptr = reinterpret_cast<__nv_bfloat16*>(h_shared.data_ptr()); for (int s = 0; s < n_shared; ++s) { bool is_last = (s == n_shared - 1); if (T <= 32) { run_shared_impl<32, 4, 2, 4>( x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr, T, H, I, s, is_last, stream ); } else if (T <= 96) { run_shared_impl<64, 4, 2, 4>( x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr, T, H, I, s, is_last, stream ); } else { run_shared_impl<128, 3, 4, 2>( x_ptr, w1_s_ptr, w2_s_ptr, h_s_ptr, out_f32_ptr, out_bf16_ptr, T, H, I, s, is_last, stream ); } } } else { size_t n_vec4 = ((size_t)T * (size_t)H) / 4; int blocks = static_cast(min((size_t)1024, (n_vec4 + 255) / 256)); cast_f32_to_bf16_kernel<<>>( out_f32_ptr, out_bf16_ptr, n_vec4 ); C10_CUDA_KERNEL_LAUNCH_CHECK(); } } """ _CPP_DECLS = """ void launch_decode_moe( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared, torch::Tensor h_buf, torch::Tensor out ); void launch_grouped_moe( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared, torch::Tensor h_routed, torch::Tensor h_shared, torch::Tensor out_fp32, torch::Tensor out_bf16, torch::Tensor expert_counts, torch::Tensor expert_offsets, torch::Tensor write_cursors, torch::Tensor expert_tile_prefix, torch::Tensor m_tile_to_expert, torch::Tensor sorted_token_ids, torch::Tensor sorted_weights, int bm_routed ); """ _EXT = None def _get_ext(): global _EXT if _EXT is None: _EXT = load_inline( name="glm52_fused_moe_sm120_v1", cpp_sources=_CPP_DECLS, cuda_sources=_CUDA_SRC, functions=["launch_decode_moe", "launch_grouped_moe"], extra_cuda_cflags=["-O3", "-arch=sm_120a", "--expt-relaxed-constexpr"], verbose=False, ) return _EXT class _Workspace: """Reusable GPU scratch buffers to avoid per-forward allocations.""" def __init__(self): self.device = None self.h_decode = None self.h_routed = None self.h_shared = None self.out_fp32 = None self.expert_counts = None self.expert_offsets = None self.write_cursors = None self.expert_tile_prefix = None self.m_tile_to_expert = None self.sorted_token_ids = None self.sorted_weights = None def ensure_decode(self, device, T: int, total_slots: int, I: int): need = T * total_slots * I if self.device != device or self.h_decode is None or self.h_decode.numel() < need: self.device = device self.h_decode = torch.empty(need, dtype=torch.bfloat16, device=device) return self.h_decode def ensure_grouped(self, device, T: int, E: int, top_k: int, H: int, I: int, bm: int): M_total = T * top_k max_m_tiles = (M_total + bm - 1) // bm + E if self.device != device: self.device = device self.h_routed = None self.h_shared = None self.out_fp32 = None self.expert_counts = None self.expert_offsets = None self.write_cursors = None self.expert_tile_prefix = None self.m_tile_to_expert = None self.sorted_token_ids = None self.sorted_weights = None if self.h_routed is None or self.h_routed.numel() < M_total * I: self.h_routed = torch.empty((M_total, I), dtype=torch.bfloat16, device=device) if self.h_shared is None or self.h_shared.numel() < T * I: self.h_shared = torch.empty((T, I), dtype=torch.bfloat16, device=device) if self.out_fp32 is None or self.out_fp32.numel() < T * H: self.out_fp32 = torch.empty((T, H), dtype=torch.float32, device=device) if self.expert_counts is None or self.expert_counts.numel() < E: self.expert_counts = torch.empty(E, dtype=torch.int32, device=device) if self.expert_offsets is None or self.expert_offsets.numel() < E + 1: self.expert_offsets = torch.empty(E + 1, dtype=torch.int32, device=device) if self.write_cursors is None or self.write_cursors.numel() < E: self.write_cursors = torch.empty(E, dtype=torch.int32, device=device) if self.expert_tile_prefix is None or self.expert_tile_prefix.numel() < E + 1: self.expert_tile_prefix = torch.empty(E + 1, dtype=torch.int32, device=device) if self.m_tile_to_expert is None or self.m_tile_to_expert.numel() < max_m_tiles: self.m_tile_to_expert = torch.empty(max_m_tiles, dtype=torch.int32, device=device) if self.sorted_token_ids is None or self.sorted_token_ids.numel() < M_total: self.sorted_token_ids = torch.empty(M_total, dtype=torch.int32, device=device) if self.sorted_weights is None or self.sorted_weights.numel() < M_total: self.sorted_weights = torch.empty(M_total, dtype=torch.float32, device=device) _WORKSPACES: dict[int, _Workspace] = {} def _get_workspace(device: torch.device) -> _Workspace: idx = device.index if device.index is not None else 0 ws = _WORKSPACES.get(idx) if ws is None: ws = _Workspace() _WORKSPACES[idx] = ws return ws class Model(nn.Module): def __init__( self, T: int, E: int, top_k: int, n_shared: int, H: int, I: int, ): super().__init__() self.T, self.E, self.top_k = T, E, top_k self.n_shared, self.H, self.I = n_shared, H, I self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16)) self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16)) self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16)) self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16)) for p in self.parameters(): nn.init.normal_(p, std=0.02) # Pre-compile extension at init time (outside timed forward) _get_ext() def forward( self, x: torch.Tensor, expert_ids: torch.Tensor, expert_weights: torch.Tensor, ) -> torch.Tensor: T, H = x.shape E = self.E top_k = self.top_k n_shared = self.n_shared I = self.I ext = _get_ext() ws = _get_workspace(x.device) out = torch.empty((T, H), dtype=torch.bfloat16, device=x.device) x_c = x.contiguous() ids_c = expert_ids.contiguous() wts_c = expert_weights.contiguous() if T <= 4 and (H % 8 == 0) and (I % 8 == 0): h_buf = ws.ensure_decode(x.device, T, top_k + n_shared, I) ext.launch_decode_moe( x_c, ids_c, wts_c, self.w1_routed, self.w2_routed, self.w1_shared, self.w2_shared, h_buf, out, ) return out # Choose routed BM based on average tokens per expert m_avg = (T * top_k) / max(E, 1) bm_override = os.environ.get("MOE_BM_OVERRIDE") if bm_override is not None: bm = int(bm_override) elif m_avg <= 20.0: bm = 32 elif m_avg <= 160.0: bm = 64 else: bm = 64 ws.ensure_grouped(x.device, T, E, top_k, H, I, bm) ext.launch_grouped_moe( x_c, ids_c, wts_c, self.w1_routed, self.w2_routed, self.w1_shared, self.w2_shared, ws.h_routed, ws.h_shared, ws.out_fp32, out, ws.expert_counts, ws.expert_offsets, ws.write_cursors, ws.expert_tile_prefix, ws.m_tile_to_expert, ws.sorted_token_ids, ws.sorted_weights, bm, ) return out