from __future__ import annotations import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline OP_TYPE = "glm52_fused_moe" SUPPORTED_PRECISIONS = ["bf16"] HARDWARE_REQUIRED = ["RTX_PRO_6000"] _CUDA_SRC = r""" #include #include #include #include #include #include #include namespace { struct alignas(16) TileInfo { int32_t expert_id; int32_t row_offset; int32_t valid_rows; int32_t pad; }; struct Workspace { torch::Tensor h_routed; torch::Tensor h_shared; torch::Tensor out_fp32; torch::Tensor sorted_token_ids; torch::Tensor sorted_weights; torch::Tensor tile_map; torch::Tensor num_tiles; torch::Tensor block_hist; torch::Tensor expert_row_offsets; __nv_bfloat16* h_routed_ptr = nullptr; __nv_bfloat16* h_shared_ptr = nullptr; float* out_fp32_ptr = nullptr; int32_t* sorted_tok_ptr = nullptr; float* sorted_wt_ptr = nullptr; TileInfo* tile_map_ptr = nullptr; int32_t* num_tiles_ptr = nullptr; int32_t* block_hist_ptr = nullptr; int32_t* expert_off_ptr = nullptr; int64_t cap_hr = 0; int64_t cap_hs = 0; int64_t cap_out = 0; int64_t cap_m = 0; int64_t cap_tm = 0; int64_t cap_bh = 0; int64_t cap_eo = 0; int dev_idx = -1; inline void ensure( int64_t M_tot, int64_t T, int64_t E, int64_t H, int64_t I, int64_t max_tiles, int64_t sort_blocks, torch::Device dev ) { int d = dev.index(); int64_t need_hr = std::max(M_tot, 32) * I; int64_t need_hs = std::max(T, 1) * I; int64_t need_out = std::max(T, 1) * H; int64_t need_m = std::max(M_tot, 32); int64_t need_tm = std::max(max_tiles, 32) * 4; int64_t need_bh = std::max(sort_blocks * E, 1024); int64_t need_eo = std::max(E + 4, 1028); if (__builtin_expect( d == dev_idx && need_hr <= cap_hr && need_hs <= cap_hs && need_out <= cap_out && need_m <= cap_m && need_tm <= cap_tm && need_bh <= cap_bh && need_eo <= cap_eo, 1)) { return; } auto opt_bf16 = torch::TensorOptions().dtype(torch::kBFloat16).device(dev); auto opt_f32 = torch::TensorOptions().dtype(torch::kFloat32).device(dev); auto opt_i32 = torch::TensorOptions().dtype(torch::kInt32).device(dev); if (d != dev_idx) { cap_hr = cap_hs = cap_out = cap_m = cap_tm = cap_bh = cap_eo = 0; dev_idx = d; } if (need_hr > cap_hr) { h_routed = torch::empty({need_hr}, opt_bf16); h_routed_ptr = reinterpret_cast<__nv_bfloat16*>(h_routed.data_ptr()); cap_hr = need_hr; } if (need_hs > cap_hs) { h_shared = torch::empty({need_hs}, opt_bf16); h_shared_ptr = reinterpret_cast<__nv_bfloat16*>(h_shared.data_ptr()); cap_hs = need_hs; } if (need_out > cap_out) { out_fp32 = torch::empty({need_out}, opt_f32); out_fp32_ptr = out_fp32.data_ptr(); cap_out = need_out; } if (need_m > cap_m) { sorted_token_ids = torch::empty({need_m}, opt_i32); sorted_weights = torch::empty({need_m}, opt_f32); sorted_tok_ptr = sorted_token_ids.data_ptr(); sorted_wt_ptr = sorted_weights.data_ptr(); cap_m = need_m; } if (need_tm > cap_tm) { tile_map = torch::empty({need_tm}, opt_i32); tile_map_ptr = reinterpret_cast(tile_map.data_ptr()); cap_tm = need_tm; } if (!num_tiles.defined() || num_tiles.device() != dev) { num_tiles = torch::empty({4}, opt_i32); num_tiles_ptr = num_tiles.data_ptr(); } if (need_bh > cap_bh) { block_hist = torch::empty({need_bh}, opt_i32); block_hist_ptr = block_hist.data_ptr(); cap_bh = need_bh; } if (need_eo > cap_eo) { expert_row_offsets = torch::empty({need_eo}, opt_i32); expert_off_ptr = expert_row_offsets.data_ptr(); cap_eo = need_eo; } } }; static Workspace g_ws; // ============================================================================ // Device PTX Helpers (SM120 Blackwell) // ============================================================================ __device__ __forceinline__ uint32_t smem_swizzle_128b(uint32_t base_smem, int r, int c_bf16) { int chunk = (c_bf16 >> 3) ^ (r & 7); return base_smem + (r << 7) + (chunk << 4); } __device__ __forceinline__ uint64_t make_policy_evict_first() { uint64_t p; asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;\n" : "=l"(p)); return p; } __device__ __forceinline__ uint64_t make_policy_evict_last() { uint64_t p; asm volatile("createpolicy.fractional.L2::evict_last.b64 %0, 1.0;\n" : "=l"(p)); return p; } __device__ __forceinline__ void mbar_init(uint32_t mbar_smem, uint32_t count) { asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(mbar_smem), "r"(count)); } __device__ __forceinline__ void mbar_expect_tx(uint32_t mbar_smem, uint32_t bytes) { asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" :: "r"(mbar_smem), "r"(bytes)); } __device__ __forceinline__ void mbar_wait(uint32_t mbar_smem, uint32_t parity) { asm volatile( "{\n" ".reg .pred P_DONE;\n" "MBAR_SPIN_%=:\n" "mbarrier.try_wait.parity.shared::cta.b64 P_DONE, [%0], %1;\n" "@!P_DONE bra MBAR_SPIN_%=;\n" "}\n" :: "r"(mbar_smem), "r"(parity) ); } __device__ __forceinline__ void tma_load_2d_hint( uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem, int32_t c0, int32_t c1, uint64_t cache_policy ) { asm volatile( "cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint" " [%0], [%1, {%3, %4}], [%2], %5;\n" :: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1), "l"(cache_policy) : "memory" ); } __device__ __forceinline__ void tma_load_2d_lru( uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem, int32_t c0, int32_t c1 ) { asm volatile( "cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes" " [%0], [%1, {%3, %4}], [%2];\n" :: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1) : "memory" ); } __device__ __forceinline__ void tma_load_3d_hint( uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem, int32_t c0, int32_t c1, int32_t c2, uint64_t cache_policy ) { asm volatile( "cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint" " [%0], [%1, {%3, %4, %5}], [%2], %6;\n" :: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2), "l"(cache_policy) : "memory" ); } __device__ __forceinline__ void tma_load_3d_lru( uint32_t dst_smem, const CUtensorMap* tmap_ptr, uint32_t mbar_smem, int32_t c0, int32_t c1, int32_t c2 ) { asm volatile( "cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes" " [%0], [%1, {%3, %4, %5}], [%2];\n" :: "r"(dst_smem), "l"(tmap_ptr), "r"(mbar_smem), "r"(c0), "r"(c1), "r"(c2) : "memory" ); } __device__ __forceinline__ void cp_async_16b_pred( uint32_t dst_smem, const void* src_gmem, int src_bytes ) { asm volatile( "cp.async.cg.shared.global [%0], [%1], 16, %2;\n" :: "r"(dst_smem), "l"(src_gmem), "r"(src_bytes) ); } __device__ __forceinline__ void ldmatrix_x4(uint32_t (&r)[4], uint32_t smem_addr) { asm volatile( "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3]) : "r"(smem_addr) ); } __device__ __forceinline__ void mma_m16n8k16_bf16( float (&d)[4], const uint32_t (&a)[4], uint32_t b0, uint32_t b1 ) { asm volatile( "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%0, %1, %2, %3};\n" : "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3]) : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1) ); } __device__ __forceinline__ void red_add_v4_f32(float* dst, float v0, float v1, float v2, float v3) { asm volatile( "red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n" :: "l"(dst), "f"(v0), "f"(v1), "f"(v2), "f"(v3) : "memory" ); } __device__ __forceinline__ uint4 ld_global_cs_u4(const void* ptr) { uint4 v; asm volatile( "ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr) ); return v; } __device__ __forceinline__ uint4 ld_global_nc_u4(const void* ptr) { uint4 v; asm volatile( "ld.global.nc.v4.u32 {%0, %1, %2, %3}, [%4];\n" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr) ); return v; } __device__ __forceinline__ float dot8_bf16_f32(uint4 a, uint4 b) { float2 ax = __bfloat1622float2(*reinterpret_cast(&a.x)); float2 ay = __bfloat1622float2(*reinterpret_cast(&a.y)); float2 az = __bfloat1622float2(*reinterpret_cast(&a.z)); float2 aw = __bfloat1622float2(*reinterpret_cast(&a.w)); float2 bx = __bfloat1622float2(*reinterpret_cast(&b.x)); float2 by = __bfloat1622float2(*reinterpret_cast(&b.y)); float2 bz = __bfloat1622float2(*reinterpret_cast(&b.z)); float2 bw = __bfloat1622float2(*reinterpret_cast(&b.w)); float s0 = fmaf(ax.x, bx.x, ax.y * bx.y); float s1 = fmaf(ay.x, by.x, ay.y * by.y); float s2 = fmaf(az.x, bz.x, az.y * bz.y); float s3 = fmaf(aw.x, bw.x, aw.y * bw.y); return (s0 + s1) + (s2 + s3); } extern __shared__ __align__(128) uint8_t g_dyn_smem[]; // ============================================================================ // Wave-Balanced Persistent Lockless Decode Path (T <= 2): 98% SM Wave Balance! // 940 CTAs = 5 CTAs/SM * 188 SMs, 128 threads/CTA = 3760 active warps, 0 SMEM! // ============================================================================ __global__ void __launch_bounds__(128, 5) decode_stage1_persistent_kernel( const __nv_bfloat16* __restrict__ x, // (T, H) const int64_t* __restrict__ expert_ids, // (T, top_k) const __nv_bfloat16* __restrict__ w1_routed, // (E, 2*I, H) const __nv_bfloat16* __restrict__ w1_shared, // (n_shared, 2*I, H) __nv_bfloat16* __restrict__ h_decode, // (T * slots_per_tok, I) float* __restrict__ out_fp32, // (T, H) - zeroed here! int T, int top_k, int n_shared, int H, int I ) { const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; // Zero out_fp32 using first few CTAs const int zero_idx = blockIdx.x * 128 + tid; const int total_f4 = (T * H) >> 2; if (zero_idx < total_f4) { reinterpret_cast(out_fp32)[zero_idx] = make_float4(0.f, 0.f, 0.f, 0.f); } const int slots_per_tok = top_k + n_shared; const int total_pairs = T * slots_per_tok; const int total_jobs = total_pairs * I; // 1 (gate, up) col per warp -> 18432 jobs at T=1! const int global_warp = blockIdx.x * 4 + warp_id; const int num_warps = gridDim.x * 4; const int H_u4 = H >> 3; for (int job = global_warp; job < total_jobs; job += num_warps) { const int pair_idx = job / I; const int i_col = job - pair_idx * I; const int t = pair_idx / slots_per_tok; const int slot = pair_idx - t * slots_per_tok; const __nv_bfloat16* w1_base; if (slot < top_k) { int64_t eid = expert_ids[(int64_t)t * top_k + slot]; w1_base = w1_routed + eid * (2LL * I * H); } else { int sid = slot - top_k; w1_base = w1_shared + (int64_t)sid * (2LL * I * H); } const uint4* x_row_u4 = reinterpret_cast(x + (int64_t)t * H); const __nv_bfloat16* g_ptr = w1_base + (int64_t)i_col * H; const __nv_bfloat16* u_ptr = w1_base + (int64_t)(I + i_col) * H; float acc_g = 0.0f, acc_u = 0.0f; #pragma unroll 8 for (int k_u4 = lane; k_u4 < H_u4; k_u4 += 32) { uint4 xv = ld_global_nc_u4(x_row_u4 + k_u4); int k_elem = k_u4 << 3; uint4 gv = ld_global_cs_u4(g_ptr + k_elem); uint4 uv = ld_global_cs_u4(u_ptr + k_elem); acc_g += dot8_bf16_f32(xv, gv); acc_u += dot8_bf16_f32(xv, uv); } #pragma unroll for (int mask = 16; mask > 0; mask >>= 1) { acc_g += __shfl_down_sync(0xffffffff, acc_g, mask); acc_u += __shfl_down_sync(0xffffffff, acc_u, mask); } if (lane == 0) { float h_val = (acc_g / (1.0f + __expf(-acc_g))) * acc_u; h_decode[(int64_t)pair_idx * I + i_col] = __float2bfloat16_rn(h_val); } } } // 940 CTAs, 64 threads/CTA (2 warps) = 1880 warps; 4 rows of w2 per warp -> 9216 jobs (4.902 jobs/warp!) __global__ void __launch_bounds__(64, 10) decode_stage2_persistent_kernel( const __nv_bfloat16* __restrict__ h_decode, // (T * slots_per_tok, I) const int64_t* __restrict__ expert_ids, // (T, top_k) const __nv_bfloat16* __restrict__ expert_weights, // (T, top_k) const __nv_bfloat16* __restrict__ w2_routed, // (E, H, I) const __nv_bfloat16* __restrict__ w2_shared, // (n_shared, H, I) float* __restrict__ out_fp32, // (T, H) int T, int top_k, int n_shared, int H, int I ) { const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int slots_per_tok = top_k + n_shared; const int total_pairs = T * slots_per_tok; const int h_quads = (H + 3) >> 2; const int total_jobs = total_pairs * h_quads; // 9216 jobs at T=1 const int global_warp = blockIdx.x * 2 + warp_id; const int num_warps = gridDim.x * 2; const int I_u4 = I >> 3; for (int job = global_warp; job < total_jobs; job += num_warps) { const int pair_idx = job / h_quads; const int h_q = job - pair_idx * h_quads; const int h0 = h_q << 2; const int t = pair_idx / slots_per_tok; const int slot = pair_idx - t * slots_per_tok; float wt = 1.0f; const __nv_bfloat16* w2_base; if (slot < top_k) { int64_t eid = expert_ids[(int64_t)t * top_k + slot]; wt = __bfloat162float(expert_weights[(int64_t)t * top_k + slot]); w2_base = w2_routed + eid * ((int64_t)H * I); } else { int sid = slot - top_k; w2_base = w2_shared + (int64_t)sid * ((int64_t)H * I); } const uint4* h_row_u4 = reinterpret_cast(h_decode + (int64_t)pair_idx * I); const __nv_bfloat16* r0 = w2_base + (int64_t)(h0 + 0) * I; const __nv_bfloat16* r1 = (h0 + 1 < H) ? (r0 + I) : r0; const __nv_bfloat16* r2 = (h0 + 2 < H) ? (r0 + 2LL * I) : r0; const __nv_bfloat16* r3 = (h0 + 3 < H) ? (r0 + 3LL * I) : r0; float a0 = 0.f, a1 = 0.f, a2 = 0.f, a3 = 0.f; #pragma unroll 8 for (int k_u4 = lane; k_u4 < I_u4; k_u4 += 32) { uint4 hv = ld_global_nc_u4(h_row_u4 + k_u4); int k_elem = k_u4 << 3; uint4 wv0 = ld_global_cs_u4(r0 + k_elem); uint4 wv1 = ld_global_cs_u4(r1 + k_elem); uint4 wv2 = ld_global_cs_u4(r2 + k_elem); uint4 wv3 = ld_global_cs_u4(r3 + k_elem); a0 += dot8_bf16_f32(hv, wv0); a1 += dot8_bf16_f32(hv, wv1); a2 += dot8_bf16_f32(hv, wv2); a3 += dot8_bf16_f32(hv, wv3); } #pragma unroll for (int mask = 16; mask > 0; mask >>= 1) { a0 += __shfl_down_sync(0xffffffff, a0, mask); a1 += __shfl_down_sync(0xffffffff, a1, mask); a2 += __shfl_down_sync(0xffffffff, a2, mask); a3 += __shfl_down_sync(0xffffffff, a3, mask); } if (lane == 0) { float* dst = out_fp32 + (int64_t)t * H + h0; if (h0 + 3 < H) { red_add_v4_f32(dst, a0 * wt, a1 * wt, a2 * wt, a3 * wt); } else { atomicAdd(dst + 0, a0 * wt); if (h0 + 1 < H) atomicAdd(dst + 1, a1 * wt); if (h0 + 2 < H) atomicAdd(dst + 2, a2 * wt); } } } } // ============================================================================ // Token Counting-Sort & M-Tile Table Kernels // ============================================================================ __global__ void __launch_bounds__(512) moe_fused_single_sort_kernel( const int64_t* __restrict__ expert_ids, const __nv_bfloat16* __restrict__ expert_weights, int32_t* __restrict__ sorted_token_ids, float* __restrict__ sorted_weights, TileInfo* __restrict__ tile_map, int32_t* __restrict__ d_num_tiles, int M_tot, int top_k, int E, int BM ) { __shared__ int32_t smem_hist[512]; __shared__ int32_t smem_row_off[513]; __shared__ int32_t smem_tile_off[513]; __shared__ int32_t smem_cur_off[512]; const int tid = threadIdx.x; if (tid < E) smem_hist[tid] = 0; __syncthreads(); int my_e[16]; int my_idx[16]; int n_local = 0; for (int idx = tid; idx < M_tot; idx += 512) { int e = static_cast(expert_ids[idx]); my_e[n_local] = e; my_idx[n_local] = idx; ++n_local; atomicAdd(&smem_hist[e], 1); } __syncthreads(); if (tid < 32) { int r_run = 0; int t_run = 0; for (int base = 0; base < E; base += 32) { int e = base + tid; int c = (e < E) ? smem_hist[e] : 0; int nt = (c + BM - 1) / BM; int r_inc = c; int t_inc = nt; #pragma unroll for (int offset = 1; offset < 32; offset <<= 1) { int nr = __shfl_up_sync(0xffffffff, r_inc, offset); int nti = __shfl_up_sync(0xffffffff, t_inc, offset); if (tid >= offset) { r_inc += nr; t_inc += nti; } } if (e < E) { smem_row_off[e] = r_run + r_inc - c; smem_cur_off[e] = r_run + r_inc - c; smem_tile_off[e] = t_run + t_inc - nt; } r_run += __shfl_sync(0xffffffff, r_inc, 31); t_run += __shfl_sync(0xffffffff, t_inc, 31); } if (tid == 0) { smem_row_off[E] = r_run; smem_tile_off[E] = t_run; *d_num_tiles = t_run; } } __syncthreads(); if (tid < E) { int count = smem_hist[tid]; int r_off = smem_row_off[tid]; int t_off = smem_tile_off[tid]; int nt = smem_tile_off[tid + 1] - t_off; for (int t = 0; t < nt; ++t) { TileInfo ti; ti.expert_id = tid; ti.row_offset = r_off + t * BM; ti.valid_rows = min(BM, count - t * BM); ti.pad = 0; tile_map[t_off + t] = ti; } } for (int i = 0; i < n_local; ++i) { int e = my_e[i]; int idx = my_idx[i]; int pos = atomicAdd(&smem_cur_off[e], 1); sorted_token_ids[pos] = idx / top_k; sorted_weights[pos] = __bfloat162float(expert_weights[idx]); } } __global__ void __launch_bounds__(256) moe_hist_kernel( const int64_t* __restrict__ expert_ids, int32_t* __restrict__ block_hist, int M_tot, int E ) { int32_t* smem_h = reinterpret_cast(g_dyn_smem); const int tid = threadIdx.x; for (int e = tid; e < E; e += 256) smem_h[e] = 0; __syncthreads(); const int n_blks = gridDim.x; const int chunk = (M_tot + n_blks - 1) / n_blks; const int start = blockIdx.x * chunk; const int end = min(M_tot, start + chunk); for (int idx = start + tid; idx < end; idx += 256) { int e = static_cast(expert_ids[idx]); atomicAdd(&smem_h[e], 1); } __syncthreads(); for (int e = tid; e < E; e += 256) { block_hist[blockIdx.x * E + e] = smem_h[e]; } } __global__ void __launch_bounds__(256) moe_scan_tiles_kernel( int32_t* __restrict__ block_hist, int32_t* __restrict__ expert_row_offsets, TileInfo* __restrict__ tile_map, int32_t* __restrict__ d_num_tiles, int sort_blocks, int E, int BM ) { int32_t* dyn_s = reinterpret_cast(g_dyn_smem); int32_t* s_counts = dyn_s; int32_t* s_row_off = dyn_s + E + 1; int32_t* s_tile_off = dyn_s + 2 * (E + 1); const int tid = threadIdx.x; for (int e = tid; e < E; e += 256) { int sum = 0; for (int b = 0; b < sort_blocks; ++b) { int c = block_hist[b * E + e]; block_hist[b * E + e] = sum; sum += c; } s_counts[e] = sum; } __syncthreads(); if (tid < 32) { int r_run = 0; int t_run = 0; for (int base = 0; base < E; base += 32) { int e = base + tid; int c = (e < E) ? s_counts[e] : 0; int nt = (c + BM - 1) / BM; int r_inc = c; int t_inc = nt; #pragma unroll for (int offset = 1; offset < 32; offset <<= 1) { int nr = __shfl_up_sync(0xffffffff, r_inc, offset); int nti = __shfl_up_sync(0xffffffff, t_inc, offset); if (tid >= offset) { r_inc += nr; t_inc += nti; } } if (e < E) { int ro = r_run + r_inc - c; s_row_off[e] = ro; expert_row_offsets[e] = ro; s_tile_off[e] = t_run + t_inc - nt; } r_run += __shfl_sync(0xffffffff, r_inc, 31); t_run += __shfl_sync(0xffffffff, t_inc, 31); } if (tid == 0) { s_row_off[E] = r_run; s_tile_off[E] = t_run; *d_num_tiles = t_run; } } __syncthreads(); for (int e = tid; e < E; e += 256) { int count = s_counts[e]; int r_off = s_row_off[e]; int t_off = s_tile_off[e]; int nt = s_tile_off[e + 1] - t_off; for (int t = 0; t < nt; ++t) { TileInfo ti; ti.expert_id = e; ti.row_offset = r_off + t * BM; ti.valid_rows = min(BM, count - t * BM); ti.pad = 0; tile_map[t_off + t] = ti; } } } __global__ void __launch_bounds__(256) moe_scatter_kernel( const int64_t* __restrict__ expert_ids, const __nv_bfloat16* __restrict__ expert_weights, const int32_t* __restrict__ block_hist, const int32_t* __restrict__ expert_row_offsets, int32_t* __restrict__ sorted_token_ids, float* __restrict__ sorted_weights, int M_tot, int top_k, int E ) { int32_t* smem_off = reinterpret_cast(g_dyn_smem); const int tid = threadIdx.x; for (int e = tid; e < E; e += 256) { smem_off[e] = expert_row_offsets[e] + block_hist[blockIdx.x * E + e]; } __syncthreads(); const int n_blks = gridDim.x; const int chunk = (M_tot + n_blks - 1) / n_blks; const int start = blockIdx.x * chunk; const int end = min(M_tot, start + chunk); for (int idx = start + tid; idx < end; idx += 256) { int e = static_cast(expert_ids[idx]); int pos = atomicAdd(&smem_off[e], 1); sorted_token_ids[pos] = idx / top_k; sorted_weights[pos] = __bfloat162float(expert_weights[idx]); } } // ============================================================================ // Shared Expert Dense TMA + MMA Kernels (Register-Pipelined + L2 Weight Reuse!) // ============================================================================ template __global__ void __launch_bounds__(NUM_WARPS * 32) dense_swiglu_stage1_kernel( const __grid_constant__ CUtensorMap tmap_A, // x: (T, H) const __grid_constant__ CUtensorMap tmap_B, // w1_shared[s]: (2*I, H) - 32 MB fits in 96 MB L2! __nv_bfloat16* __restrict__ h_out, // (T, I) int T, int H, int I ) { constexpr int BN_OUT = 64; constexpr int BK = 64; constexpr int NUM_THREADS = NUM_WARPS * 32; constexpr int WARPS_M = NUM_WARPS / 2; constexpr int WM = BM / WARPS_M; constexpr int M_TILES = WM / 16; // Schedule M-tiles adjacent per N-tile so w1_shared slice is reused across all M-tiles in L2! const int m_base = blockIdx.x * BM; const int n_base = blockIdx.y * BN_OUT; if (m_base >= T || n_base >= I) return; const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int wm_idx = warp_id >> 1; const int wn_idx = warp_id & 1; constexpr uint32_t A_STAGE_BYTES = BM * 128; constexpr uint32_t B_HALF_BYTES = BN_OUT * 128; constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_HALF_BYTES; constexpr uint32_t EPI_BYTES = BM * 72 * sizeof(__nv_bfloat16); constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u; uint8_t* dyn_smem = g_dyn_smem; uint64_t* mbar = reinterpret_cast(dyn_smem + MBAR_OFF); uint32_t smem_base = static_cast(__cvta_generic_to_shared(dyn_smem)); uint32_t sMbar[STAGES]; #pragma unroll for (int s = 0; s < STAGES; ++s) { sMbar[s] = static_cast(__cvta_generic_to_shared(&mbar[s])); } if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1); } __syncthreads(); const uint64_t pol_A = make_policy_evict_last(); const int num_k_tiles = (H + BK - 1) / BK; #pragma unroll for (int kt = 0; kt < STAGES - 1; ++kt) { if (kt < num_k_tiles && tid == 0) { uint32_t s_off = smem_base + kt * STAGE_BYTES; mbar_expect_tx(sMbar[kt], STAGE_BYTES); tma_load_2d_hint(s_off, &tmap_A, sMbar[kt], kt * BK, m_base, pol_A); tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[kt], kt * BK, n_base); tma_load_2d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_B, sMbar[kt], kt * BK, I + n_base); } } float acc_gate[M_TILES][4][4]; float acc_up[M_TILES][4][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) #pragma unroll for (int ni = 0; ni < 4; ++ni) #pragma unroll for (int r = 0; r < 4; ++r) { acc_gate[mi][ni][r] = 0.0f; acc_up[mi][ni][r] = 0.0f; } const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3); const int a_lane_c = (lane >> 4) << 3; const int b_lane_r = (lane & 7) + ((lane >> 4) << 3); const int b_lane_c = ((lane >> 3) & 1) << 3; const int bn0 = wn_idx * 32 + b_lane_r; for (int kt = 0; kt < num_k_tiles; ++kt) { int kt_next = kt + STAGES - 1; if (kt_next < num_k_tiles && tid == 0) { int p_next = kt_next % STAGES; uint32_t s_off = smem_base + p_next * STAGE_BYTES; mbar_expect_tx(sMbar[p_next], STAGE_BYTES); tma_load_2d_hint(s_off, &tmap_A, sMbar[p_next], kt_next * BK, m_base, pol_A); tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[p_next], kt_next * BK, n_base); tma_load_2d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_B, sMbar[p_next], kt_next * BK, I + n_base); } int p_cur = kt % STAGES; if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1); __syncthreads(); uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES; uint32_t sBg_cur = sA_cur + A_STAGE_BYTES; uint32_t sBu_cur = sBg_cur + B_HALF_BYTES; // Register-level double buffer across ks = 0, 16, 32, 48 uint32_t a_reg[2][M_TILES][4]; uint32_t bg0_reg[2][4], bg1_reg[2][4], bu0_reg[2][4], bu1_reg[2][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c)); } ldmatrix_x4(bg0_reg[0], smem_swizzle_128b(sBg_cur, bn0, b_lane_c)); ldmatrix_x4(bg1_reg[0], smem_swizzle_128b(sBg_cur, bn0 + 16, b_lane_c)); ldmatrix_x4(bu0_reg[0], smem_swizzle_128b(sBu_cur, bn0, b_lane_c)); ldmatrix_x4(bu1_reg[0], smem_swizzle_128b(sBu_cur, bn0 + 16, b_lane_c)); #pragma unroll for (int ks_idx = 0; ks_idx < 4; ++ks_idx) { int cur_b = ks_idx & 1; if (ks_idx < 3) { int nxt_b = cur_b ^ 1; int ks_nxt = (ks_idx + 1) * 16; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c)); } ldmatrix_x4(bg0_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0, ks_nxt + b_lane_c)); ldmatrix_x4(bg1_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0 + 16, ks_nxt + b_lane_c)); ldmatrix_x4(bu0_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0, ks_nxt + b_lane_c)); ldmatrix_x4(bu1_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0 + 16, ks_nxt + b_lane_c)); } #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { mma_m16n8k16_bf16(acc_gate[mi][0], a_reg[cur_b][mi], bg0_reg[cur_b][0], bg0_reg[cur_b][1]); mma_m16n8k16_bf16(acc_gate[mi][1], a_reg[cur_b][mi], bg0_reg[cur_b][2], bg0_reg[cur_b][3]); mma_m16n8k16_bf16(acc_gate[mi][2], a_reg[cur_b][mi], bg1_reg[cur_b][0], bg1_reg[cur_b][1]); mma_m16n8k16_bf16(acc_gate[mi][3], a_reg[cur_b][mi], bg1_reg[cur_b][2], bg1_reg[cur_b][3]); mma_m16n8k16_bf16(acc_up[mi][0], a_reg[cur_b][mi], bu0_reg[cur_b][0], bu0_reg[cur_b][1]); mma_m16n8k16_bf16(acc_up[mi][1], a_reg[cur_b][mi], bu0_reg[cur_b][2], bu0_reg[cur_b][3]); mma_m16n8k16_bf16(acc_up[mi][2], a_reg[cur_b][mi], bu1_reg[cur_b][0], bu1_reg[cur_b][1]); mma_m16n8k16_bf16(acc_up[mi][3], a_reg[cur_b][mi], bu1_reg[cur_b][2], bu1_reg[cur_b][3]); } } __syncthreads(); } constexpr int EPI_STRIDE = 72; __nv_bfloat16* smem_epi = reinterpret_cast<__nv_bfloat16*>(dyn_smem); const int gid = lane >> 2; const int tid_g = lane & 3; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r0 = wm_idx * WM + mi * 16 + gid; int r1 = r0 + 8; #pragma unroll for (int ni = 0; ni < 4; ++ni) { int c = wn_idx * 32 + ni * 8 + tid_g * 2; float g00 = acc_gate[mi][ni][0], g01 = acc_gate[mi][ni][1]; float g10 = acc_gate[mi][ni][2], g11 = acc_gate[mi][ni][3]; float u00 = acc_up[mi][ni][0], u01 = acc_up[mi][ni][1]; float u10 = acc_up[mi][ni][2], u11 = acc_up[mi][ni][3]; float h00 = (g00 / (1.0f + __expf(-g00))) * u00; float h01 = (g01 / (1.0f + __expf(-g01))) * u01; float h10 = (g10 / (1.0f + __expf(-g10))) * u10; float h11 = (g11 / (1.0f + __expf(-g11))) * u11; *reinterpret_cast<__nv_bfloat162*>(&smem_epi[r0 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h00, h01); *reinterpret_cast<__nv_bfloat162*>(&smem_epi[r1 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h10, h11); } } __syncthreads(); const int valid_m = min(BM, T - m_base); for (int idx = tid; idx < valid_m * 8; idx += NUM_THREADS) { int r = idx >> 3; int c8 = (idx & 7) << 3; if (n_base + c8 + 7 < I) { uint4 v = *reinterpret_cast(&smem_epi[r * EPI_STRIDE + c8]); *reinterpret_cast(&h_out[(int64_t)(m_base + r) * I + n_base + c8]) = v; } else { for (int j = 0; j < 8 && n_base + c8 + j < I; ++j) { h_out[(int64_t)(m_base + r) * I + n_base + c8 + j] = smem_epi[r * EPI_STRIDE + c8 + j]; } } } } template __global__ void __launch_bounds__(NUM_WARPS * 32) dense_down_stage2_kernel( const __grid_constant__ CUtensorMap tmap_A, // h_shared: (T, I) const __grid_constant__ CUtensorMap tmap_B, // w2_shared[s]: (H, I) - 16 MB fits in 96 MB L2! float* __restrict__ out_fp32, // (T, H) int T, int H, int I ) { constexpr int BN = 128; constexpr int BK = 64; constexpr int WARPS_M = NUM_WARPS / 2; constexpr int WM = BM / WARPS_M; constexpr int M_TILES = WM / 16; const int m_base = blockIdx.x * BM; const int n_base = blockIdx.y * BN; if (m_base >= T || n_base >= H) return; const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int wm_idx = warp_id >> 1; const int wn_idx = warp_id & 1; constexpr uint32_t A_STAGE_BYTES = BM * 128; constexpr uint32_t B_STAGE_BYTES = BN * 128; constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES; constexpr uint32_t EPI_BYTES = BM * 132 * sizeof(float); constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u; uint8_t* dyn_smem = g_dyn_smem; uint64_t* mbar = reinterpret_cast(dyn_smem + MBAR_OFF); uint32_t smem_base = static_cast(__cvta_generic_to_shared(dyn_smem)); uint32_t sMbar[STAGES]; #pragma unroll for (int s = 0; s < STAGES; ++s) { sMbar[s] = static_cast(__cvta_generic_to_shared(&mbar[s])); } if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1); } __syncthreads(); const uint64_t pol_A = make_policy_evict_last(); const int num_k_tiles = (I + BK - 1) / BK; #pragma unroll for (int kt = 0; kt < STAGES - 1; ++kt) { if (kt < num_k_tiles && tid == 0) { uint32_t s_off = smem_base + kt * STAGE_BYTES; mbar_expect_tx(sMbar[kt], STAGE_BYTES); tma_load_2d_hint(s_off, &tmap_A, sMbar[kt], kt * BK, m_base, pol_A); tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[kt], kt * BK, n_base); } } float acc[M_TILES][8][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) #pragma unroll for (int ni = 0; ni < 8; ++ni) #pragma unroll for (int r = 0; r < 4; ++r) acc[mi][ni][r] = 0.0f; const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3); const int a_lane_c = (lane >> 4) << 3; const int b_lane_r = (lane & 7) + ((lane >> 4) << 3); const int b_lane_c = ((lane >> 3) & 1) << 3; for (int kt = 0; kt < num_k_tiles; ++kt) { int kt_next = kt + STAGES - 1; if (kt_next < num_k_tiles && tid == 0) { int p_next = kt_next % STAGES; uint32_t s_off = smem_base + p_next * STAGE_BYTES; mbar_expect_tx(sMbar[p_next], STAGE_BYTES); tma_load_2d_hint(s_off, &tmap_A, sMbar[p_next], kt_next * BK, m_base, pol_A); tma_load_2d_lru(s_off + A_STAGE_BYTES, &tmap_B, sMbar[p_next], kt_next * BK, n_base); } int p_cur = kt % STAGES; if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1); __syncthreads(); uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES; uint32_t sB_cur = sA_cur + A_STAGE_BYTES; uint32_t a_reg[2][M_TILES][4]; uint32_t b_reg[2][4][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c)); } #pragma unroll for (int np = 0; np < 4; ++np) { ldmatrix_x4(b_reg[0][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, b_lane_c)); } #pragma unroll for (int ks_idx = 0; ks_idx < 4; ++ks_idx) { int cur_b = ks_idx & 1; if (ks_idx < 3) { int nxt_b = cur_b ^ 1; int ks_nxt = (ks_idx + 1) * 16; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c)); } #pragma unroll for (int np = 0; np < 4; ++np) { ldmatrix_x4(b_reg[nxt_b][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, ks_nxt + b_lane_c)); } } #pragma unroll for (int np = 0; np < 4; ++np) { #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { mma_m16n8k16_bf16(acc[mi][np * 2 + 0], a_reg[cur_b][mi], b_reg[cur_b][np][0], b_reg[cur_b][np][1]); mma_m16n8k16_bf16(acc[mi][np * 2 + 1], a_reg[cur_b][mi], b_reg[cur_b][np][2], b_reg[cur_b][np][3]); } } } __syncthreads(); } constexpr int EPI_STRIDE = 132; float* smem_epi = reinterpret_cast(dyn_smem); const int gid = lane >> 2; const int tid_g = lane & 3; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r0 = wm_idx * WM + mi * 16 + gid; int r1 = r0 + 8; #pragma unroll for (int ni = 0; ni < 8; ++ni) { int c = wn_idx * 64 + ni * 8 + tid_g * 2; *reinterpret_cast(&smem_epi[r0 * EPI_STRIDE + c]) = make_float2(acc[mi][ni][0], acc[mi][ni][1]); *reinterpret_cast(&smem_epi[r1 * EPI_STRIDE + c]) = make_float2(acc[mi][ni][2], acc[mi][ni][3]); } } __syncthreads(); const int valid_m = min(BM, T - m_base); for (int m = warp_id; m < valid_m; m += NUM_WARPS) { float* dst_row = out_fp32 + (int64_t)(m_base + m) * H + n_base; #pragma unroll for (int c = lane * 4; c < BN; c += 128) { if (n_base + c + 3 < H) { float4 v = *reinterpret_cast(&smem_epi[m * EPI_STRIDE + c]); if constexpr (ACCUMULATE) { float4 prev = *reinterpret_cast(dst_row + c); v.x += prev.x; v.y += prev.y; v.z += prev.z; v.w += prev.w; } *reinterpret_cast(dst_row + c) = v; } else { for (int j = 0; j < 4 && n_base + c + j < H; ++j) { if constexpr (ACCUMULATE) dst_row[c + j] += smem_epi[m * EPI_STRIDE + c + j]; else dst_row[c + j] = smem_epi[m * EPI_STRIDE + c + j]; } } } } } // ============================================================================ // Routed Experts Grouped GEMM Stage 1 (Hole-Free Swizzled + Reg-Pipelined!) // ============================================================================ template __global__ void __launch_bounds__(NUM_WARPS * 32) routed_swiglu_stage1_kernel( const __nv_bfloat16* __restrict__ x, // (T, H) const __grid_constant__ CUtensorMap tmap_w1, // w1_routed: (E, 2*I, H) const TileInfo* __restrict__ tile_map, const int32_t* __restrict__ d_num_tiles, const int32_t* __restrict__ sorted_token_ids, __nv_bfloat16* __restrict__ h_interm, // (M_tot, I) int H, int I, int n_tiles ) { constexpr int BN_OUT = 64; constexpr int BK = 64; constexpr int NUM_THREADS = NUM_WARPS * 32; constexpr int WARPS_M = NUM_WARPS / 2; constexpr int WM = BM / WARPS_M; constexpr int M_TILES = WM / 16; const int total_m = *d_num_tiles; const int job = blockIdx.x; if (job >= total_m * n_tiles) return; int m_tile, n_tile; if constexpr (GROUP_N > 1) { const int span = total_m * GROUP_N; const int ng = job / span; const int rem = job - ng * span; const int g_eff = min(GROUP_N, n_tiles - ng * GROUP_N); m_tile = rem / g_eff; n_tile = ng * GROUP_N + (rem - m_tile * g_eff); if (m_tile >= total_m) return; } else { n_tile = job / total_m; m_tile = job - n_tile * total_m; } const int n_base = n_tile * BN_OUT; const TileInfo ti = tile_map[m_tile]; const int expert_id = ti.expert_id; const int row_offset = ti.row_offset; const int valid_rows = ti.valid_rows; const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int wm_idx = warp_id >> 1; const int wn_idx = warp_id & 1; constexpr uint32_t A_STAGE_BYTES = BM * 128; constexpr uint32_t B_HALF_BYTES = BN_OUT * 128; constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + 2 * B_HALF_BYTES; constexpr uint32_t EPI_BYTES = BM * 72 * sizeof(__nv_bfloat16); constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u; uint8_t* dyn_smem = g_dyn_smem; uint64_t* mbar_B = reinterpret_cast(dyn_smem + MBAR_OFF); uint32_t smem_base = static_cast(__cvta_generic_to_shared(dyn_smem)); uint32_t sMbar[STAGES]; #pragma unroll for (int s = 0; s < STAGES; ++s) { sMbar[s] = static_cast(__cvta_generic_to_shared(&mbar_B[s])); } if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1); } constexpr int A_CHUNKS_PER_THREAD = (BM * 8) / NUM_THREADS; const __nv_bfloat16* my_a_ptr[A_CHUNKS_PER_THREAD]; int my_a_bytes[A_CHUNKS_PER_THREAD]; uint32_t my_a_smem_off[A_CHUNKS_PER_THREAD]; #pragma unroll for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) { int idx = tid + c * NUM_THREADS; int r = idx >> 3; int col8 = (idx & 7) << 3; my_a_smem_off[c] = smem_swizzle_128b(0, r, col8); if (r < valid_rows) { int32_t tok = sorted_token_ids[row_offset + r]; my_a_ptr[c] = x + (int64_t)tok * H + col8; my_a_bytes[c] = 16; } else { my_a_ptr[c] = x; my_a_bytes[c] = 0; } } __syncthreads(); const uint64_t pol_B = STREAM_B ? make_policy_evict_first() : 0ULL; const int num_k_tiles = (H + BK - 1) / BK; #pragma unroll for (int kt = 0; kt < STAGES - 1; ++kt) { if (kt < num_k_tiles) { uint32_t s_off = smem_base + kt * STAGE_BYTES; if (tid == 0) { mbar_expect_tx(sMbar[kt], 2 * B_HALF_BYTES); if constexpr (STREAM_B) { tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[kt], kt * BK, n_base, expert_id, pol_B); tma_load_3d_hint(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[kt], kt * BK, I + n_base, expert_id, pol_B); } else { tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[kt], kt * BK, n_base, expert_id); tma_load_3d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[kt], kt * BK, I + n_base, expert_id); } } #pragma unroll for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) { cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt * BK, my_a_bytes[c]); } } asm volatile("cp.async.commit_group;\n"); } float acc_gate[M_TILES][4][4]; float acc_up[M_TILES][4][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) #pragma unroll for (int ni = 0; ni < 4; ++ni) #pragma unroll for (int r = 0; r < 4; ++r) { acc_gate[mi][ni][r] = 0.0f; acc_up[mi][ni][r] = 0.0f; } const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3); const int a_lane_c = (lane >> 4) << 3; const int b_lane_r = (lane & 7) + ((lane >> 4) << 3); const int b_lane_c = ((lane >> 3) & 1) << 3; const int bn0 = wn_idx * 32 + b_lane_r; for (int kt = 0; kt < num_k_tiles; ++kt) { int kt_next = kt + STAGES - 1; if (kt_next < num_k_tiles) { int p_next = kt_next % STAGES; uint32_t s_off = smem_base + p_next * STAGE_BYTES; if (tid == 0) { mbar_expect_tx(sMbar[p_next], 2 * B_HALF_BYTES); if constexpr (STREAM_B) { tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, n_base, expert_id, pol_B); tma_load_3d_hint(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, I + n_base, expert_id, pol_B); } else { tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, n_base, expert_id); tma_load_3d_lru(s_off + A_STAGE_BYTES + B_HALF_BYTES, &tmap_w1, sMbar[p_next], kt_next * BK, I + n_base, expert_id); } } #pragma unroll for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) { cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt_next * BK, my_a_bytes[c]); } } asm volatile("cp.async.commit_group;\n"); asm volatile("cp.async.wait_group %0;\n" :: "n"(STAGES - 1)); int p_cur = kt % STAGES; if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1); __syncthreads(); uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES; uint32_t sBg_cur = sA_cur + A_STAGE_BYTES; uint32_t sBu_cur = sBg_cur + B_HALF_BYTES; uint32_t a_reg[2][M_TILES][4]; uint32_t bg0_reg[2][4], bg1_reg[2][4], bu0_reg[2][4], bu1_reg[2][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c)); } ldmatrix_x4(bg0_reg[0], smem_swizzle_128b(sBg_cur, bn0, b_lane_c)); ldmatrix_x4(bg1_reg[0], smem_swizzle_128b(sBg_cur, bn0 + 16, b_lane_c)); ldmatrix_x4(bu0_reg[0], smem_swizzle_128b(sBu_cur, bn0, b_lane_c)); ldmatrix_x4(bu1_reg[0], smem_swizzle_128b(sBu_cur, bn0 + 16, b_lane_c)); #pragma unroll for (int ks_idx = 0; ks_idx < 4; ++ks_idx) { int cur_b = ks_idx & 1; if (ks_idx < 3) { int nxt_b = cur_b ^ 1; int ks_nxt = (ks_idx + 1) * 16; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c)); } ldmatrix_x4(bg0_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0, ks_nxt + b_lane_c)); ldmatrix_x4(bg1_reg[nxt_b], smem_swizzle_128b(sBg_cur, bn0 + 16, ks_nxt + b_lane_c)); ldmatrix_x4(bu0_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0, ks_nxt + b_lane_c)); ldmatrix_x4(bu1_reg[nxt_b], smem_swizzle_128b(sBu_cur, bn0 + 16, ks_nxt + b_lane_c)); } #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { mma_m16n8k16_bf16(acc_gate[mi][0], a_reg[cur_b][mi], bg0_reg[cur_b][0], bg0_reg[cur_b][1]); mma_m16n8k16_bf16(acc_gate[mi][1], a_reg[cur_b][mi], bg0_reg[cur_b][2], bg0_reg[cur_b][3]); mma_m16n8k16_bf16(acc_gate[mi][2], a_reg[cur_b][mi], bg1_reg[cur_b][0], bg1_reg[cur_b][1]); mma_m16n8k16_bf16(acc_gate[mi][3], a_reg[cur_b][mi], bg1_reg[cur_b][2], bg1_reg[cur_b][3]); mma_m16n8k16_bf16(acc_up[mi][0], a_reg[cur_b][mi], bu0_reg[cur_b][0], bu0_reg[cur_b][1]); mma_m16n8k16_bf16(acc_up[mi][1], a_reg[cur_b][mi], bu0_reg[cur_b][2], bu0_reg[cur_b][3]); mma_m16n8k16_bf16(acc_up[mi][2], a_reg[cur_b][mi], bu1_reg[cur_b][0], bu1_reg[cur_b][1]); mma_m16n8k16_bf16(acc_up[mi][3], a_reg[cur_b][mi], bu1_reg[cur_b][2], bu1_reg[cur_b][3]); } } __syncthreads(); } asm volatile("cp.async.wait_group 0;\n"); constexpr int EPI_STRIDE = 72; __nv_bfloat16* smem_epi = reinterpret_cast<__nv_bfloat16*>(dyn_smem); const int gid = lane >> 2; const int tid_g = lane & 3; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r0 = wm_idx * WM + mi * 16 + gid; int r1 = r0 + 8; #pragma unroll for (int ni = 0; ni < 4; ++ni) { int c = wn_idx * 32 + ni * 8 + tid_g * 2; float g00 = acc_gate[mi][ni][0], g01 = acc_gate[mi][ni][1]; float g10 = acc_gate[mi][ni][2], g11 = acc_gate[mi][ni][3]; float u00 = acc_up[mi][ni][0], u01 = acc_up[mi][ni][1]; float u10 = acc_up[mi][ni][2], u11 = acc_up[mi][ni][3]; float h00 = (g00 / (1.0f + __expf(-g00))) * u00; float h01 = (g01 / (1.0f + __expf(-g01))) * u01; float h10 = (g10 / (1.0f + __expf(-g10))) * u10; float h11 = (g11 / (1.0f + __expf(-g11))) * u11; *reinterpret_cast<__nv_bfloat162*>(&smem_epi[r0 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h00, h01); *reinterpret_cast<__nv_bfloat162*>(&smem_epi[r1 * EPI_STRIDE + c]) = __floats2bfloat162_rn(h10, h11); } } __syncthreads(); for (int idx = tid; idx < valid_rows * 8; idx += NUM_THREADS) { int r = idx >> 3; int c8 = (idx & 7) << 3; if (n_base + c8 + 7 < I) { uint4 v = *reinterpret_cast(&smem_epi[r * EPI_STRIDE + c8]); *reinterpret_cast(&h_interm[(int64_t)(row_offset + r) * I + n_base + c8]) = v; } else { for (int j = 0; j < 8 && n_base + c8 + j < I; ++j) { h_interm[(int64_t)(row_offset + r) * I + n_base + c8 + j] = smem_epi[r * EPI_STRIDE + c8 + j]; } } } } // ============================================================================ // Routed Experts Grouped GEMM Stage 2 (N-Banded L2-Resident AtomicAdd!) // ============================================================================ template __global__ void __launch_bounds__(NUM_WARPS * 32) routed_down_stage2_kernel( const __nv_bfloat16* __restrict__ h_interm, // (M_tot, I) const __grid_constant__ CUtensorMap tmap_w2, // w2_routed: (E, H, I) const TileInfo* __restrict__ tile_map, const int32_t* __restrict__ d_num_tiles, const int32_t* __restrict__ sorted_token_ids, const float* __restrict__ sorted_weights, float* __restrict__ out_fp32, // (T, H) int H, int I, int n_tiles ) { constexpr int BN = 128; constexpr int BK = 64; constexpr int NUM_THREADS = NUM_WARPS * 32; constexpr int WARPS_M = NUM_WARPS / 2; constexpr int WM = BM / WARPS_M; constexpr int M_TILES = WM / 16; const int total_m = *d_num_tiles; const int job = blockIdx.x; if (job >= total_m * n_tiles) return; int m_tile, n_tile; if constexpr (GROUP_N > 1) { const int span = total_m * GROUP_N; const int ng = job / span; const int rem = job - ng * span; const int g_eff = min(GROUP_N, n_tiles - ng * GROUP_N); m_tile = rem / g_eff; n_tile = ng * GROUP_N + (rem - m_tile * g_eff); if (m_tile >= total_m) return; } else { n_tile = job / total_m; m_tile = job - n_tile * total_m; } const int n_base = n_tile * BN; const TileInfo ti = tile_map[m_tile]; const int expert_id = ti.expert_id; const int row_offset = ti.row_offset; const int valid_rows = ti.valid_rows; constexpr uint32_t A_STAGE_BYTES = BM * 128; constexpr uint32_t B_STAGE_BYTES = BN * 128; constexpr uint32_t STAGE_BYTES = A_STAGE_BYTES + B_STAGE_BYTES; constexpr uint32_t EPI_BYTES = BM * 132 * sizeof(float); constexpr uint32_t MBAR_OFF = ((STAGES * STAGE_BYTES > EPI_BYTES ? STAGES * STAGE_BYTES : EPI_BYTES) + 127u) & ~127u; uint8_t* dyn_smem = g_dyn_smem; uint64_t* mbar_B = reinterpret_cast(dyn_smem + MBAR_OFF); const int tid = threadIdx.x; const int warp_id = tid >> 5; const int lane = tid & 31; const int wm_idx = warp_id >> 1; const int wn_idx = warp_id & 1; uint32_t smem_base = static_cast(__cvta_generic_to_shared(dyn_smem)); uint32_t sMbar[STAGES]; #pragma unroll for (int s = 0; s < STAGES; ++s) { sMbar[s] = static_cast(__cvta_generic_to_shared(&mbar_B[s])); } if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbar_init(sMbar[s], 1); } constexpr int A_CHUNKS_PER_THREAD = (BM * 8) / NUM_THREADS; const __nv_bfloat16* my_a_ptr[A_CHUNKS_PER_THREAD]; int my_a_bytes[A_CHUNKS_PER_THREAD]; uint32_t my_a_smem_off[A_CHUNKS_PER_THREAD]; #pragma unroll for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) { int idx = tid + c * NUM_THREADS; int r = idx >> 3; int col8 = (idx & 7) << 3; my_a_smem_off[c] = smem_swizzle_128b(0, r, col8); if (r < valid_rows) { my_a_ptr[c] = h_interm + (int64_t)(row_offset + r) * I + col8; my_a_bytes[c] = 16; } else { my_a_ptr[c] = h_interm; my_a_bytes[c] = 0; } } __syncthreads(); const uint64_t pol_B = STREAM_B ? make_policy_evict_first() : 0ULL; const int num_k_tiles = (I + BK - 1) / BK; #pragma unroll for (int kt = 0; kt < STAGES - 1; ++kt) { if (kt < num_k_tiles) { uint32_t s_off = smem_base + kt * STAGE_BYTES; if (tid == 0) { mbar_expect_tx(sMbar[kt], B_STAGE_BYTES); if constexpr (STREAM_B) { tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[kt], kt * BK, n_base, expert_id, pol_B); } else { tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[kt], kt * BK, n_base, expert_id); } } #pragma unroll for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) { cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt * BK, my_a_bytes[c]); } } asm volatile("cp.async.commit_group;\n"); } float acc[M_TILES][8][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) #pragma unroll for (int ni = 0; ni < 8; ++ni) #pragma unroll for (int r = 0; r < 4; ++r) acc[mi][ni][r] = 0.0f; const int a_lane_r = (lane & 7) + (((lane >> 3) & 1) << 3); const int a_lane_c = (lane >> 4) << 3; const int b_lane_r = (lane & 7) + ((lane >> 4) << 3); const int b_lane_c = ((lane >> 3) & 1) << 3; for (int kt = 0; kt < num_k_tiles; ++kt) { int kt_next = kt + STAGES - 1; if (kt_next < num_k_tiles) { int p_next = kt_next % STAGES; uint32_t s_off = smem_base + p_next * STAGE_BYTES; if (tid == 0) { mbar_expect_tx(sMbar[p_next], B_STAGE_BYTES); if constexpr (STREAM_B) { tma_load_3d_hint(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[p_next], kt_next * BK, n_base, expert_id, pol_B); } else { tma_load_3d_lru(s_off + A_STAGE_BYTES, &tmap_w2, sMbar[p_next], kt_next * BK, n_base, expert_id); } } #pragma unroll for (int c = 0; c < A_CHUNKS_PER_THREAD; ++c) { cp_async_16b_pred(s_off + my_a_smem_off[c], my_a_ptr[c] + kt_next * BK, my_a_bytes[c]); } } asm volatile("cp.async.commit_group;\n"); asm volatile("cp.async.wait_group %0;\n" :: "n"(STAGES - 1)); int p_cur = kt % STAGES; if (lane == 0) mbar_wait(sMbar[p_cur], (kt / STAGES) & 1); __syncthreads(); uint32_t sA_cur = smem_base + p_cur * STAGE_BYTES; uint32_t sB_cur = sA_cur + A_STAGE_BYTES; uint32_t a_reg[2][M_TILES][4]; uint32_t b_reg[2][4][4]; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[0][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, a_lane_c)); } #pragma unroll for (int np = 0; np < 4; ++np) { ldmatrix_x4(b_reg[0][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, b_lane_c)); } #pragma unroll for (int ks_idx = 0; ks_idx < 4; ++ks_idx) { int cur_b = ks_idx & 1; if (ks_idx < 3) { int nxt_b = cur_b ^ 1; int ks_nxt = (ks_idx + 1) * 16; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { ldmatrix_x4(a_reg[nxt_b][mi], smem_swizzle_128b(sA_cur, wm_idx * WM + mi * 16 + a_lane_r, ks_nxt + a_lane_c)); } #pragma unroll for (int np = 0; np < 4; ++np) { ldmatrix_x4(b_reg[nxt_b][np], smem_swizzle_128b(sB_cur, wn_idx * 64 + np * 16 + b_lane_r, ks_nxt + b_lane_c)); } } #pragma unroll for (int np = 0; np < 4; ++np) { #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { mma_m16n8k16_bf16(acc[mi][np * 2 + 0], a_reg[cur_b][mi], b_reg[cur_b][np][0], b_reg[cur_b][np][1]); mma_m16n8k16_bf16(acc[mi][np * 2 + 1], a_reg[cur_b][mi], b_reg[cur_b][np][2], b_reg[cur_b][np][3]); } } } __syncthreads(); } asm volatile("cp.async.wait_group 0;\n"); // Store tok/wt right inside the already-free SMEM above EPI_BYTES! constexpr int EPI_STRIDE = 132; float* smem_epi = reinterpret_cast(dyn_smem); int32_t* smem_tok = reinterpret_cast(dyn_smem + EPI_BYTES); float* smem_wt = reinterpret_cast(dyn_smem + EPI_BYTES + BM * sizeof(int32_t)); if (tid < BM) { if (tid < valid_rows) { smem_tok[tid] = sorted_token_ids[row_offset + tid]; smem_wt[tid] = sorted_weights[row_offset + tid]; } else { smem_tok[tid] = 0; smem_wt[tid] = 0.0f; } } __syncthreads(); const int gid = lane >> 2; const int tid_g = lane & 3; #pragma unroll for (int mi = 0; mi < M_TILES; ++mi) { int r0 = wm_idx * WM + mi * 16 + gid; int r1 = r0 + 8; float wt0 = smem_wt[r0]; float wt1 = smem_wt[r1]; #pragma unroll for (int ni = 0; ni < 8; ++ni) { int c = wn_idx * 64 + ni * 8 + tid_g * 2; *reinterpret_cast(&smem_epi[r0 * EPI_STRIDE + c]) = make_float2(acc[mi][ni][0] * wt0, acc[mi][ni][1] * wt0); *reinterpret_cast(&smem_epi[r1 * EPI_STRIDE + c]) = make_float2(acc[mi][ni][2] * wt1, acc[mi][ni][3] * wt1); } } __syncthreads(); for (int m = warp_id; m < valid_rows; m += NUM_WARPS) { int32_t tok = smem_tok[m]; float* dst_row = out_fp32 + (int64_t)tok * H + n_base; #pragma unroll for (int c = lane * 4; c < BN; c += 128) { if (n_base + c + 3 < H) { float4 v = *reinterpret_cast(&smem_epi[m * EPI_STRIDE + c]); red_add_v4_f32(dst_row + c, v.x, v.y, v.z, v.w); } else { for (int j = 0; j < 4 && n_base + c + j < H; ++j) { atomicAdd(dst_row + c + j, smem_epi[m * EPI_STRIDE + c + j]); } } } } } // ============================================================================ // Vectorized FP32 -> BF16 Final Cast Kernel // ============================================================================ __global__ void __launch_bounds__(256) cast_fp32_to_bf16_kernel( const float* __restrict__ src, __nv_bfloat16* __restrict__ dst, int64_t n_elems ) { int64_t idx8 = ((int64_t)blockIdx.x * 256 + threadIdx.x) << 3; if (idx8 + 7 < n_elems) { float4 v0 = *reinterpret_cast(src + idx8); float4 v1 = *reinterpret_cast(src + idx8 + 4); __nv_bfloat162 b0 = __floats2bfloat162_rn(v0.x, v0.y); __nv_bfloat162 b1 = __floats2bfloat162_rn(v0.z, v0.w); __nv_bfloat162 b2 = __floats2bfloat162_rn(v1.x, v1.y); __nv_bfloat162 b3 = __floats2bfloat162_rn(v1.z, v1.w); uint4 out_u4; out_u4.x = *reinterpret_cast(&b0); out_u4.y = *reinterpret_cast(&b1); out_u4.z = *reinterpret_cast(&b2); out_u4.w = *reinterpret_cast(&b3); *reinterpret_cast(dst + idx8) = out_u4; } else { for (int64_t i = idx8; i < n_elems; ++i) { dst[i] = __float2bfloat16_rn(src[i]); } } } // ============================================================================ // Host TMA Descriptor Builders & Kernel Launchers // ============================================================================ inline CUtensorMap make_tmap_2d_bf16( const void* ptr, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols ) { CUtensorMap tmap{}; cuuint64_t globalDim[2] = {cols, rows}; cuuint64_t globalStrides[1] = {cols * sizeof(__nv_bfloat16)}; cuuint32_t boxDim[2] = {box_cols, box_rows}; cuuint32_t elemStrides[2] = {1, 1}; CUresult res = cuTensorMapEncodeTiled( &tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 2, const_cast(ptr), globalDim, globalStrides, boxDim, elemStrides, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE ); TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 2D failed: ", (int)res); return tmap; } inline CUtensorMap make_tmap_3d_bf16( const void* ptr, uint64_t batches, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols ) { CUtensorMap tmap{}; cuuint64_t globalDim[3] = {cols, rows, batches}; cuuint64_t globalStrides[2] = {cols * sizeof(__nv_bfloat16), rows * cols * sizeof(__nv_bfloat16)}; cuuint32_t boxDim[3] = {box_cols, box_rows, 1}; cuuint32_t elemStrides[3] = {1, 1, 1}; CUresult res = cuTensorMapEncodeTiled( &tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3, const_cast(ptr), globalDim, globalStrides, boxDim, elemStrides, CU_TENSOR_MAP_INTERLEAVE_NONE, CU_TENSOR_MAP_SWIZZLE_128B, CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE ); TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 3D failed: ", (int)res); return tmap; } template inline void ensure_dyn_smem(KernelT kernel, int smem_bytes) { if (smem_bytes > 48 * 1024) { cudaFuncSetAttribute(kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_bytes); } } template void launch_routed_pipeline( const __nv_bfloat16* x_ptr, const __nv_bfloat16* w1_ptr, const __nv_bfloat16* w2_ptr, const TileInfo* tile_map_ptr, const int32_t* d_num_tiles_ptr, const int32_t* sorted_tok_ptr, const float* sorted_wt_ptr, __nv_bfloat16* h_routed_ptr, float* out_fp32_ptr, int E, int H, int I, int max_m_tiles, cudaStream_t stream ) { CUtensorMap tmap_w1 = make_tmap_3d_bf16(w1_ptr, E, 2 * I, H, 64, 64); CUtensorMap tmap_w2 = make_tmap_3d_bf16(w2_ptr, E, H, I, 128, 64); constexpr int s1_stage_bytes = STAGES * (BM * 128 + 2 * 64 * 128); constexpr int s1_epi_bytes = BM * 72 * sizeof(__nv_bfloat16); constexpr int s1_main = (((s1_stage_bytes > s1_epi_bytes) ? s1_stage_bytes : s1_epi_bytes) + 127) & ~127; constexpr int s1_smem = s1_main + 64; constexpr int s2_stage_bytes = STAGES * (BM * 128 + 128 * 128); constexpr int s2_epi_bytes = BM * 132 * sizeof(float) + BM * 8; constexpr int s2_main = (((s2_stage_bytes > s2_epi_bytes) ? s2_stage_bytes : s2_epi_bytes) + 127) & ~127; constexpr int s2_smem = s2_main + 64; auto k1 = routed_swiglu_stage1_kernel; auto k2 = routed_down_stage2_kernel; static bool configured = false; if (!configured) { ensure_dyn_smem(k1, s1_smem); ensure_dyn_smem(k2, s2_smem); configured = true; } int n_tiles_s1 = (I + 63) / 64; k1<<>>( x_ptr, tmap_w1, tile_map_ptr, d_num_tiles_ptr, sorted_tok_ptr, h_routed_ptr, H, I, n_tiles_s1 ); int n_tiles_s2 = (H + 127) / 128; k2<<>>( h_routed_ptr, tmap_w2, tile_map_ptr, d_num_tiles_ptr, sorted_tok_ptr, sorted_wt_ptr, out_fp32_ptr, H, I, n_tiles_s2 ); } } // namespace torch::Tensor glm52_fused_moe_forward( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared ) { if (__builtin_expect(!x.is_contiguous(), 0)) x = x.contiguous(); if (__builtin_expect(!expert_ids.is_contiguous(), 0)) expert_ids = expert_ids.contiguous(); if (__builtin_expect(!expert_weights.is_contiguous(), 0)) expert_weights = expert_weights.contiguous(); const int T = static_cast(x.size(0)); const int H = static_cast(x.size(1)); const int top_k = static_cast(expert_ids.size(1)); const int E = static_cast(w1_routed.size(0)); const int I = static_cast(w2_routed.size(2)); const int n_shared = static_cast(w1_shared.size(0)); auto out = torch::empty({T, H}, x.options()); if (__builtin_expect(T == 0, 0)) return out; cudaStream_t stream = at::cuda::getCurrentCUDAStream(); // Wave-Balanced Persistent Lockless Decode Path (T <= 2) if (T <= 2 && (H & 7) == 0 && (I & 7) == 0 && (top_k + n_shared) > 0) { const int slots_per_tok = top_k + n_shared; const int total_pairs = T * slots_per_tok; g_ws.ensure(total_pairs, T, E, H, I, 32, 1, x.device()); const auto* x_ptr = reinterpret_cast(x.data_ptr()); const auto* ids_ptr = expert_ids.data_ptr(); const auto* wts_ptr = reinterpret_cast(expert_weights.data_ptr()); const auto* w1r_ptr = reinterpret_cast(w1_routed.data_ptr()); const auto* w2r_ptr = reinterpret_cast(w2_routed.data_ptr()); const auto* w1s_ptr = n_shared > 0 ? reinterpret_cast(w1_shared.data_ptr()) : nullptr; const auto* w2s_ptr = n_shared > 0 ? reinterpret_cast(w2_shared.data_ptr()) : nullptr; auto* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()); // 940 CTAs = 5 * 188 SMs (exact uniform wave balance across all 188 SMs!) decode_stage1_persistent_kernel<<<940, 128, 0, stream>>>( x_ptr, ids_ptr, w1r_ptr, w1s_ptr, g_ws.h_routed_ptr, g_ws.out_fp32_ptr, T, top_k, n_shared, H, I ); decode_stage2_persistent_kernel<<<940, 64, 0, stream>>>( g_ws.h_routed_ptr, ids_ptr, wts_ptr, w2r_ptr, w2s_ptr, g_ws.out_fp32_ptr, T, top_k, n_shared, H, I ); int64_t total_elems = (int64_t)T * H; int cast_blocks = static_cast((total_elems + 2047) / 2048); cast_fp32_to_bf16_kernel<<>>(g_ws.out_fp32_ptr, out_bf16_ptr, total_elems); return out; } // General Tensor Core TMA + MMA Path (all T) const int M_tot = T * top_k; const int avg_m = M_tot / std::max(E, 1); const int BM_routed = (avg_m <= 20) ? 32 : ((avg_m <= 56) ? 64 : 128); const int max_m_tiles = (M_tot + BM_routed - 1) / BM_routed + E; const int sort_blocks = std::min(64, std::max(1, (M_tot + 255) / 256)); g_ws.ensure(M_tot, T, E, H, I, max_m_tiles, sort_blocks, x.device()); const auto* x_ptr = reinterpret_cast(x.data_ptr()); const auto* ids_ptr = expert_ids.data_ptr(); const auto* wts_ptr = reinterpret_cast(expert_weights.data_ptr()); const auto* w1r_ptr = reinterpret_cast(w1_routed.data_ptr()); const auto* w2r_ptr = reinterpret_cast(w2_routed.data_ptr()); auto* out_bf16_ptr = reinterpret_cast<__nv_bfloat16*>(out.data_ptr()); // 1. Sort routed tokens & construct tile map if (M_tot > 0) { if (M_tot <= 8192 && E <= 512) { moe_fused_single_sort_kernel<<<1, 512, 0, stream>>>( ids_ptr, wts_ptr, g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr, M_tot, top_k, E, BM_routed ); } else { int hist_smem = E * sizeof(int32_t); moe_hist_kernel<<>>(ids_ptr, g_ws.block_hist_ptr, M_tot, E); int scan_smem = 3 * (E + 1) * sizeof(int32_t); moe_scan_tiles_kernel<<<1, 256, scan_smem, stream>>>( g_ws.block_hist_ptr, g_ws.expert_off_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr, sort_blocks, E, BM_routed ); moe_scatter_kernel<<>>( ids_ptr, wts_ptr, g_ws.block_hist_ptr, g_ws.expert_off_ptr, g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr, M_tot, top_k, E ); } } // 2. Shared experts (writes out_fp32 directly for s=0, eliminating memset!) if (n_shared == 0) { cudaMemsetAsync(g_ws.out_fp32_ptr, 0, (size_t)T * H * sizeof(float), stream); } else { const auto* w1s_all = reinterpret_cast(w1_shared.data_ptr()); const auto* w2s_all = reinterpret_cast(w2_shared.data_ptr()); constexpr int BM_S = 128; constexpr int ST_S = 3; constexpr int NW_S = 8; constexpr int smem_ds1 = ST_S * (BM_S * 128 + 2 * 64 * 128) + 128; constexpr int smem_ds2 = ST_S * (BM_S * 128 + 128 * 128) + 128; static bool shared_cfg = false; if (!shared_cfg) { ensure_dyn_smem(dense_swiglu_stage1_kernel, smem_ds1); ensure_dyn_smem(dense_down_stage2_kernel, smem_ds2); ensure_dyn_smem(dense_down_stage2_kernel, smem_ds2); shared_cfg = true; } CUtensorMap tmap_x = make_tmap_2d_bf16(x_ptr, T, H, BM_S, 64); CUtensorMap tmap_hs = make_tmap_2d_bf16(g_ws.h_shared_ptr, T, I, BM_S, 64); // blockIdx.x = M-tile (varies fastest -> w1_shared/w2_shared N-slice stays hot in L2 across all M-tiles!) dim3 grid_ds1((T + BM_S - 1) / BM_S, (I + 63) / 64); dim3 grid_ds2((T + BM_S - 1) / BM_S, (H + 127) / 128); for (int s = 0; s < n_shared; ++s) { const __nv_bfloat16* w1s = w1s_all + (int64_t)s * (2LL * I * H); const __nv_bfloat16* w2s = w2s_all + (int64_t)s * ((int64_t)H * I); CUtensorMap tmap_w1s = make_tmap_2d_bf16(w1s, 2 * I, H, 64, 64); CUtensorMap tmap_w2s = make_tmap_2d_bf16(w2s, H, I, 128, 64); dense_swiglu_stage1_kernel<<>>( tmap_x, tmap_w1s, g_ws.h_shared_ptr, T, H, I ); if (s == 0) { dense_down_stage2_kernel<<>>( tmap_hs, tmap_w2s, g_ws.out_fp32_ptr, T, H, I ); } else { dense_down_stage2_kernel<<>>( tmap_hs, tmap_w2s, g_ws.out_fp32_ptr, T, H, I ); } } } // 3. Routed experts Grouped GEMM (Stage 1 + Stage 2) if (M_tot > 0) { if (BM_routed == 32) { // Single M-tile per expert -> STREAM_B=true protects x & out_fp32 in L2! launch_routed_pipeline<32, 2, 4, true, 4, 8>( x_ptr, w1r_ptr, w2r_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr, g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr, g_ws.h_routed_ptr, g_ws.out_fp32_ptr, E, H, I, max_m_tiles, stream ); } else if (BM_routed == 64) { launch_routed_pipeline<64, 2, 4, true, 4, 8>( x_ptr, w1r_ptr, w2r_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr, g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr, g_ws.h_routed_ptr, g_ws.out_fp32_ptr, E, H, I, max_m_tiles, stream ); } else { // Multi M-tile per expert -> STREAM_B=false keeps weights in L2 across adjacent M-tiles! // GROUP_N_S1=2 maximizes M-tile weight L2 reuse; GROUP_N_S2=8 keeps out_fp32 band (<=33.5 MB) in L2! launch_routed_pipeline<128, 3, 8, false, 2, 8>( x_ptr, w1r_ptr, w2r_ptr, g_ws.tile_map_ptr, g_ws.num_tiles_ptr, g_ws.sorted_tok_ptr, g_ws.sorted_wt_ptr, g_ws.h_routed_ptr, g_ws.out_fp32_ptr, E, H, I, max_m_tiles, stream ); } } // 4. Cast out_fp32 -> out_bf16 int64_t total_elems = (int64_t)T * H; int cast_blocks = static_cast((total_elems + 2047) / 2048); cast_fp32_to_bf16_kernel<<>>(g_ws.out_fp32_ptr, out_bf16_ptr, total_elems); return out; } """ _CPP_SRC = r""" #include torch::Tensor glm52_fused_moe_forward( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared ); """ _EXT = load_inline( name="glm52_fused_moe_sm120_ext_v4", cpp_sources=_CPP_SRC, cuda_sources=_CUDA_SRC, functions=["glm52_fused_moe_forward"], extra_cuda_cflags=["-O3", "-arch=sm_120a"], extra_ldflags=["-lcuda"], verbose=False, ) class Model(nn.Module): def __init__( self, T: int, E: int, top_k: int, n_shared: int, H: int, I: int, ): super().__init__() self.T, self.E, self.top_k = T, E, top_k self.n_shared, self.H, self.I = n_shared, H, I self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16)) self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16)) self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16)) self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16)) for p in self.parameters(): nn.init.normal_(p, std=0.02) def forward( self, x: torch.Tensor, expert_ids: torch.Tensor, expert_weights: torch.Tensor, ) -> torch.Tensor: return _EXT.glm52_fused_moe_forward( x, expert_ids, expert_weights, self.w1_routed, self.w2_routed, self.w1_shared, self.w2_shared, )