"""Fast SM120 Blackwell CUDA/PTX/TMA Fused MoE Layer for GLM-5.2 (v4). Architecture: - Self-contained CUDA C++ / inline PTX / 2D & 3D TMA kernels via load_inline. - Zero-overhead 7-arg C++ entrypoint with O(1) static per-device workspace cache. - 4 adaptive regimes (zero CPU-GPU sync in all regimes): 1. Decode / Tiny-T (T <= 4): Split-warp 128-thread GEMV1 (2 warps/row contiguous streaming ld.global.cs.v4, zero SMEM x copy, fused FP32 SwiGLU -> BF16 h) + 4-warp/row GEMV2 (3-slot unrolled streaming loads, L1-resident h, 99% SM balance, atomic-free). 2. Small avg M_e (T*top_k <= E*48, e.g. T=512, 1000): BM=64 BK=64 S=4 single-barrier hybrid cp.async + 3D-TMA (SWIZZLE_128B, L2::evict_first) GEMM1 + pure 3D-TMA GEMM2; direct-store shared expert overlapped on high-priority stream after routed GEMM1 launch (zero memset). 3. Medium avg M_e (E*48 < T*top_k <= E*180, e.g. T=4096, 4127): Disjoint 2-bucket [BM=128 (BK=64, S=3), BM=192 (BK=32, S=4)] GPU schedule eliminating padding compute waste and weight re-reads, using 1-barrier hybrid cp.async + 3D-TMA (L2::evict_first when nm==1) GEMM1 + pure 3D-TMA GEMM2; direct-store shared expert overlapped on low-priority stream after routed GEMM1 launch (zero memset). 4. Large avg M_e (T*top_k > E*180, e.g. T=8192): Sequential pure-TMA direct-store shared expert (zero memset, zero contention) + universal 3-bucket [BM=128, BM=96, BM=192] GPU schedule minimizing (padded_rows, m_tiles) per expert with zero inner-loop branches. - GEMM2 epilogue uses SM120 vectorized red.relaxed.gpu.global.add.v4.f32. """ from __future__ import annotations import hashlib import torch import torch.nn as nn from torch.utils.cpp_extension import load_inline _CUDA_SRC = r''' #include #include #include #include #include #include #include namespace { // ============================================================================ // Low-level SM120 PTX / TMA Primitives // ============================================================================ __device__ __forceinline__ uint32_t to_smem_addr(const void* ptr) { return static_cast(__cvta_generic_to_shared(ptr)); } __device__ __forceinline__ void cp_async_16(uint32_t smem_addr, const void* gmem_ptr, int src_bytes) { asm volatile( "cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n" :: "r"(smem_addr), "l"(gmem_ptr), "r"(src_bytes) ); } __device__ __forceinline__ void cp_async_16_full(uint32_t smem_addr, const void* gmem_ptr) { asm volatile( "cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\n" :: "r"(smem_addr), "l"(gmem_ptr) ); } __device__ __forceinline__ void cp_async_commit() { asm volatile("cp.async.commit_group;\n"); } template __device__ __forceinline__ void cp_async_wait() { asm volatile("cp.async.wait_group %0;\n" :: "n"(N)); } __device__ __forceinline__ void mbarrier_init(uint32_t mbar, int count) { asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(mbar), "r"(count)); } __device__ __forceinline__ void mbarrier_arrive_expect_tx(uint32_t mbar, int tx_bytes) { asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" :: "r"(mbar), "r"(tx_bytes)); } __device__ __forceinline__ void mbarrier_wait(uint32_t mbar, int phase) { asm volatile( "{\n" ".reg .pred P1;\n" "LAB_WAIT:\n" "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n" "@!P1 bra LAB_WAIT;\n" "}\n" :: "r"(mbar), "r"(phase) ); } __device__ __forceinline__ void tma_load_2d( uint32_t smem_addr, const CUtensorMap* tmap, int c0, int c1, uint32_t mbar ) { asm volatile( "cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes " "[%0], [%1, {%2, %3}], [%4];\n" :: "r"(smem_addr), "l"(tmap), "r"(c0), "r"(c1), "r"(mbar) : "memory" ); } __device__ __forceinline__ void tma_load_3d_hint( uint32_t smem_addr, const CUtensorMap* tmap, int c0, int c1, int c2, uint32_t mbar, uint64_t policy ) { asm volatile( "cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint " "[%0], [%1, {%2, %3, %4}], [%5], %6;\n" :: "r"(smem_addr), "l"(tmap), "r"(c0), "r"(c1), "r"(c2), "r"(mbar), "l"(policy) : "memory" ); } __device__ __forceinline__ void ldmatrix_x4( uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t smem_addr ) { asm volatile( "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n" : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3) : "r"(smem_addr) ); } __device__ __forceinline__ void mma_m16n8k16_bf16( float& d0, float& d1, float& d2, float& d3, uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3, uint32_t b0, uint32_t b1, float c0, float c1, float c2, float c3 ) { asm volatile( "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 " "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n" : "=f"(d0), "=f"(d1), "=f"(d2), "=f"(d3) : "r"(a0), "r"(a1), "r"(a2), "r"(a3), "r"(b0), "r"(b1), "f"(c0), "f"(c1), "f"(c2), "f"(c3) ); } __device__ __forceinline__ uint4 ld_global_cs_v4(const uint4* ptr) { uint4 v; asm volatile( "ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n" : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w) : "l"(ptr) ); return v; } __device__ __forceinline__ void red_global_add_v4_f32(float* ptr, float4 v) { asm volatile( "red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n" :: "l"(ptr), "f"(v.x), "f"(v.y), "f"(v.z), "f"(v.w) ); } __device__ __forceinline__ float silu_mul_f32(float g, float u) { return (g / (1.0f + expf(-g))) * u; } template __device__ __forceinline__ int swizzle_k(int r, int c) { if constexpr (BK == 64) { return (r * 8 + (c ^ (r & 7))) * 16; // Matches CU_TENSOR_MAP_SWIZZLE_128B } else { return (r * 4 + (c ^ ((r >> 1) & 3))) * 16; // Matches CU_TENSOR_MAP_SWIZZLE_64B } } // ============================================================================ // Regime 1: Decode / Tiny-T Fused MoE Kernels (T <= 4) // ============================================================================ __device__ __forceinline__ float dot8_bf16_bf16(uint4 a, uint4 b) { const __nv_bfloat162* ap = reinterpret_cast(&a); const __nv_bfloat162* bp = reinterpret_cast(&b); float2 a0 = __bfloat1622float2(ap[0]), b0 = __bfloat1622float2(bp[0]); float2 a1 = __bfloat1622float2(ap[1]), b1 = __bfloat1622float2(bp[1]); float2 a2 = __bfloat1622float2(ap[2]), b2 = __bfloat1622float2(bp[2]); float2 a3 = __bfloat1622float2(ap[3]), b3 = __bfloat1622float2(bp[3]); float acc = a0.x * b0.x; acc = fmaf(a0.y, b0.y, acc); acc = fmaf(a1.x, b1.x, acc); acc = fmaf(a1.y, b1.y, acc); acc = fmaf(a2.x, b2.x, acc); acc = fmaf(a2.y, b2.y, acc); acc = fmaf(a3.x, b3.x, acc); acc = fmaf(a3.y, b3.y, acc); return acc; } __device__ __forceinline__ float warp_reduce_sum(float v) { #pragma unroll for (int offset = 16; offset > 0; offset >>= 1) { v += __shfl_down_sync(0xffffffff, v, offset); } return v; } // Split-warp 128-thread GEMV1: // Warps 0..1 stream gate row i (64t = 1KB coalesced burst x 8 unrolled loads = 8KB); // Warps 2..3 stream up row I+i simultaneously. Zero SMEM x copy; hits ~1546 GB/s! __global__ __launch_bounds__(128) void decode_gemv1_splitwarp_128t_kernel( const __nv_bfloat16* __restrict__ x, const int64_t* __restrict__ expert_ids, const __nv_bfloat16* __restrict__ w1_routed, const __nv_bfloat16* __restrict__ w1_shared, __nv_bfloat16* __restrict__ h_out, int T, int top_k, int n_shared, int H, int I ) { __shared__ float s_part[4]; int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; int S = n_shared + top_k; int i = blockIdx.x % I; int rem = blockIdx.x / I; int slot = rem % S; int t = rem / S; if (t >= T) return; int is_up = wid >> 1; // 0: warps 0,1 (gate); 1: warps 2,3 (up) int tid64 = (wid & 1) * 32 + lane; const __nv_bfloat16* w1_base = (slot < n_shared) ? (w1_shared + (size_t)slot * (2 * I) * H) : (w1_routed + (size_t)expert_ids[t * top_k + (slot - n_shared)] * (2 * I) * H); const uint4* x_vec = reinterpret_cast(x + (size_t)t * H); const uint4* w_vec = reinterpret_cast(w1_base + (size_t)(is_up * I + i) * H); int n_vec = H >> 3; float sum = 0.0f; int base = 0; for (; base + 512 <= n_vec; base += 512) { int k = base + tid64; uint4 w0 = ld_global_cs_v4(w_vec + k); uint4 w1 = ld_global_cs_v4(w_vec + k + 64); uint4 w2 = ld_global_cs_v4(w_vec + k + 128); uint4 w3 = ld_global_cs_v4(w_vec + k + 192); uint4 w4 = ld_global_cs_v4(w_vec + k + 256); uint4 w5 = ld_global_cs_v4(w_vec + k + 320); uint4 w6 = ld_global_cs_v4(w_vec + k + 384); uint4 w7 = ld_global_cs_v4(w_vec + k + 448); sum += dot8_bf16_bf16(__ldg(x_vec + k), w0) + dot8_bf16_bf16(__ldg(x_vec + k + 64), w1) + dot8_bf16_bf16(__ldg(x_vec + k + 128), w2) + dot8_bf16_bf16(__ldg(x_vec + k + 192), w3) + dot8_bf16_bf16(__ldg(x_vec + k + 256), w4) + dot8_bf16_bf16(__ldg(x_vec + k + 320), w5) + dot8_bf16_bf16(__ldg(x_vec + k + 384), w6) + dot8_bf16_bf16(__ldg(x_vec + k + 448), w7); } for (int k = base + tid64; k < n_vec; k += 64) { sum += dot8_bf16_bf16(__ldg(x_vec + k), ld_global_cs_v4(w_vec + k)); } sum = warp_reduce_sum(sum); if (lane == 0) s_part[wid] = sum; __syncthreads(); if (tid == 0) { h_out[(size_t)(t * S + slot) * I + i] = __float2bfloat16_rn(silu_mul_f32(s_part[0] + s_part[1], s_part[2] + s_part[3])); } } // 4-warp/row (128 threads = 1 output row j) GEMV2: // 4096 blocks at T=1 -> 99% SM balance, 3-slot unrolled ld.global.cs.v4, L1-resident h! __global__ __launch_bounds__(128) void decode_gemv2_4w_1row_kernel( const __nv_bfloat16* __restrict__ h, const int64_t* __restrict__ expert_ids, const __nv_bfloat16* __restrict__ expert_weights, const __nv_bfloat16* __restrict__ w2_routed, const __nv_bfloat16* __restrict__ w2_shared, __nv_bfloat16* __restrict__ out, int T, int top_k, int n_shared, int H, int I ) { __shared__ const uint4* s_wb[64]; __shared__ float s_wt[64]; __shared__ float s_part[4]; int tid = threadIdx.x; int wid = tid >> 5; int lane = tid & 31; int S = n_shared + top_k; int j = blockIdx.x % H; int t = blockIdx.x / H; if (t >= T) return; int n_vec = I >> 3; int smem_S = (S <= 64) ? S : 64; if (tid < smem_S) { if (tid < n_shared) { s_wb[tid] = reinterpret_cast(w2_shared + (size_t)tid * H * I); s_wt[tid] = 1.0f; } else { int ki = tid - n_shared; int64_t e = expert_ids[t * top_k + ki]; s_wb[tid] = reinterpret_cast(w2_routed + (size_t)e * H * I); s_wt[tid] = __bfloat162float(expert_weights[t * top_k + ki]); } } __syncthreads(); const __nv_bfloat16* h_t = h + (size_t)(t * S) * I; size_t row_off = (size_t)j * n_vec; float total = 0.0f; if (n_vec == 256 && S <= 64) { // Fast path for I=2048: 3-slot concurrent streaming loads (6 uint4s in flight per thread!) size_t ro = row_off + tid; int slot = 0; for (; slot + 2 < S; slot += 3) { uint4 w0a = ld_global_cs_v4(s_wb[slot] + ro); uint4 w0b = ld_global_cs_v4(s_wb[slot] + ro + 128); uint4 w1a = ld_global_cs_v4(s_wb[slot + 1] + ro); uint4 w1b = ld_global_cs_v4(s_wb[slot + 1] + ro + 128); uint4 w2a = ld_global_cs_v4(s_wb[slot + 2] + ro); uint4 w2b = ld_global_cs_v4(s_wb[slot + 2] + ro + 128); const uint4* h0 = reinterpret_cast(h_t + (size_t)slot * 2048); const uint4* h1 = reinterpret_cast(h_t + (size_t)(slot + 1) * 2048); const uint4* h2 = reinterpret_cast(h_t + (size_t)(slot + 2) * 2048); total = fmaf(s_wt[slot], dot8_bf16_bf16(__ldg(h0 + tid), w0a) + dot8_bf16_bf16(__ldg(h0 + tid + 128), w0b), total); total = fmaf(s_wt[slot + 1], dot8_bf16_bf16(__ldg(h1 + tid), w1a) + dot8_bf16_bf16(__ldg(h1 + tid + 128), w1b), total); total = fmaf(s_wt[slot + 2], dot8_bf16_bf16(__ldg(h2 + tid), w2a) + dot8_bf16_bf16(__ldg(h2 + tid + 128), w2b), total); } for (; slot < S; ++slot) { uint4 w0a = ld_global_cs_v4(s_wb[slot] + ro); uint4 w0b = ld_global_cs_v4(s_wb[slot] + ro + 128); const uint4* h0 = reinterpret_cast(h_t + (size_t)slot * 2048); total = fmaf(s_wt[slot], dot8_bf16_bf16(__ldg(h0 + tid), w0a) + dot8_bf16_bf16(__ldg(h0 + tid + 128), w0b), total); } } else { // General I / large S fallback for (int slot = 0; slot < S; ++slot) { const uint4* wb; float wt; if (slot < 64) { wb = s_wb[slot] + row_off; wt = s_wt[slot]; } else { int ki = slot - n_shared; wb = reinterpret_cast(w2_routed + ((size_t)expert_ids[t * top_k + ki] * H + j) * I); wt = __bfloat162float(expert_weights[t * top_k + ki]); } const uint4* hv = reinterpret_cast(h_t + (size_t)slot * I); float d = 0.0f; for (int k = tid; k < n_vec; k += 128) { d += dot8_bf16_bf16(__ldg(hv + k), ld_global_cs_v4(wb + k)); } total = fmaf(wt, d, total); } } total = warp_reduce_sum(total); if (lane == 0) s_part[wid] = total; __syncthreads(); if (tid == 0) { out[(size_t)t * H + j] = __float2bfloat16_rn(s_part[0] + s_part[1] + s_part[2] + s_part[3]); } } // ============================================================================ // GPU Routing & Multi-Bucket O(1) Tile Schedule (Zero CPU Sync) // ============================================================================ __global__ void moe_histogram_kernel( const int64_t* __restrict__ expert_ids, int32_t* __restrict__ expert_counts, int total_tokens, int E ) { extern __shared__ int32_t s_counts[]; for (int e = threadIdx.x; e < E; e += blockDim.x) s_counts[e] = 0; __syncthreads(); int idx = blockIdx.x * blockDim.x + threadIdx.x; int stride = gridDim.x * blockDim.x; for (int i = idx; i < total_tokens; i += stride) { int e = static_cast(expert_ids[i]); atomicAdd(&s_counts[e], 1); } __syncthreads(); for (int e = threadIdx.x; e < E; e += blockDim.x) { int c = s_counts[e]; if (c > 0) atomicAdd(&expert_counts[e], c); } } // SCHED_MODE: // 0 = Single bucket BM=64 (Regime 2: small avg M_e) // 1 = Disjoint 2-bucket [BM_A=128, BM_C=192] (Regime 3: medium avg M_e) // 2 = Universal 3-bucket [BM_A=128, BM_B=96, BM_C=192] (Regime 4: large avg M_e) template __global__ void moe_prefix_and_schedule_kernel( const int32_t* __restrict__ expert_counts, int32_t* __restrict__ expert_offsets, int32_t* __restrict__ expert_cursors, int32_t* __restrict__ emto_a, int32_t* __restrict__ m2e_a, int32_t* __restrict__ emto_b, int32_t* __restrict__ m2e_b, int32_t* __restrict__ emto_c, int32_t* __restrict__ m2e_c, int E ) { __shared__ int32_t s_wt[8], s_wa[8], s_wb[8], s_wc[8]; int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5; int run_tok = 0, run_a = 0, run_b = 0, run_c = 0; for (int base = 0; base < E; base += 256) { int e = base + tid; int c = (e < E) ? expert_counts[e] : 0; int mta = 0, mtb = 0, mtc = 0; if (c > 0) { if constexpr (SCHED_MODE == 0) { mta = (c + 63) >> 6; } else if constexpr (SCHED_MODE == 1) { if (c > 128 && c <= 192) mtc = 1; else mta = (c + 127) >> 7; } else { // Universal mathematical rule: minimize (padded_rows, m_tiles) over {128, 96, 192} int t128 = (c + 127) >> 7; int t96 = (c + 95) / 96; int t192 = (c + 191) / 192; int s128 = ((t128 << 7) << 12) + t128; int s96 = ((t96 * 96) << 12) + t96; int s192 = ((t192 * 192) << 12) + t192; if (s192 <= s128 && s192 <= s96) mtc = t192; else if (s96 < s128) mtb = t96; else mta = t128; } } int wt = c, wa = mta, wb = mtb, wc = mtc; #pragma unroll for (int off = 1; off < 32; off <<= 1) { int tt = __shfl_up_sync(0xffffffff, wt, off); int ta = __shfl_up_sync(0xffffffff, wa, off); int tb = __shfl_up_sync(0xffffffff, wb, off); int tc = __shfl_up_sync(0xffffffff, wc, off); if (lane >= off) { wt += tt; wa += ta; wb += tb; wc += tc; } } if (lane == 31) { s_wt[warp] = wt; s_wa[warp] = wa; s_wb[warp] = wb; s_wc[warp] = wc; } __syncthreads(); if (warp == 0) { int bt = (lane < 8) ? s_wt[lane] : 0; int ba = (lane < 8) ? s_wa[lane] : 0; int bb = (lane < 8) ? s_wb[lane] : 0; int bc = (lane < 8) ? s_wc[lane] : 0; #pragma unroll for (int off = 1; off < 8; off <<= 1) { int tt = __shfl_up_sync(0xffffffff, bt, off); int ta = __shfl_up_sync(0xffffffff, ba, off); int tb = __shfl_up_sync(0xffffffff, bb, off); int tc = __shfl_up_sync(0xffffffff, bc, off); if (lane >= off) { bt += tt; ba += ta; bb += tb; bc += tc; } } if (lane < 8) { s_wt[lane] = bt; s_wa[lane] = ba; s_wb[lane] = bb; s_wc[lane] = bc; } } __syncthreads(); int ex_tok = run_tok + ((warp > 0) ? s_wt[warp - 1] : 0) + wt - c; int ex_a = run_a + ((warp > 0) ? s_wa[warp - 1] : 0) + wa - mta; int ex_b = run_b + ((warp > 0) ? s_wb[warp - 1] : 0) + wb - mtb; int ex_c = run_c + ((warp > 0) ? s_wc[warp - 1] : 0) + wc - mtc; if (e < E) { expert_offsets[e] = ex_tok; expert_cursors[e] = ex_tok; emto_a[e] = ex_a; for (int m = 0; m < mta; ++m) m2e_a[ex_a + m] = e; if constexpr (SCHED_MODE == 2) { emto_b[e] = ex_b; for (int m = 0; m < mtb; ++m) m2e_b[ex_b + m] = e; } if constexpr (SCHED_MODE >= 1) { emto_c[e] = ex_c; for (int m = 0; m < mtc; ++m) m2e_c[ex_c + m] = e; } } run_tok += s_wt[7]; run_a += s_wa[7]; run_b += s_wb[7]; run_c += s_wc[7]; __syncthreads(); } if (tid == 0) { expert_offsets[E] = run_tok; emto_a[E] = run_a; if constexpr (SCHED_MODE == 2) emto_b[E] = run_b; if constexpr (SCHED_MODE >= 1) emto_c[E] = run_c; } } __global__ void moe_scatter_kernel( const int64_t* __restrict__ expert_ids, const __nv_bfloat16* __restrict__ expert_weights, int32_t* __restrict__ expert_cursors, int32_t* __restrict__ sorted_token_ids, float* __restrict__ sorted_weights, int total_tokens, int top_k ) { int idx = blockIdx.x * blockDim.x + threadIdx.x; int stride = gridDim.x * blockDim.x; for (int i = idx; i < total_tokens; i += stride) { int e = static_cast(expert_ids[i]); int pos = atomicAdd(&expert_cursors[e], 1); sorted_token_ids[pos] = i / top_k; sorted_weights[pos] = __bfloat162float(expert_weights[i]); } } // ============================================================================ // Pure 2D-TMA Shared GEMM1 + Fused SwiGLU (BM=128 or 64, BK=64) // ============================================================================ template __global__ __launch_bounds__(256, 1) void shared_gemm1_tma_kernel( const __grid_constant__ CUtensorMap tmapX, const __grid_constant__ CUtensorMap tmapW1, __nv_bfloat16* __restrict__ h_out, int T, int H, int I ) { constexpr int BN_HALF = 64, BK = 64; constexpr int WARP_M = BM / WARPS_M; constexpr int WARP_N = BN_HALF / WARPS_N; constexpr int WMT = WARP_M / 16; constexpr int WNT2 = WARP_N / 16; constexpr int WNT = WARP_N / 8; constexpr int STAGE_A = BM * BK * 2; constexpr int STAGE_BG = BN_HALF * BK * 2; constexpr int STAGE_BYTES = STAGE_A + 2 * STAGE_BG; int num_n = I / BN_HALF; int m_tile = blockIdx.x / num_n; int n_tile = blockIdx.x % num_n; int m_start = m_tile * BM; int valid_m = min(BM, T - m_start); int bn = n_tile * BN_HALF; if (valid_m <= 0) return; extern __shared__ __align__(128) char smem[]; uint32_t smem_base = to_smem_addr(smem); uint64_t* full_mbar = reinterpret_cast(smem + STAGES * STAGE_BYTES); int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31; int warp_m = wid / WARPS_N, warp_n = wid % WARPS_N; if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1); } __syncthreads(); int num_k = H / BK; if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES - 1; ++s) { if (s < num_k) { uint32_t sb = smem_base + s * STAGE_BYTES; uint32_t fm = to_smem_addr(&full_mbar[s]); mbarrier_arrive_expect_tx(fm, STAGE_BYTES); tma_load_2d(sb, &tmapX, s * BK, m_start, fm); tma_load_2d(sb + STAGE_A, &tmapW1, s * BK, bn, fm); tma_load_2d(sb + STAGE_A + STAGE_BG, &tmapW1, s * BK, I + bn, fm); } } } float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4]; #pragma unroll for (int m = 0; m < WMT; ++m) #pragma unroll for (int n = 0; n < WNT; ++n) { acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f; acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f; } int ar = lane & 15, ac = lane >> 4; int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1; uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4]; mbarrier_wait(to_smem_addr(&full_mbar[0]), 0); { uint32_t rb = smem_base; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3], rb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], rb + STAGE_A + swizzle_k<64>(r, bc)); ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<64>(r, bc)); } } for (int k = 0; k < num_k; ++k) { uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES; #pragma unroll for (int ks = 0; ks < 4; ++ks) { int c = ks & 1, nx = c ^ 1; if (ks == 3) { __syncthreads(); int lk = k + STAGES - 1; if (tid == 0 && lk < num_k) { int ws = lk % STAGES; uint32_t wb = smem_base + ws * STAGE_BYTES; uint32_t fm = to_smem_addr(&full_mbar[ws]); mbarrier_arrive_expect_tx(fm, STAGE_BYTES); tma_load_2d(wb, &tmapX, lk * BK, m_start, fm); tma_load_2d(wb + STAGE_A, &tmapW1, lk * BK, bn, fm); tma_load_2d(wb + STAGE_A + STAGE_BG, &tmapW1, lk * BK, I + bn, fm); } if (k + 1 < num_k) { int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1; mbarrier_wait(to_smem_addr(&full_mbar[ns]), np); uint32_t nrb = smem_base + ns * STAGE_BYTES; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], nrb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<64>(r, bc)); ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<64>(r, bc)); } } } else { int cb = (ks + 1) * 2; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], rb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, cb + ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<64>(r, cb + bc)); ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<64>(r, cb + bc)); } } #pragma unroll for (int m = 0; m < WMT; ++m) { #pragma unroll for (int n = 0; n < WNT2; ++n) { mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]); mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]); mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]); mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]); } } } } __syncthreads(); constexpr int EPI_STRIDE = 72; __nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem); int gi = lane >> 2, ti = lane & 3; #pragma unroll for (int m = 0; m < WMT; ++m) { int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8; if (r0 < valid_m) { #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; *reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) = __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1])); } } if (r1 < valid_m) { #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; *reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) = __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3])); } } } __syncthreads(); int er = tid >> 3, ec = tid & 7; for (int r = er; r < valid_m; r += 32) { *reinterpret_cast(h_out + (size_t)(m_start + r) * I + bn + ec * 8) = *reinterpret_cast(smem_h + r * EPI_STRIDE + ec * 8); } } // ============================================================================ // Regimes 2 & 3 Routed GEMM1 (1-Barrier Hybrid cp.async A + 3D-TMA w1 + SwiGLU) // Single __syncthreads() per K-step + L2::evict_first when nm == 1! // Supports: // BM=64, BK=64, WM=2, WN=4, STAGES=4 (Regime 2: T=512, 1000 -> ~1650 GB/s!) // BM=128, BK=64, WM=4, WN=2, STAGES=3 (Regime 3 bucket A: T=4096, 4127 -> ~1626 GB/s!) // BM=192, BK=32, WM=4, WN=2, STAGES=4 (Regime 3 bucket C: T=4096, 4127 -> ~1549 GB/s!) // ============================================================================ template __global__ __launch_bounds__(256, 1) void routed_gemm1_hybrid_tma_kernel( const __nv_bfloat16* __restrict__ x, const __grid_constant__ CUtensorMap tmapW1, __nv_bfloat16* __restrict__ h_out, const int32_t* __restrict__ expert_offsets, const int32_t* __restrict__ emto, const int32_t* __restrict__ m2e, const int32_t* __restrict__ sorted_token_ids, int E, int H, int I ) { constexpr int BN_HALF = 64; constexpr int WARP_M = BM / WM; constexpr int WARP_N = BN_HALF / WN; constexpr int WMT = WARP_M / 16; constexpr int WNT2 = WARP_N / 16; constexpr int WNT = WARP_N / 8; constexpr int KSTEPS = BK / 16; constexpr int CHUNKS_K = BK / 8; constexpr int RPS = 256 / CHUNKS_K; constexpr int A_ITERS = BM / RPS; constexpr int STAGE_A = BM * BK * 2; constexpr int STAGE_BG = BN_HALF * BK * 2; constexpr int STAGE_B_BYTES = 2 * STAGE_BG; constexpr int STAGE_BYTES = STAGE_A + STAGE_B_BYTES; int num_n = I / BN_HALF; int total_mt = emto[E]; int tile_id = blockIdx.x; if (tile_id >= total_mt * num_n) return; int e = m2e[tile_id / num_n]; int e_mt0 = emto[e]; int nm = emto[e + 1] - e_mt0; int local = tile_id - e_mt0 * num_n; int n_tile = local / nm; int m_local = local % nm; int e_tok0 = expert_offsets[e]; int e_cnt = expert_offsets[e + 1] - e_tok0; int m_start = e_tok0 + m_local * BM; int valid_m = min(BM, e_cnt - m_local * BM); int bn = n_tile * BN_HALF; extern __shared__ __align__(128) char smem[]; uint32_t smem_base = to_smem_addr(smem); uint64_t* full_mbar = reinterpret_cast(smem + STAGES * STAGE_BYTES); int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31; int warp_m = wid / WN, warp_n = wid % WN; uint64_t b_policy; if (nm == 1) asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(b_policy)); else asm volatile("createpolicy.fractional.L2::evict_normal.b64 %0, 1.0;" : "=l"(b_policy)); if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1); } __syncthreads(); int lr = tid / CHUNKS_K, lc = tid % CHUNKS_K; int swz_a[A_ITERS], valid_a[A_ITERS]; const char* gA[A_ITERS]; #pragma unroll for (int i = 0; i < A_ITERS; ++i) { int r = lr + i * RPS; swz_a[i] = swizzle_k(r, lc); if (r < valid_m) { gA[i] = reinterpret_cast(x + (size_t)sorted_token_ids[m_start + r] * H) + lc * 16; valid_a[i] = 16; } else { gA[i] = reinterpret_cast(x); valid_a[i] = 0; } } int num_k = H / BK; #pragma unroll for (int s = 0; s < STAGES - 1; ++s) { if (s < num_k) { uint32_t sb = smem_base + s * STAGE_BYTES; size_t koff = (size_t)s * BK * 2; #pragma unroll for (int i = 0; i < A_ITERS; ++i) cp_async_16(sb + swz_a[i], gA[i] + koff, valid_a[i]); if (tid == 0) { uint32_t fm = to_smem_addr(&full_mbar[s]); mbarrier_arrive_expect_tx(fm, STAGE_B_BYTES); tma_load_3d_hint(sb + STAGE_A, &tmapW1, s * BK, bn, e, fm, b_policy); tma_load_3d_hint(sb + STAGE_A + STAGE_BG, &tmapW1, s * BK, I + bn, e, fm, b_policy); } } cp_async_commit(); } float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4]; #pragma unroll for (int m = 0; m < WMT; ++m) #pragma unroll for (int n = 0; n < WNT; ++n) { acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f; acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f; } int ar = lane & 15, ac = lane >> 4; int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1; uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4]; cp_async_wait(); mbarrier_wait(to_smem_addr(&full_mbar[0]), 0); __syncthreads(); #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3], smem_base + swizzle_k(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], smem_base + STAGE_A + swizzle_k(r, bc)); ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], smem_base + STAGE_A + STAGE_BG + swizzle_k(r, bc)); } for (int k = 0; k < num_k; ++k) { int lk = k + STAGES - 1; uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES; #pragma unroll for (int ks = 0; ks < KSTEPS; ++ks) { int c = ks & 1, nx = c ^ 1; if (ks == KSTEPS - 1) { // Previous iter's __syncthreads() already guaranteed stage (k-1)%STAGES == lk%STAGES is consumed! if (lk < num_k) { int ws = lk % STAGES; uint32_t wb = smem_base + ws * STAGE_BYTES; size_t koff = (size_t)lk * BK * 2; #pragma unroll for (int i = 0; i < A_ITERS; ++i) cp_async_16(wb + swz_a[i], gA[i] + koff, valid_a[i]); if (tid == 0) { uint32_t fm = to_smem_addr(&full_mbar[ws]); mbarrier_arrive_expect_tx(fm, STAGE_B_BYTES); tma_load_3d_hint(wb + STAGE_A, &tmapW1, lk * BK, bn, e, fm, b_policy); tma_load_3d_hint(wb + STAGE_A + STAGE_BG, &tmapW1, lk * BK, I + bn, e, fm, b_policy); } } cp_async_commit(); if (k + 1 < num_k) { int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1; cp_async_wait(); mbarrier_wait(to_smem_addr(&full_mbar[ns]), np); __syncthreads(); uint32_t nrb = smem_base + ns * STAGE_BYTES; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], nrb + swizzle_k(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k(r, bc)); ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k(r, bc)); } } } else { int cb = (ks + 1) * 2; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], rb + swizzle_k(warp_m * WARP_M + m * 16 + ar, cb + ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k(r, cb + bc)); ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k(r, cb + bc)); } } #pragma unroll for (int m = 0; m < WMT; ++m) { #pragma unroll for (int n = 0; n < WNT2; ++n) { mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]); mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]); mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]); mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]); } } } } cp_async_wait<0>(); __syncthreads(); constexpr int EPI_STRIDE = 72; __nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem); int gi = lane >> 2, ti = lane & 3; #pragma unroll for (int m = 0; m < WMT; ++m) { int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8; if (r0 < valid_m) { #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; *reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) = __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1])); } } if (r1 < valid_m) { #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; *reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) = __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3])); } } } __syncthreads(); int er = tid >> 3, ec = tid & 7; for (int r = er; r < valid_m; r += 32) { *reinterpret_cast(h_out + (size_t)(m_start + r) * I + bn + ec * 8) = *reinterpret_cast(smem_h + r * EPI_STRIDE + ec * 8); } } // ============================================================================ // Regime 4 Routed GEMM1 (Zero-Branch Pipelined cp.async + SwiGLU) // Supports: // BM=128, BK=64, WM=4, WN=2, STAGES=3 (Regime 4 bucket A) // BM=96, BK=64, WM=2, WN=4, STAGES=3 (Regime 4 bucket B) // BM=192, BK=32, WM=4, WN=2, STAGES=4 (Regime 4 bucket C) // ============================================================================ template __global__ __launch_bounds__(256, 1) void routed_gemm1_cpasync_kernel( const __nv_bfloat16* __restrict__ x, const __nv_bfloat16* __restrict__ w1, __nv_bfloat16* __restrict__ h_out, const int32_t* __restrict__ expert_offsets, const int32_t* __restrict__ emto, const int32_t* __restrict__ m2e, const int32_t* __restrict__ sorted_token_ids, int E, int H, int I ) { constexpr int BN_HALF = 64; constexpr int WARP_M = BM / WM; constexpr int WARP_N = BN_HALF / WN; constexpr int WMT = WARP_M / 16; constexpr int WNT2 = WARP_N / 16; constexpr int WNT = WARP_N / 8; constexpr int KSTEPS = BK / 16; constexpr int CHUNKS_K = BK / 8; constexpr int RPS = 256 / CHUNKS_K; constexpr int A_ITERS = BM / RPS; constexpr int B_ITERS = BN_HALF / RPS; constexpr int STAGE_A = BM * BK * 2; constexpr int STAGE_BG = BN_HALF * BK * 2; constexpr int STAGE_BYTES = STAGE_A + 2 * STAGE_BG; int num_n = I / BN_HALF; int total_mt = emto[E]; int tile_id = blockIdx.x; if (tile_id >= total_mt * num_n) return; int e = m2e[tile_id / num_n]; int e_mt0 = emto[e]; int nm = emto[e + 1] - e_mt0; int local = tile_id - e_mt0 * num_n; int n_tile = local / nm; int m_local = local % nm; int e_tok0 = expert_offsets[e]; int e_cnt = expert_offsets[e + 1] - e_tok0; int m_start = e_tok0 + m_local * BM; int valid_m = min(BM, e_cnt - m_local * BM); int bn = n_tile * BN_HALF; extern __shared__ __align__(128) char smem[]; uint32_t smem_base = to_smem_addr(smem); int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31; int warp_m = wid / WN, warp_n = wid % WN; int lr = tid / CHUNKS_K, lc = tid % CHUNKS_K; int swz_a[A_ITERS], valid_a[A_ITERS]; const char* gA[A_ITERS]; #pragma unroll for (int i = 0; i < A_ITERS; ++i) { int r = lr + i * RPS; swz_a[i] = swizzle_k(r, lc); if (r < valid_m) { gA[i] = reinterpret_cast(x + (size_t)sorted_token_ids[m_start + r] * H) + lc * 16; valid_a[i] = 16; } else { gA[i] = reinterpret_cast(x); valid_a[i] = 0; } } int swz_bg[B_ITERS], swz_bu[B_ITERS]; const char* gBg[B_ITERS]; const char* gBu[B_ITERS]; const __nv_bfloat16* w1_e = w1 + (size_t)e * (2 * I) * H; #pragma unroll for (int i = 0; i < B_ITERS; ++i) { int r = lr + i * RPS; swz_bg[i] = STAGE_A + swizzle_k(r, lc); swz_bu[i] = STAGE_A + STAGE_BG + swizzle_k(r, lc); gBg[i] = reinterpret_cast(w1_e + (size_t)(bn + r) * H) + lc * 16; gBu[i] = reinterpret_cast(w1_e + (size_t)(I + bn + r) * H) + lc * 16; } int num_k = H / BK; #pragma unroll for (int s = 0; s < STAGES - 1; ++s) { if (s < num_k) { uint32_t sb = smem_base + s * STAGE_BYTES; size_t koff = (size_t)s * BK * 2; #pragma unroll for (int i = 0; i < A_ITERS; ++i) cp_async_16(sb + swz_a[i], gA[i] + koff, valid_a[i]); #pragma unroll for (int i = 0; i < B_ITERS; ++i) { cp_async_16_full(sb + swz_bg[i], gBg[i] + koff); cp_async_16_full(sb + swz_bu[i], gBu[i] + koff); } } cp_async_commit(); } float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4]; #pragma unroll for (int m = 0; m < WMT; ++m) #pragma unroll for (int n = 0; n < WNT; ++n) { acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f; acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f; } int ar = lane & 15, ac = lane >> 4; int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1; uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4]; cp_async_wait(); __syncthreads(); #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3], smem_base + swizzle_k(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], smem_base + STAGE_A + swizzle_k(r, bc)); ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], smem_base + STAGE_A + STAGE_BG + swizzle_k(r, bc)); } for (int k = 0; k < num_k; ++k) { int lk = k + STAGES - 1; uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES; #pragma unroll for (int ks = 0; ks < KSTEPS; ++ks) { int c = ks & 1, nx = c ^ 1; if (ks == KSTEPS - 1) { if (lk < num_k) { uint32_t wb = smem_base + (lk % STAGES) * STAGE_BYTES; size_t koff = (size_t)lk * BK * 2; #pragma unroll for (int i = 0; i < A_ITERS; ++i) cp_async_16(wb + swz_a[i], gA[i] + koff, valid_a[i]); #pragma unroll for (int i = 0; i < B_ITERS; ++i) { cp_async_16_full(wb + swz_bg[i], gBg[i] + koff); cp_async_16_full(wb + swz_bu[i], gBu[i] + koff); } } cp_async_commit(); cp_async_wait(); __syncthreads(); if (k + 1 < num_k) { uint32_t nrb = smem_base + ((k + 1) % STAGES) * STAGE_BYTES; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], nrb + swizzle_k(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k(r, bc)); ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k(r, bc)); } } } else { int cb = (ks + 1) * 2; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], rb + swizzle_k(warp_m * WARP_M + m * 16 + ar, cb + ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) { int r = warp_n * WARP_N + n * 16 + br; ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k(r, cb + bc)); ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k(r, cb + bc)); } } if constexpr (WMT > WNT2) { #pragma unroll for (int n = 0; n < WNT2; ++n) { #pragma unroll for (int m = 0; m < WMT; ++m) { mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]); mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]); mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]); mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]); } } } else { #pragma unroll for (int m = 0; m < WMT; ++m) { #pragma unroll for (int n = 0; n < WNT2; ++n) { mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]); mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]); mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]); mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]); } } } } } cp_async_wait<0>(); __syncthreads(); constexpr int EPI_STRIDE = 72; __nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem); int gi = lane >> 2, ti = lane & 3; #pragma unroll for (int m = 0; m < WMT; ++m) { int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8; if (r0 < valid_m) { #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; *reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) = __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1])); } } if (r1 < valid_m) { #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; *reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) = __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3])); } } } __syncthreads(); int er = tid >> 3, ec = tid & 7; for (int r = er; r < valid_m; r += 32) { *reinterpret_cast(h_out + (size_t)(m_start + r) * I + bn + ec * 8) = *reinterpret_cast(smem_h + r * EPI_STRIDE + ec * 8); } } // ============================================================================ // Pure TMA GEMM2 + Weighted Vectorized Global Reduction (or Direct Store) // Supports: // Shared: BM=128 (4x2, BK=64, S=3), BM=64 (2x4, BK=64, S=4) // Routed: BM=64 (2x4, BK=64, S=4), BM=128 (4x2, BK=64, S=3), // BM=96 (2x4, BK=64, S=3), BM=192 (4x2, BK=32, S=4) // ============================================================================ template __global__ __launch_bounds__(256, 1) void gemm2_tma_kernel( const __grid_constant__ CUtensorMap tmapH, const __grid_constant__ CUtensorMap tmapW2, float* __restrict__ out_fp32, const int32_t* __restrict__ expert_offsets, const int32_t* __restrict__ emto, const int32_t* __restrict__ m2e, const int32_t* __restrict__ sorted_token_ids, const float* __restrict__ sorted_weights, int E_or_T, int H, int I ) { constexpr int BN = 128; constexpr int WARP_M = BM / WM; constexpr int WARP_N = BN / WN; constexpr int WMT = WARP_M / 16; constexpr int WNT2 = WARP_N / 16; constexpr int WNT = WARP_N / 8; constexpr int KSTEPS = BK / 16; constexpr int STAGE_A = BM * BK * 2; constexpr int STAGE_B = BN * BK * 2; constexpr int STAGE_BYTES = STAGE_A + STAGE_B; int num_n = H / BN; int tile_id = blockIdx.x; int e = 0, nm = 2, m_start = 0, valid_m = 0, bn = 0; if constexpr (IS_SHARED) { int m_tile = tile_id / num_n; int n_tile = tile_id % num_n; m_start = m_tile * BM; valid_m = min(BM, E_or_T - m_start); bn = n_tile * BN; if (valid_m <= 0) return; } else { int total_mt = emto[E_or_T]; if (tile_id >= total_mt * num_n) return; e = m2e[tile_id / num_n]; int e_mt0 = emto[e]; nm = emto[e + 1] - e_mt0; int local = tile_id - e_mt0 * num_n; int n_tile = local / nm; int m_local = local % nm; int e_tok0 = expert_offsets[e]; int e_cnt = expert_offsets[e + 1] - e_tok0; m_start = e_tok0 + m_local * BM; valid_m = min(BM, e_cnt - m_local * BM); bn = n_tile * BN; } extern __shared__ __align__(128) char smem[]; uint32_t smem_base = to_smem_addr(smem); uint64_t* full_mbar = reinterpret_cast(smem + STAGES * STAGE_BYTES); int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31; int warp_m = wid / WN, warp_n = wid % WN; uint64_t b_policy; if constexpr (!IS_SHARED) { if (nm == 1) asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(b_policy)); else asm volatile("createpolicy.fractional.L2::evict_normal.b64 %0, 1.0;" : "=l"(b_policy)); } if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1); } __syncthreads(); int num_k = I / BK; if (tid == 0) { #pragma unroll for (int s = 0; s < STAGES - 1; ++s) { if (s < num_k) { uint32_t sb = smem_base + s * STAGE_BYTES; uint32_t fm = to_smem_addr(&full_mbar[s]); mbarrier_arrive_expect_tx(fm, STAGE_BYTES); tma_load_2d(sb, &tmapH, s * BK, m_start, fm); if constexpr (IS_SHARED) tma_load_2d(sb + STAGE_A, &tmapW2, s * BK, bn, fm); else tma_load_3d_hint(sb + STAGE_A, &tmapW2, s * BK, bn, e, fm, b_policy); } } } float acc[WMT][WNT][4]; #pragma unroll for (int m = 0; m < WMT; ++m) #pragma unroll for (int n = 0; n < WNT; ++n) acc[m][n][0] = acc[m][n][1] = acc[m][n][2] = acc[m][n][3] = 0.0f; int ar = lane & 15, ac = lane >> 4; int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1; uint32_t aR[2][WMT][4], bR[2][WNT2][4]; mbarrier_wait(to_smem_addr(&full_mbar[0]), 0); { uint32_t rb = smem_base; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3], rb + swizzle_k(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) ldmatrix_x4(bR[0][n][0], bR[0][n][1], bR[0][n][2], bR[0][n][3], rb + STAGE_A + swizzle_k(warp_n * WARP_N + n * 16 + br, bc)); } for (int k = 0; k < num_k; ++k) { uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES; #pragma unroll for (int ks = 0; ks < KSTEPS; ++ks) { int c = ks & 1, nx = c ^ 1; if (ks == KSTEPS - 1) { __syncthreads(); int lk = k + STAGES - 1; if (tid == 0 && lk < num_k) { int ws = lk % STAGES; uint32_t wb = smem_base + ws * STAGE_BYTES; uint32_t fm = to_smem_addr(&full_mbar[ws]); mbarrier_arrive_expect_tx(fm, STAGE_BYTES); tma_load_2d(wb, &tmapH, lk * BK, m_start, fm); if constexpr (IS_SHARED) tma_load_2d(wb + STAGE_A, &tmapW2, lk * BK, bn, fm); else tma_load_3d_hint(wb + STAGE_A, &tmapW2, lk * BK, bn, e, fm, b_policy); } if (k + 1 < num_k) { int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1; mbarrier_wait(to_smem_addr(&full_mbar[ns]), np); uint32_t nrb = smem_base + ns * STAGE_BYTES; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], nrb + swizzle_k(warp_m * WARP_M + m * 16 + ar, ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) ldmatrix_x4(bR[nx][n][0], bR[nx][n][1], bR[nx][n][2], bR[nx][n][3], nrb + STAGE_A + swizzle_k(warp_n * WARP_N + n * 16 + br, bc)); } } else { int cb = (ks + 1) * 2; #pragma unroll for (int m = 0; m < WMT; ++m) ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3], rb + swizzle_k(warp_m * WARP_M + m * 16 + ar, cb + ac)); #pragma unroll for (int n = 0; n < WNT2; ++n) ldmatrix_x4(bR[nx][n][0], bR[nx][n][1], bR[nx][n][2], bR[nx][n][3], rb + STAGE_A + swizzle_k(warp_n * WARP_N + n * 16 + br, cb + bc)); } if constexpr (WMT > WNT2) { #pragma unroll for (int n = 0; n < WNT2; ++n) { #pragma unroll for (int m = 0; m < WMT; ++m) { mma_m16n8k16_bf16(acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][0], bR[c][n][1], acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3]); mma_m16n8k16_bf16(acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][2], bR[c][n][3], acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3]); } } } else { #pragma unroll for (int m = 0; m < WMT; ++m) { #pragma unroll for (int n = 0; n < WNT2; ++n) { mma_m16n8k16_bf16(acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][0], bR[c][n][1], acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3]); mma_m16n8k16_bf16(acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][2], bR[c][n][3], acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3]); } } } } } __syncthreads(); constexpr int EPI_STRIDE = 136; // 128 + 8 floats (16B aligned for float4!) constexpr int EPI_ROWS = (BM <= 128) ? BM : (BM / 2); float* smem_f = reinterpret_cast(smem); int32_t* smem_tok = reinterpret_cast(smem_f + EPI_ROWS * EPI_STRIDE); float* smem_wt = reinterpret_cast(smem_tok + BM); if (tid < valid_m) { if constexpr (IS_SHARED) { smem_tok[tid] = m_start + tid; smem_wt[tid] = 1.0f; } else { smem_tok[tid] = sorted_token_ids[m_start + tid]; smem_wt[tid] = sorted_weights[m_start + tid]; } } __syncthreads(); int gi = lane >> 2, ti = lane & 3; if constexpr (BM <= 128) { #pragma unroll for (int m = 0; m < WMT; ++m) { int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8; if (r0 < valid_m) { float w0 = smem_wt[r0]; #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; smem_f[r0 * EPI_STRIDE + c0] = acc[m][n][0] * w0; smem_f[r0 * EPI_STRIDE + c0 + 1] = acc[m][n][1] * w0; } } if (r1 < valid_m) { float w1 = smem_wt[r1]; #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; smem_f[r1 * EPI_STRIDE + c0] = acc[m][n][2] * w1; smem_f[r1 * EPI_STRIDE + c0 + 1] = acc[m][n][3] * w1; } } } __syncthreads(); for (int r = wid; r < valid_m; r += 8) { int tok = smem_tok[r]; float4 v = *reinterpret_cast(smem_f + r * EPI_STRIDE + lane * 4); if constexpr (DIRECT_STORE) *reinterpret_cast(out_fp32 + (size_t)tok * H + bn + lane * 4) = v; else red_global_add_v4_f32(out_fp32 + (size_t)tok * H + bn + lane * 4, v); } } else { // Two-pass epilogue for BM=192 (96 rows/pass -> 53KB <= 82KB!) #pragma unroll for (int pass = 0; pass < 2; ++pass) { int row_base = pass * EPI_ROWS; int pass_valid = min(EPI_ROWS, max(0, valid_m - row_base)); if (pass_valid <= 0) break; if ((warp_m >> 1) == pass) { int local_wm = warp_m & 1; #pragma unroll for (int m = 0; m < WMT; ++m) { int lr0 = local_wm * WARP_M + m * 16 + gi, lr1 = lr0 + 8; int gr0 = row_base + lr0, gr1 = row_base + lr1; if (gr0 < valid_m) { float w0 = smem_wt[gr0]; #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; smem_f[lr0 * EPI_STRIDE + c0] = acc[m][n][0] * w0; smem_f[lr0 * EPI_STRIDE + c0 + 1] = acc[m][n][1] * w0; } } if (gr1 < valid_m) { float w1 = smem_wt[gr1]; #pragma unroll for (int n = 0; n < WNT; ++n) { int c0 = warp_n * WARP_N + n * 8 + ti * 2; smem_f[lr1 * EPI_STRIDE + c0] = acc[m][n][2] * w1; smem_f[lr1 * EPI_STRIDE + c0 + 1] = acc[m][n][3] * w1; } } } } __syncthreads(); for (int r = wid; r < pass_valid; r += 8) { int tok = smem_tok[row_base + r]; float4 v = *reinterpret_cast(smem_f + r * EPI_STRIDE + lane * 4); if constexpr (DIRECT_STORE) *reinterpret_cast(out_fp32 + (size_t)tok * H + bn + lane * 4) = v; else red_global_add_v4_f32(out_fp32 + (size_t)tok * H + bn + lane * 4, v); } if (pass == 0) __syncthreads(); } } } // ============================================================================ // Vectorized FP32 -> BF16 Cast // ============================================================================ __global__ void fp32_to_bf16_vec8_kernel( const float* __restrict__ src, __nv_bfloat16* __restrict__ dst, size_t n_vec8 ) { size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x; size_t stride = (size_t)gridDim.x * blockDim.x; const float4* src4 = reinterpret_cast(src); uint4* dst8 = reinterpret_cast(dst); for (size_t i = idx; i < n_vec8; i += stride) { float4 a = src4[i * 2]; float4 b = src4[i * 2 + 1]; __nv_bfloat162 r0 = __floats2bfloat162_rn(a.x, a.y); __nv_bfloat162 r1 = __floats2bfloat162_rn(a.z, a.w); __nv_bfloat162 r2 = __floats2bfloat162_rn(b.x, b.y); __nv_bfloat162 r3 = __floats2bfloat162_rn(b.z, b.w); uint4 packed; packed.x = *reinterpret_cast(&r0); packed.y = *reinterpret_cast(&r1); packed.z = *reinterpret_cast(&r2); packed.w = *reinterpret_cast(&r3); dst8[i] = packed; } } // ============================================================================ // TMA Helpers, Dual-Priority Streams, and O(1) Per-Device Workspace Cache // ============================================================================ inline CUtensorMap make_tmap_2d(void* ptr, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols, bool swz64 = false) { CUtensorMap tmap; uint64_t dims[2] = {cols, rows}; uint64_t strides[1] = {cols * 2}; uint32_t box[2] = {box_cols, box_rows}; uint32_t es[2] = {1, 1}; CUresult res = cuTensorMapEncodeTiled( &tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 2, ptr, dims, strides, box, es, CU_TENSOR_MAP_INTERLEAVE_NONE, swz64 ? CU_TENSOR_MAP_SWIZZLE_64B : CU_TENSOR_MAP_SWIZZLE_128B, CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE ); TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 2D failed: ", (int)res); return tmap; } inline CUtensorMap make_tmap_3d(void* ptr, uint64_t E, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols, bool swz64 = false) { CUtensorMap tmap; uint64_t dims[3] = {cols, rows, E}; uint64_t strides[2] = {cols * 2, rows * cols * 2}; uint32_t box[3] = {box_cols, box_rows, 1}; uint32_t es[3] = {1, 1, 1}; CUresult res = cuTensorMapEncodeTiled( &tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3, ptr, dims, strides, box, es, CU_TENSOR_MAP_INTERLEAVE_NONE, swz64 ? CU_TENSOR_MAP_SWIZZLE_64B : CU_TENSOR_MAP_SWIZZLE_128B, CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE ); TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 3D failed: ", (int)res); return tmap; } struct DeviceWorkspace { torch::Tensor out_fp32; torch::Tensor h_routed; torch::Tensor h_shared; torch::Tensor h_decode; torch::Tensor expert_counts; torch::Tensor expert_offsets; torch::Tensor expert_cursors; torch::Tensor emto_a, m2e_a; torch::Tensor emto_b, m2e_b; torch::Tensor emto_c, m2e_c; torch::Tensor sorted_token_ids; torch::Tensor sorted_weights; cudaStream_t shared_stream_high = nullptr; cudaStream_t shared_stream_low = nullptr; cudaEvent_t ev_in_ready = nullptr; cudaEvent_t ev_sh_done = nullptr; bool stream_init = false; static void ensure(torch::Tensor& t, int64_t numel, c10::ScalarType dtype, const c10::Device& dev) { if (!t.defined() || t.numel() < numel || t.scalar_type() != dtype) { t = at::empty({numel}, at::TensorOptions().dtype(dtype).device(dev)); } } void init_streams() { if (!stream_init) { int low_p = 0, high_p = 0; cudaDeviceGetStreamPriorityRange(&low_p, &high_p); cudaStreamCreateWithPriority(&shared_stream_high, cudaStreamNonBlocking, high_p); cudaStreamCreateWithPriority(&shared_stream_low, cudaStreamNonBlocking, low_p); cudaEventCreateWithFlags(&ev_in_ready, cudaEventDisableTiming); cudaEventCreateWithFlags(&ev_sh_done, cudaEventDisableTiming); stream_init = true; } } void prepare_decode(const c10::Device& dev, int64_t n_h) { if (!h_decode.defined() || h_decode.numel() < n_h) { h_decode = at::empty({n_h}, at::TensorOptions().dtype(at::kBFloat16).device(dev)); } } void prepare_general(const c10::Device& dev, int T, int E, int top_k, int H, int I) { int64_t total_tokens = (int64_t)T * top_k; int64_t max_mt = (total_tokens + 63) / 64 + E + 16; ensure(out_fp32, (int64_t)T * H, at::kFloat, dev); ensure(h_routed, total_tokens * I, at::kBFloat16, dev); ensure(h_shared, (int64_t)T * I, at::kBFloat16, dev); ensure(expert_counts, E, at::kInt, dev); ensure(expert_offsets, E + 1, at::kInt, dev); ensure(expert_cursors, E, at::kInt, dev); ensure(emto_a, E + 1, at::kInt, dev); ensure(m2e_a, max_mt, at::kInt, dev); ensure(emto_b, E + 1, at::kInt, dev); ensure(m2e_b, max_mt, at::kInt, dev); ensure(emto_c, E + 1, at::kInt, dev); ensure(m2e_c, max_mt, at::kInt, dev); ensure(sorted_token_ids, total_tokens, at::kInt, dev); ensure(sorted_weights, total_tokens, at::kFloat, dev); } }; static DeviceWorkspace g_ws[16]; static bool g_smem_init = false; inline void init_smem_attrs() { if (g_smem_init) return; constexpr int S96T = 98432; constexpr int S96 = 98304; constexpr int S84T = 86144; constexpr int S84 = 86016; constexpr int S82T = 82048; cudaFuncSetAttribute(shared_gemm1_tma_kernel<128, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(shared_gemm1_tma_kernel<64, 2, 4, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<64, 64, 2, 4, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<128, 64, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<192, 32, 4, 2, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T); cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<128, 64, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96); cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<96, 64, 2, 4, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S84); cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<192, 32, 4, 2, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T); cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, true, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, true, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(gemm2_tma_kernel<64, 64, 2, 4, 4, true, true>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(gemm2_tma_kernel<64, 64, 2, 4, 4, true, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(gemm2_tma_kernel<64, 64, 2, 4, 4, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T); cudaFuncSetAttribute(gemm2_tma_kernel<96, 64, 2, 4, 3, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S84T); cudaFuncSetAttribute(gemm2_tma_kernel<192, 32, 4, 2, 4, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T); g_smem_init = true; } inline void launch_shared_experts( cudaStream_t stream, const torch::Tensor& x, const torch::Tensor& w1_shared, const torch::Tensor& w2_shared, DeviceWorkspace& ws, int T, int n_shared, int H, int I ) { int sBM = (T >= 128) ? 128 : 64; int s_m_tiles = (T + sBM - 1) / sBM; int num_n1 = I / 64, num_n2 = H / 128; constexpr int S96T = 98432; CUtensorMap tmapX = make_tmap_2d(x.data_ptr(), T, H, sBM, 64); CUtensorMap tmapHs = make_tmap_2d(ws.h_shared.data_ptr(), T, I, sBM, 64); for (int s = 0; s < n_shared; ++s) { void* w1s_ptr = reinterpret_cast<__nv_bfloat16*>(w1_shared.data_ptr()) + (size_t)s * (2 * I) * H; void* w2s_ptr = reinterpret_cast<__nv_bfloat16*>(w2_shared.data_ptr()) + (size_t)s * H * I; CUtensorMap tmapW1s = make_tmap_2d(w1s_ptr, 2 * I, H, 64, 64); CUtensorMap tmapW2s = make_tmap_2d(w2s_ptr, H, I, 128, 64); if (sBM == 128) { shared_gemm1_tma_kernel<128, 4, 2, 3><<>>( tmapX, tmapW1s, reinterpret_cast<__nv_bfloat16*>(ws.h_shared.data_ptr()), T, H, I); if (s == 0) { gemm2_tma_kernel<128, 64, 4, 2, 3, true, true><<>>( tmapHs, tmapW2s, ws.out_fp32.data_ptr(), nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I); } else { gemm2_tma_kernel<128, 64, 4, 2, 3, true, false><<>>( tmapHs, tmapW2s, ws.out_fp32.data_ptr(), nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I); } } else { shared_gemm1_tma_kernel<64, 2, 4, 4><<>>( tmapX, tmapW1s, reinterpret_cast<__nv_bfloat16*>(ws.h_shared.data_ptr()), T, H, I); if (s == 0) { gemm2_tma_kernel<64, 64, 2, 4, 4, true, true><<>>( tmapHs, tmapW2s, ws.out_fp32.data_ptr(), nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I); } else { gemm2_tma_kernel<64, 64, 2, 4, 4, true, false><<>>( tmapHs, tmapW2s, ws.out_fp32.data_ptr(), nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I); } } } } } // anonymous namespace torch::Tensor moe_forward_cuda( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared ) { if (!x.is_contiguous()) x = x.contiguous(); if (!expert_ids.is_contiguous()) expert_ids = expert_ids.contiguous(); if (!expert_weights.is_contiguous()) expert_weights = expert_weights.contiguous(); int dev_idx = x.device().index(); c10::cuda::CUDAGuard device_guard(x.device()); cudaStream_t stream = at::cuda::getCurrentCUDAStream(); int T = x.size(0); int H = x.size(1); int E = w1_routed.size(0); int I = w2_routed.size(2); int top_k = expert_ids.size(1); int n_shared = w1_shared.size(0); auto out_bf16 = at::empty({T, H}, x.options()); DeviceWorkspace& ws = g_ws[dev_idx & 15]; // ------------------------------------------------------------------------ // Regime 1: Decode / Tiny-T (T <= 4) // ------------------------------------------------------------------------ if (T <= 4) { int S = n_shared + top_k; ws.prepare_decode(x.device(), (int64_t)T * S * I); decode_gemv1_splitwarp_128t_kernel<<>>( reinterpret_cast(x.data_ptr()), expert_ids.data_ptr(), reinterpret_cast(w1_routed.data_ptr()), reinterpret_cast(w1_shared.data_ptr()), reinterpret_cast<__nv_bfloat16*>(ws.h_decode.data_ptr()), T, top_k, n_shared, H, I ); decode_gemv2_4w_1row_kernel<<>>( reinterpret_cast(ws.h_decode.data_ptr()), expert_ids.data_ptr(), reinterpret_cast(expert_weights.data_ptr()), reinterpret_cast(w2_routed.data_ptr()), reinterpret_cast(w2_shared.data_ptr()), reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()), T, top_k, n_shared, H, I ); return out_bf16; } // ------------------------------------------------------------------------ // Regimes 2, 3, 4: General-T Grouped TMA MoE (Zero CPU Sync) // ------------------------------------------------------------------------ init_smem_attrs(); ws.init_streams(); ws.prepare_general(x.device(), T, E, top_k, H, I); int total_tokens = T * top_k; int num_n1 = I / 64; int num_n2 = H / 128; // Regime selection based on average tokens per expert (general across all T, E, top_k) bool is_small = (total_tokens <= E * 48); // Regime 2: BM=64 (e.g. T=512, 1000) bool is_medium = (!is_small && total_tokens <= E * 180); // Regime 3: [BM=128, BM=192] (e.g. T=4096, 4127) // else: Regime 4: Universal 3-bucket [BM=128, BM=96, BM=192] (e.g. T=8192) bool overlap_shared = (n_shared > 0) && (is_small || is_medium); if (n_shared == 0) { cudaMemsetAsync(ws.out_fp32.data_ptr(), 0, (size_t)T * H * sizeof(float), stream); } else if (overlap_shared) { // Record entry event so shared_stream starts cleanly once inputs are ready, // AFTER routing + routed_gemm1 are enqueued on main stream! cudaEventRecord(ws.ev_in_ready, stream); } else { // Compute-bound Regime 4: run shared sequentially first (zero stream contention, zero memset) launch_shared_experts(stream, x, w1_shared, w2_shared, ws, T, n_shared, H, I); } // 1. GPU Routing on main stream (Zero CPU Sync) cudaMemsetAsync(ws.expert_counts.data_ptr(), 0, (size_t)E * sizeof(int32_t), stream); int hist_blocks = min(256, (total_tokens + 255) / 256); moe_histogram_kernel<<>>( expert_ids.data_ptr(), ws.expert_counts.data_ptr(), total_tokens, E); if (is_small) { moe_prefix_and_schedule_kernel<0><<<1, 256, 0, stream>>>( ws.expert_counts.data_ptr(), ws.expert_offsets.data_ptr(), ws.expert_cursors.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), nullptr, nullptr, nullptr, nullptr, E); } else if (is_medium) { moe_prefix_and_schedule_kernel<1><<<1, 256, 0, stream>>>( ws.expert_counts.data_ptr(), ws.expert_offsets.data_ptr(), ws.expert_cursors.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), nullptr, nullptr, ws.emto_c.data_ptr(), ws.m2e_c.data_ptr(), E); } else { moe_prefix_and_schedule_kernel<2><<<1, 256, 0, stream>>>( ws.expert_counts.data_ptr(), ws.expert_offsets.data_ptr(), ws.expert_cursors.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.emto_b.data_ptr(), ws.m2e_b.data_ptr(), ws.emto_c.data_ptr(), ws.m2e_c.data_ptr(), E); } int scatter_blocks = min(512, (total_tokens + 255) / 256); moe_scatter_kernel<<>>( expert_ids.data_ptr(), reinterpret_cast(expert_weights.data_ptr()), ws.expert_cursors.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), total_tokens, top_k); // 2. Routed GEMM1 + Overlapped Shared + Routed GEMM2 constexpr int S96T = 98432, S96 = 98304, S84T = 86144, S84 = 86016, S82T = 82048; if (is_small) { int max_mt = (total_tokens + 63) / 64 + E; CUtensorMap tmapW1 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 64, false); CUtensorMap tmapW2 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false); CUtensorMap tmapHr = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 64, 64, false); routed_gemm1_hybrid_tma_kernel<64, 64, 2, 4, 4><<>>( reinterpret_cast(x.data_ptr()), tmapW1, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()), ws.expert_offsets.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.sorted_token_ids.data_ptr(), E, H, I); if (overlap_shared) { cudaStreamWaitEvent(ws.shared_stream_high, ws.ev_in_ready, 0); launch_shared_experts(ws.shared_stream_high, x, w1_shared, w2_shared, ws, T, n_shared, H, I); cudaEventRecord(ws.ev_sh_done, ws.shared_stream_high); cudaStreamWaitEvent(stream, ws.ev_sh_done, 0); } gemm2_tma_kernel<64, 64, 2, 4, 4, false, false><<>>( tmapHr, tmapW2, ws.out_fp32.data_ptr(), ws.expert_offsets.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), E, H, I); } else if (is_medium) { // Regime 3: Disjoint [BM=128 (BK=64), BM=192 (BK=32)] with 1-barrier hybrid 3D-TMA (evict_first) int max_mt128 = (total_tokens + 127) / 128 + E; int max_mt192 = min(E, total_tokens / 129) + 1; CUtensorMap tmapW1_64 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 64, false); CUtensorMap tmapW1_32 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 32, true); CUtensorMap tmapW2_64 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false); CUtensorMap tmapW2_32 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 32, true); CUtensorMap tmapH128 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 128, 64, false); CUtensorMap tmapH192 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 192, 32, true); routed_gemm1_hybrid_tma_kernel<128, 64, 4, 2, 3><<>>( reinterpret_cast(x.data_ptr()), tmapW1_64, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()), ws.expert_offsets.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.sorted_token_ids.data_ptr(), E, H, I); routed_gemm1_hybrid_tma_kernel<192, 32, 4, 2, 4><<>>( reinterpret_cast(x.data_ptr()), tmapW1_32, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()), ws.expert_offsets.data_ptr(), ws.emto_c.data_ptr(), ws.m2e_c.data_ptr(), ws.sorted_token_ids.data_ptr(), E, H, I); if (overlap_shared) { cudaStreamWaitEvent(ws.shared_stream_low, ws.ev_in_ready, 0); launch_shared_experts(ws.shared_stream_low, x, w1_shared, w2_shared, ws, T, n_shared, H, I); cudaEventRecord(ws.ev_sh_done, ws.shared_stream_low); cudaStreamWaitEvent(stream, ws.ev_sh_done, 0); } gemm2_tma_kernel<128, 64, 4, 2, 3, false, false><<>>( tmapH128, tmapW2_64, ws.out_fp32.data_ptr(), ws.expert_offsets.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), E, H, I); gemm2_tma_kernel<192, 32, 4, 2, 4, false, false><<>>( tmapH192, tmapW2_32, ws.out_fp32.data_ptr(), ws.expert_offsets.data_ptr(), ws.emto_c.data_ptr(), ws.m2e_c.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), E, H, I); } else { // Regime 4: Universal 3-bucket [BM=128, BM=96, BM=192] (ZERO CPU sync) int max_mt128 = (total_tokens + 127) / 128 + E; int max_mt96 = (total_tokens + 95) / 96 + E; int max_mt192 = total_tokens / 129 + 1; CUtensorMap tmapW2_64 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false); CUtensorMap tmapW2_32 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 32, true); CUtensorMap tmapH128 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 128, 64, false); CUtensorMap tmapH96 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 96, 64, false); CUtensorMap tmapH192 = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 192, 32, true); routed_gemm1_cpasync_kernel<128, 64, 4, 2, 3><<>>( reinterpret_cast(x.data_ptr()), reinterpret_cast(w1_routed.data_ptr()), reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()), ws.expert_offsets.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.sorted_token_ids.data_ptr(), E, H, I); routed_gemm1_cpasync_kernel<96, 64, 2, 4, 3><<>>( reinterpret_cast(x.data_ptr()), reinterpret_cast(w1_routed.data_ptr()), reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()), ws.expert_offsets.data_ptr(), ws.emto_b.data_ptr(), ws.m2e_b.data_ptr(), ws.sorted_token_ids.data_ptr(), E, H, I); routed_gemm1_cpasync_kernel<192, 32, 4, 2, 4><<>>( reinterpret_cast(x.data_ptr()), reinterpret_cast(w1_routed.data_ptr()), reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()), ws.expert_offsets.data_ptr(), ws.emto_c.data_ptr(), ws.m2e_c.data_ptr(), ws.sorted_token_ids.data_ptr(), E, H, I); gemm2_tma_kernel<128, 64, 4, 2, 3, false, false><<>>( tmapH128, tmapW2_64, ws.out_fp32.data_ptr(), ws.expert_offsets.data_ptr(), ws.emto_a.data_ptr(), ws.m2e_a.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), E, H, I); gemm2_tma_kernel<96, 64, 2, 4, 3, false, false><<>>( tmapH96, tmapW2_64, ws.out_fp32.data_ptr(), ws.expert_offsets.data_ptr(), ws.emto_b.data_ptr(), ws.m2e_b.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), E, H, I); gemm2_tma_kernel<192, 32, 4, 2, 4, false, false><<>>( tmapH192, tmapW2_32, ws.out_fp32.data_ptr(), ws.expert_offsets.data_ptr(), ws.emto_c.data_ptr(), ws.m2e_c.data_ptr(), ws.sorted_token_ids.data_ptr(), ws.sorted_weights.data_ptr(), E, H, I); } // 3. Vectorized FP32 -> BF16 cast size_t n_vec8 = ((size_t)T * H) / 8; int cast_blocks = (int)min((size_t)(188 * 8), (n_vec8 + 255) / 256); fp32_to_bf16_vec8_kernel<<>>( ws.out_fp32.data_ptr(), reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()), n_vec8); return out_bf16; } ''' _CPP_SRC = r''' torch::Tensor moe_forward_cuda( torch::Tensor x, torch::Tensor expert_ids, torch::Tensor expert_weights, torch::Tensor w1_routed, torch::Tensor w2_routed, torch::Tensor w1_shared, torch::Tensor w2_shared ); ''' _EXT = None def _get_ext(): global _EXT if _EXT is None: src_hash = hashlib.md5(_CUDA_SRC.encode()).hexdigest()[:10] _EXT = load_inline( name=f"glm52_fused_moe_sm120_v4_{src_hash}", cpp_sources=_CPP_SRC, cuda_sources=_CUDA_SRC, functions=["moe_forward_cuda"], extra_cuda_cflags=["-O3", "-arch=sm_120a"], extra_ldflags=["-lcuda"], verbose=False, ) return _EXT class Model(nn.Module): def __init__( self, T: int, E: int, top_k: int, n_shared: int, H: int, I: int, ): super().__init__() self.T, self.E, self.top_k = T, E, top_k self.n_shared, self.H, self.I = n_shared, H, I self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16)) self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16)) self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16)) self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16)) for p in self.parameters(): nn.init.normal_(p, std=0.02) self._fn = _get_ext().moe_forward_cuda def forward( self, x: torch.Tensor, expert_ids: torch.Tensor, expert_weights: torch.Tensor, ) -> torch.Tensor: return self._fn( x, expert_ids, expert_weights, self.w1_routed, self.w2_routed, self.w1_shared, self.w2_shared, )