KernelBench cuda · RTX PRO 6000

GLM-5.3 Fused MoE Gemini 4 Neon (XHigh)

11.1%geomean peak fraction across shapes

manually audited: clean

Solo rerun of Gemini 4 Neon (XHigh) on GLM-5.3 Fused MoE (following the lock-blocked V1 0.1010 run). Wrote a self-contained 1,977-line SM120 CUDA C++/PTX/TMA kernel suite with zero CPU-GPU synchronization across four adaptive regimes: split-warp 128-thread streaming GEMV for decode (T <= 4); single-barrier hybrid cp.async + 3D-TMA (L2::evict_first when nm==1) with high-priority direct-store shared-expert overlap for small batches; disjoint 2-bucket [BM=128, BM=192] hybrid 3D-TMA with low-priority shared overlap for medium batches; and sequential direct-store shared + universal 3-bucket [BM=128, BM=96, BM=192] schedule minimizing (padded_rows, m_tiles) for large batches, all reducing via SM120 vectorized red.relaxed.gpu.global.add.v4.f32. Lands 0.1105 (#2 overall, +9.4% over V1, fastest on board at T=1, T=512, and T=1000).

harnessagyagent session3h 0mtotal wall3h 11mcheck5mbenchmark5moutput tokens—gpu-lock wait0sgpu-lock held1h 37mregimecompute

Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth

shape 08.829 ms42.0%1.47 TB/s · 82% of 1.8 TB/s HBM · also 210 TFLOPS (42% of compute)
shape 19.069 ms41.2%1.43 TB/s · 80% of 1.8 TB/s HBM · also 206 TFLOPS (41% of compute)
shape 20.299 ms0.3%43.24 TB/s · 100% of 1.8 TB/s HBM · also 2 TFLOPS (0% of compute)
shape 314.215 ms52.2%261 TFLOPS · 52% of 500 TF bf16 peak · also 0.92 TB/s (51% of HBM)
shape 47.921 ms5.9%1.63 TB/s · 91% of 1.8 TB/s HBM · also 29 TFLOPS (6% of compute)
shape 57.993 ms11.3%1.62 TB/s · 90% of 1.8 TB/s HBM · also 57 TFLOPS (11% of compute)

compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)

geomean(42.0% · 41.2% · 0.3% · 52.2% · 5.9% · 11.3%) = 11.0%

Kernel source (redacted)
"""Fast SM120 Blackwell CUDA/PTX/TMA Fused MoE Layer for GLM-5.2 (v4).

Architecture:
  - Self-contained CUDA C++ / inline PTX / 2D & 3D TMA kernels via load_inline.
  - Zero-overhead 7-arg C++ entrypoint with O(1) static per-device workspace cache.
  - 4 adaptive regimes (zero CPU-GPU sync in all regimes):
      1. Decode / Tiny-T (T <= 4):
         Split-warp 128-thread GEMV1 (2 warps/row contiguous streaming ld.global.cs.v4,
         zero SMEM x copy, fused FP32 SwiGLU -> BF16 h) + 4-warp/row GEMV2 (3-slot
         unrolled streaming loads, L1-resident h, 99% SM balance, atomic-free).
      2. Small avg M_e (T*top_k <= E*48, e.g. T=512, 1000):
         BM=64 BK=64 S=4 single-barrier hybrid cp.async + 3D-TMA (SWIZZLE_128B,
         L2::evict_first) GEMM1 + pure 3D-TMA GEMM2; direct-store shared expert
         overlapped on high-priority stream after routed GEMM1 launch (zero memset).
      3. Medium avg M_e (E*48 < T*top_k <= E*180, e.g. T=4096, 4127):
         Disjoint 2-bucket [BM=128 (BK=64, S=3), BM=192 (BK=32, S=4)] GPU schedule
         eliminating padding compute waste and weight re-reads, using 1-barrier
         hybrid cp.async + 3D-TMA (L2::evict_first when nm==1) GEMM1 + pure 3D-TMA
         GEMM2; direct-store shared expert overlapped on low-priority stream after
         routed GEMM1 launch (zero memset).
      4. Large avg M_e (T*top_k > E*180, e.g. T=8192):
         Sequential pure-TMA direct-store shared expert (zero memset, zero contention)
         + universal 3-bucket [BM=128, BM=96, BM=192] GPU schedule minimizing
         (padded_rows, m_tiles) per expert with zero inner-loop branches.
  - GEMM2 epilogue uses SM120 vectorized red.relaxed.gpu.global.add.v4.f32.
"""
from __future__ import annotations

import hashlib
import torch
import torch.nn as nn
from torch.utils.cpp_extension import load_inline

_CUDA_SRC = r'''
#include <cuda.h>
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <ATen/cuda/CUDAContext.h>
#include <c10/cuda/CUDAGuard.h>
#include <stdint.h>
#include <math.h>

namespace {

// ============================================================================
// Low-level SM120 PTX / TMA Primitives
// ============================================================================

__device__ __forceinline__ uint32_t to_smem_addr(const void* ptr) {
    return static_cast<uint32_t>(__cvta_generic_to_shared(ptr));
}

__device__ __forceinline__ void cp_async_16(uint32_t smem_addr, const void* gmem_ptr, int src_bytes) {
    asm volatile(
        "cp.async.cg.shared.global.L2::128B [%0], [%1], 16, %2;\n"
        :: "r"(smem_addr), "l"(gmem_ptr), "r"(src_bytes)
    );
}

__device__ __forceinline__ void cp_async_16_full(uint32_t smem_addr, const void* gmem_ptr) {
    asm volatile(
        "cp.async.cg.shared.global.L2::128B [%0], [%1], 16;\n"
        :: "r"(smem_addr), "l"(gmem_ptr)
    );
}

__device__ __forceinline__ void cp_async_commit() {
    asm volatile("cp.async.commit_group;\n");
}

template <int N>
__device__ __forceinline__ void cp_async_wait() {
    asm volatile("cp.async.wait_group %0;\n" :: "n"(N));
}

__device__ __forceinline__ void mbarrier_init(uint32_t mbar, int count) {
    asm volatile("mbarrier.init.shared::cta.b64 [%0], %1;\n" :: "r"(mbar), "r"(count));
}

__device__ __forceinline__ void mbarrier_arrive_expect_tx(uint32_t mbar, int tx_bytes) {
    asm volatile("mbarrier.arrive.expect_tx.shared::cta.b64 _, [%0], %1;\n" :: "r"(mbar), "r"(tx_bytes));
}

__device__ __forceinline__ void mbarrier_wait(uint32_t mbar, int phase) {
    asm volatile(
        "{\n"
        ".reg .pred P1;\n"
        "LAB_WAIT:\n"
        "mbarrier.try_wait.parity.shared::cta.b64 P1, [%0], %1;\n"
        "@!P1 bra LAB_WAIT;\n"
        "}\n"
        :: "r"(mbar), "r"(phase)
    );
}

__device__ __forceinline__ void tma_load_2d(
    uint32_t smem_addr, const CUtensorMap* tmap, int c0, int c1, uint32_t mbar
) {
    asm volatile(
        "cp.async.bulk.tensor.2d.shared::cta.global.mbarrier::complete_tx::bytes "
        "[%0], [%1, {%2, %3}], [%4];\n"
        :: "r"(smem_addr), "l"(tmap), "r"(c0), "r"(c1), "r"(mbar)
        : "memory"
    );
}

__device__ __forceinline__ void tma_load_3d_hint(
    uint32_t smem_addr, const CUtensorMap* tmap, int c0, int c1, int c2, uint32_t mbar, uint64_t policy
) {
    asm volatile(
        "cp.async.bulk.tensor.3d.shared::cta.global.mbarrier::complete_tx::bytes.L2::cache_hint "
        "[%0], [%1, {%2, %3, %4}], [%5], %6;\n"
        :: "r"(smem_addr), "l"(tmap), "r"(c0), "r"(c1), "r"(c2), "r"(mbar), "l"(policy)
        : "memory"
    );
}

__device__ __forceinline__ void ldmatrix_x4(
    uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t smem_addr
) {
    asm volatile(
        "ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0, %1, %2, %3}, [%4];\n"
        : "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
        : "r"(smem_addr)
    );
}

__device__ __forceinline__ void mma_m16n8k16_bf16(
    float& d0, float& d1, float& d2, float& d3,
    uint32_t a0, uint32_t a1, uint32_t a2, uint32_t a3,
    uint32_t b0, uint32_t b1,
    float c0, float c1, float c2, float c3
) {
    asm volatile(
        "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
        "{%0, %1, %2, %3}, {%4, %5, %6, %7}, {%8, %9}, {%10, %11, %12, %13};\n"
        : "=f"(d0), "=f"(d1), "=f"(d2), "=f"(d3)
        : "r"(a0), "r"(a1), "r"(a2), "r"(a3),
          "r"(b0), "r"(b1),
          "f"(c0), "f"(c1), "f"(c2), "f"(c3)
    );
}

__device__ __forceinline__ uint4 ld_global_cs_v4(const uint4* ptr) {
    uint4 v;
    asm volatile(
        "ld.global.cs.v4.u32 {%0, %1, %2, %3}, [%4];\n"
        : "=r"(v.x), "=r"(v.y), "=r"(v.z), "=r"(v.w)
        : "l"(ptr)
    );
    return v;
}

__device__ __forceinline__ void red_global_add_v4_f32(float* ptr, float4 v) {
    asm volatile(
        "red.relaxed.gpu.global.add.v4.f32 [%0], {%1, %2, %3, %4};\n"
        :: "l"(ptr), "f"(v.x), "f"(v.y), "f"(v.z), "f"(v.w)
    );
}

__device__ __forceinline__ float silu_mul_f32(float g, float u) {
    return (g / (1.0f + expf(-g))) * u;
}

template <int BK>
__device__ __forceinline__ int swizzle_k(int r, int c) {
    if constexpr (BK == 64) {
        return (r * 8 + (c ^ (r & 7))) * 16;          // Matches CU_TENSOR_MAP_SWIZZLE_128B
    } else {
        return (r * 4 + (c ^ ((r >> 1) & 3))) * 16;   // Matches CU_TENSOR_MAP_SWIZZLE_64B
    }
}

// ============================================================================
// Regime 1: Decode / Tiny-T Fused MoE Kernels (T <= 4)
// ============================================================================

__device__ __forceinline__ float dot8_bf16_bf16(uint4 a, uint4 b) {
    const __nv_bfloat162* ap = reinterpret_cast<const __nv_bfloat162*>(&a);
    const __nv_bfloat162* bp = reinterpret_cast<const __nv_bfloat162*>(&b);
    float2 a0 = __bfloat1622float2(ap[0]), b0 = __bfloat1622float2(bp[0]);
    float2 a1 = __bfloat1622float2(ap[1]), b1 = __bfloat1622float2(bp[1]);
    float2 a2 = __bfloat1622float2(ap[2]), b2 = __bfloat1622float2(bp[2]);
    float2 a3 = __bfloat1622float2(ap[3]), b3 = __bfloat1622float2(bp[3]);
    float acc = a0.x * b0.x;
    acc = fmaf(a0.y, b0.y, acc);
    acc = fmaf(a1.x, b1.x, acc);
    acc = fmaf(a1.y, b1.y, acc);
    acc = fmaf(a2.x, b2.x, acc);
    acc = fmaf(a2.y, b2.y, acc);
    acc = fmaf(a3.x, b3.x, acc);
    acc = fmaf(a3.y, b3.y, acc);
    return acc;
}

__device__ __forceinline__ float warp_reduce_sum(float v) {
    #pragma unroll
    for (int offset = 16; offset > 0; offset >>= 1) {
        v += __shfl_down_sync(0xffffffff, v, offset);
    }
    return v;
}

// Split-warp 128-thread GEMV1:
// Warps 0..1 stream gate row i (64t = 1KB coalesced burst x 8 unrolled loads = 8KB);
// Warps 2..3 stream up row I+i simultaneously. Zero SMEM x copy; hits ~1546 GB/s!
__global__ __launch_bounds__(128)
void decode_gemv1_splitwarp_128t_kernel(
    const __nv_bfloat16* __restrict__ x,
    const int64_t* __restrict__ expert_ids,
    const __nv_bfloat16* __restrict__ w1_routed,
    const __nv_bfloat16* __restrict__ w1_shared,
    __nv_bfloat16* __restrict__ h_out,
    int T, int top_k, int n_shared, int H, int I
) {
    __shared__ float s_part[4];
    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;

    int S = n_shared + top_k;
    int i = blockIdx.x % I;
    int rem = blockIdx.x / I;
    int slot = rem % S;
    int t = rem / S;
    if (t >= T) return;

    int is_up = wid >> 1;          // 0: warps 0,1 (gate); 1: warps 2,3 (up)
    int tid64 = (wid & 1) * 32 + lane;

    const __nv_bfloat16* w1_base = (slot < n_shared)
        ? (w1_shared + (size_t)slot * (2 * I) * H)
        : (w1_routed + (size_t)expert_ids[t * top_k + (slot - n_shared)] * (2 * I) * H);

    const uint4* x_vec = reinterpret_cast<const uint4*>(x + (size_t)t * H);
    const uint4* w_vec = reinterpret_cast<const uint4*>(w1_base + (size_t)(is_up * I + i) * H);

    int n_vec = H >> 3;
    float sum = 0.0f;
    int base = 0;
    for (; base + 512 <= n_vec; base += 512) {
        int k = base + tid64;
        uint4 w0 = ld_global_cs_v4(w_vec + k);
        uint4 w1 = ld_global_cs_v4(w_vec + k + 64);
        uint4 w2 = ld_global_cs_v4(w_vec + k + 128);
        uint4 w3 = ld_global_cs_v4(w_vec + k + 192);
        uint4 w4 = ld_global_cs_v4(w_vec + k + 256);
        uint4 w5 = ld_global_cs_v4(w_vec + k + 320);
        uint4 w6 = ld_global_cs_v4(w_vec + k + 384);
        uint4 w7 = ld_global_cs_v4(w_vec + k + 448);

        sum += dot8_bf16_bf16(__ldg(x_vec + k),       w0)
             + dot8_bf16_bf16(__ldg(x_vec + k + 64),  w1)
             + dot8_bf16_bf16(__ldg(x_vec + k + 128), w2)
             + dot8_bf16_bf16(__ldg(x_vec + k + 192), w3)
             + dot8_bf16_bf16(__ldg(x_vec + k + 256), w4)
             + dot8_bf16_bf16(__ldg(x_vec + k + 320), w5)
             + dot8_bf16_bf16(__ldg(x_vec + k + 384), w6)
             + dot8_bf16_bf16(__ldg(x_vec + k + 448), w7);
    }
    for (int k = base + tid64; k < n_vec; k += 64) {
        sum += dot8_bf16_bf16(__ldg(x_vec + k), ld_global_cs_v4(w_vec + k));
    }

    sum = warp_reduce_sum(sum);
    if (lane == 0) s_part[wid] = sum;
    __syncthreads();
    if (tid == 0) {
        h_out[(size_t)(t * S + slot) * I + i] =
            __float2bfloat16_rn(silu_mul_f32(s_part[0] + s_part[1], s_part[2] + s_part[3]));
    }
}

// 4-warp/row (128 threads = 1 output row j) GEMV2:
// 4096 blocks at T=1 -> 99% SM balance, 3-slot unrolled ld.global.cs.v4, L1-resident h!
__global__ __launch_bounds__(128)
void decode_gemv2_4w_1row_kernel(
    const __nv_bfloat16* __restrict__ h,
    const int64_t* __restrict__ expert_ids,
    const __nv_bfloat16* __restrict__ expert_weights,
    const __nv_bfloat16* __restrict__ w2_routed,
    const __nv_bfloat16* __restrict__ w2_shared,
    __nv_bfloat16* __restrict__ out,
    int T, int top_k, int n_shared, int H, int I
) {
    __shared__ const uint4* s_wb[64];
    __shared__ float s_wt[64];
    __shared__ float s_part[4];

    int tid = threadIdx.x;
    int wid = tid >> 5;
    int lane = tid & 31;
    int S = n_shared + top_k;

    int j = blockIdx.x % H;
    int t = blockIdx.x / H;
    if (t >= T) return;

    int n_vec = I >> 3;
    int smem_S = (S <= 64) ? S : 64;
    if (tid < smem_S) {
        if (tid < n_shared) {
            s_wb[tid] = reinterpret_cast<const uint4*>(w2_shared + (size_t)tid * H * I);
            s_wt[tid] = 1.0f;
        } else {
            int ki = tid - n_shared;
            int64_t e = expert_ids[t * top_k + ki];
            s_wb[tid] = reinterpret_cast<const uint4*>(w2_routed + (size_t)e * H * I);
            s_wt[tid] = __bfloat162float(expert_weights[t * top_k + ki]);
        }
    }
    __syncthreads();

    const __nv_bfloat16* h_t = h + (size_t)(t * S) * I;
    size_t row_off = (size_t)j * n_vec;
    float total = 0.0f;

    if (n_vec == 256 && S <= 64) {
        // Fast path for I=2048: 3-slot concurrent streaming loads (6 uint4s in flight per thread!)
        size_t ro = row_off + tid;
        int slot = 0;
        for (; slot + 2 < S; slot += 3) {
            uint4 w0a = ld_global_cs_v4(s_wb[slot]     + ro);
            uint4 w0b = ld_global_cs_v4(s_wb[slot]     + ro + 128);
            uint4 w1a = ld_global_cs_v4(s_wb[slot + 1] + ro);
            uint4 w1b = ld_global_cs_v4(s_wb[slot + 1] + ro + 128);
            uint4 w2a = ld_global_cs_v4(s_wb[slot + 2] + ro);
            uint4 w2b = ld_global_cs_v4(s_wb[slot + 2] + ro + 128);

            const uint4* h0 = reinterpret_cast<const uint4*>(h_t + (size_t)slot * 2048);
            const uint4* h1 = reinterpret_cast<const uint4*>(h_t + (size_t)(slot + 1) * 2048);
            const uint4* h2 = reinterpret_cast<const uint4*>(h_t + (size_t)(slot + 2) * 2048);

            total = fmaf(s_wt[slot],     dot8_bf16_bf16(__ldg(h0 + tid), w0a) + dot8_bf16_bf16(__ldg(h0 + tid + 128), w0b), total);
            total = fmaf(s_wt[slot + 1], dot8_bf16_bf16(__ldg(h1 + tid), w1a) + dot8_bf16_bf16(__ldg(h1 + tid + 128), w1b), total);
            total = fmaf(s_wt[slot + 2], dot8_bf16_bf16(__ldg(h2 + tid), w2a) + dot8_bf16_bf16(__ldg(h2 + tid + 128), w2b), total);
        }
        for (; slot < S; ++slot) {
            uint4 w0a = ld_global_cs_v4(s_wb[slot] + ro);
            uint4 w0b = ld_global_cs_v4(s_wb[slot] + ro + 128);
            const uint4* h0 = reinterpret_cast<const uint4*>(h_t + (size_t)slot * 2048);
            total = fmaf(s_wt[slot], dot8_bf16_bf16(__ldg(h0 + tid), w0a) + dot8_bf16_bf16(__ldg(h0 + tid + 128), w0b), total);
        }
    } else {
        // General I / large S fallback
        for (int slot = 0; slot < S; ++slot) {
            const uint4* wb;
            float wt;
            if (slot < 64) {
                wb = s_wb[slot] + row_off;
                wt = s_wt[slot];
            } else {
                int ki = slot - n_shared;
                wb = reinterpret_cast<const uint4*>(w2_routed + ((size_t)expert_ids[t * top_k + ki] * H + j) * I);
                wt = __bfloat162float(expert_weights[t * top_k + ki]);
            }
            const uint4* hv = reinterpret_cast<const uint4*>(h_t + (size_t)slot * I);
            float d = 0.0f;
            for (int k = tid; k < n_vec; k += 128) {
                d += dot8_bf16_bf16(__ldg(hv + k), ld_global_cs_v4(wb + k));
            }
            total = fmaf(wt, d, total);
        }
    }

    total = warp_reduce_sum(total);
    if (lane == 0) s_part[wid] = total;
    __syncthreads();
    if (tid == 0) {
        out[(size_t)t * H + j] = __float2bfloat16_rn(s_part[0] + s_part[1] + s_part[2] + s_part[3]);
    }
}

// ============================================================================
// GPU Routing & Multi-Bucket O(1) Tile Schedule (Zero CPU Sync)
// ============================================================================

__global__ void moe_histogram_kernel(
    const int64_t* __restrict__ expert_ids,
    int32_t* __restrict__ expert_counts,
    int total_tokens,
    int E
) {
    extern __shared__ int32_t s_counts[];
    for (int e = threadIdx.x; e < E; e += blockDim.x) s_counts[e] = 0;
    __syncthreads();

    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    int stride = gridDim.x * blockDim.x;
    for (int i = idx; i < total_tokens; i += stride) {
        int e = static_cast<int>(expert_ids[i]);
        atomicAdd(&s_counts[e], 1);
    }
    __syncthreads();

    for (int e = threadIdx.x; e < E; e += blockDim.x) {
        int c = s_counts[e];
        if (c > 0) atomicAdd(&expert_counts[e], c);
    }
}

// SCHED_MODE:
//   0 = Single bucket BM=64 (Regime 2: small avg M_e)
//   1 = Disjoint 2-bucket [BM_A=128, BM_C=192] (Regime 3: medium avg M_e)
//   2 = Universal 3-bucket [BM_A=128, BM_B=96, BM_C=192] (Regime 4: large avg M_e)
template <int SCHED_MODE>
__global__ void moe_prefix_and_schedule_kernel(
    const int32_t* __restrict__ expert_counts,
    int32_t* __restrict__ expert_offsets,
    int32_t* __restrict__ expert_cursors,
    int32_t* __restrict__ emto_a,
    int32_t* __restrict__ m2e_a,
    int32_t* __restrict__ emto_b,
    int32_t* __restrict__ m2e_b,
    int32_t* __restrict__ emto_c,
    int32_t* __restrict__ m2e_c,
    int E
) {
    __shared__ int32_t s_wt[8], s_wa[8], s_wb[8], s_wc[8];
    int tid = threadIdx.x, lane = tid & 31, warp = tid >> 5;
    int run_tok = 0, run_a = 0, run_b = 0, run_c = 0;

    for (int base = 0; base < E; base += 256) {
        int e = base + tid;
        int c = (e < E) ? expert_counts[e] : 0;
        int mta = 0, mtb = 0, mtc = 0;
        if (c > 0) {
            if constexpr (SCHED_MODE == 0) {
                mta = (c + 63) >> 6;
            } else if constexpr (SCHED_MODE == 1) {
                if (c > 128 && c <= 192) mtc = 1;
                else                     mta = (c + 127) >> 7;
            } else {
                // Universal mathematical rule: minimize (padded_rows, m_tiles) over {128, 96, 192}
                int t128 = (c + 127) >> 7;
                int t96  = (c + 95) / 96;
                int t192 = (c + 191) / 192;
                int s128 = ((t128 << 7) << 12) + t128;
                int s96  = ((t96  * 96) << 12) + t96;
                int s192 = ((t192 * 192) << 12) + t192;
                if (s192 <= s128 && s192 <= s96) mtc = t192;
                else if (s96 < s128)             mtb = t96;
                else                             mta = t128;
            }
        }

        int wt = c, wa = mta, wb = mtb, wc = mtc;
        #pragma unroll
        for (int off = 1; off < 32; off <<= 1) {
            int tt = __shfl_up_sync(0xffffffff, wt, off);
            int ta = __shfl_up_sync(0xffffffff, wa, off);
            int tb = __shfl_up_sync(0xffffffff, wb, off);
            int tc = __shfl_up_sync(0xffffffff, wc, off);
            if (lane >= off) { wt += tt; wa += ta; wb += tb; wc += tc; }
        }
        if (lane == 31) { s_wt[warp] = wt; s_wa[warp] = wa; s_wb[warp] = wb; s_wc[warp] = wc; }
        __syncthreads();

        if (warp == 0) {
            int bt = (lane < 8) ? s_wt[lane] : 0;
            int ba = (lane < 8) ? s_wa[lane] : 0;
            int bb = (lane < 8) ? s_wb[lane] : 0;
            int bc = (lane < 8) ? s_wc[lane] : 0;
            #pragma unroll
            for (int off = 1; off < 8; off <<= 1) {
                int tt = __shfl_up_sync(0xffffffff, bt, off);
                int ta = __shfl_up_sync(0xffffffff, ba, off);
                int tb = __shfl_up_sync(0xffffffff, bb, off);
                int tc = __shfl_up_sync(0xffffffff, bc, off);
                if (lane >= off) { bt += tt; ba += ta; bb += tb; bc += tc; }
            }
            if (lane < 8) { s_wt[lane] = bt; s_wa[lane] = ba; s_wb[lane] = bb; s_wc[lane] = bc; }
        }
        __syncthreads();

        int ex_tok = run_tok + ((warp > 0) ? s_wt[warp - 1] : 0) + wt - c;
        int ex_a   = run_a   + ((warp > 0) ? s_wa[warp - 1] : 0) + wa - mta;
        int ex_b   = run_b   + ((warp > 0) ? s_wb[warp - 1] : 0) + wb - mtb;
        int ex_c   = run_c   + ((warp > 0) ? s_wc[warp - 1] : 0) + wc - mtc;

        if (e < E) {
            expert_offsets[e] = ex_tok;
            expert_cursors[e] = ex_tok;
            emto_a[e] = ex_a;
            for (int m = 0; m < mta; ++m) m2e_a[ex_a + m] = e;
            if constexpr (SCHED_MODE == 2) {
                emto_b[e] = ex_b;
                for (int m = 0; m < mtb; ++m) m2e_b[ex_b + m] = e;
            }
            if constexpr (SCHED_MODE >= 1) {
                emto_c[e] = ex_c;
                for (int m = 0; m < mtc; ++m) m2e_c[ex_c + m] = e;
            }
        }

        run_tok += s_wt[7];
        run_a   += s_wa[7];
        run_b   += s_wb[7];
        run_c   += s_wc[7];
        __syncthreads();
    }

    if (tid == 0) {
        expert_offsets[E] = run_tok;
        emto_a[E] = run_a;
        if constexpr (SCHED_MODE == 2) emto_b[E] = run_b;
        if constexpr (SCHED_MODE >= 1) emto_c[E] = run_c;
    }
}

__global__ void moe_scatter_kernel(
    const int64_t* __restrict__ expert_ids,
    const __nv_bfloat16* __restrict__ expert_weights,
    int32_t* __restrict__ expert_cursors,
    int32_t* __restrict__ sorted_token_ids,
    float* __restrict__ sorted_weights,
    int total_tokens,
    int top_k
) {
    int idx = blockIdx.x * blockDim.x + threadIdx.x;
    int stride = gridDim.x * blockDim.x;
    for (int i = idx; i < total_tokens; i += stride) {
        int e = static_cast<int>(expert_ids[i]);
        int pos = atomicAdd(&expert_cursors[e], 1);
        sorted_token_ids[pos] = i / top_k;
        sorted_weights[pos] = __bfloat162float(expert_weights[i]);
    }
}

// ============================================================================
// Pure 2D-TMA Shared GEMM1 + Fused SwiGLU (BM=128 or 64, BK=64)
// ============================================================================

template <int BM, int WARPS_M, int WARPS_N, int STAGES>
__global__ __launch_bounds__(256, 1)
void shared_gemm1_tma_kernel(
    const __grid_constant__ CUtensorMap tmapX,
    const __grid_constant__ CUtensorMap tmapW1,
    __nv_bfloat16* __restrict__ h_out,
    int T, int H, int I
) {
    constexpr int BN_HALF = 64, BK = 64;
    constexpr int WARP_M = BM / WARPS_M;
    constexpr int WARP_N = BN_HALF / WARPS_N;
    constexpr int WMT = WARP_M / 16;
    constexpr int WNT2 = WARP_N / 16;
    constexpr int WNT = WARP_N / 8;
    constexpr int STAGE_A = BM * BK * 2;
    constexpr int STAGE_BG = BN_HALF * BK * 2;
    constexpr int STAGE_BYTES = STAGE_A + 2 * STAGE_BG;

    int num_n = I / BN_HALF;
    int m_tile = blockIdx.x / num_n;
    int n_tile = blockIdx.x % num_n;
    int m_start = m_tile * BM;
    int valid_m = min(BM, T - m_start);
    int bn = n_tile * BN_HALF;
    if (valid_m <= 0) return;

    extern __shared__ __align__(128) char smem[];
    uint32_t smem_base = to_smem_addr(smem);
    uint64_t* full_mbar = reinterpret_cast<uint64_t*>(smem + STAGES * STAGE_BYTES);

    int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
    int warp_m = wid / WARPS_N, warp_n = wid % WARPS_N;

    if (tid == 0) {
        #pragma unroll
        for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1);
    }
    __syncthreads();

    int num_k = H / BK;
    if (tid == 0) {
        #pragma unroll
        for (int s = 0; s < STAGES - 1; ++s) {
            if (s < num_k) {
                uint32_t sb = smem_base + s * STAGE_BYTES;
                uint32_t fm = to_smem_addr(&full_mbar[s]);
                mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
                tma_load_2d(sb, &tmapX, s * BK, m_start, fm);
                tma_load_2d(sb + STAGE_A, &tmapW1, s * BK, bn, fm);
                tma_load_2d(sb + STAGE_A + STAGE_BG, &tmapW1, s * BK, I + bn, fm);
            }
        }
    }

    float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4];
    #pragma unroll
    for (int m = 0; m < WMT; ++m)
        #pragma unroll
        for (int n = 0; n < WNT; ++n) {
            acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f;
            acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f;
        }

    int ar = lane & 15, ac = lane >> 4;
    int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
    uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4];

    mbarrier_wait(to_smem_addr(&full_mbar[0]), 0);
    {
        uint32_t rb = smem_base;
        #pragma unroll
        for (int m = 0; m < WMT; ++m)
            ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
                        rb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, ac));
        #pragma unroll
        for (int n = 0; n < WNT2; ++n) {
            int r = warp_n * WARP_N + n * 16 + br;
            ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], rb + STAGE_A + swizzle_k<64>(r, bc));
            ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<64>(r, bc));
        }
    }

    for (int k = 0; k < num_k; ++k) {
        uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
        #pragma unroll
        for (int ks = 0; ks < 4; ++ks) {
            int c = ks & 1, nx = c ^ 1;
            if (ks == 3) {
                __syncthreads();
                int lk = k + STAGES - 1;
                if (tid == 0 && lk < num_k) {
                    int ws = lk % STAGES;
                    uint32_t wb = smem_base + ws * STAGE_BYTES;
                    uint32_t fm = to_smem_addr(&full_mbar[ws]);
                    mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
                    tma_load_2d(wb, &tmapX, lk * BK, m_start, fm);
                    tma_load_2d(wb + STAGE_A, &tmapW1, lk * BK, bn, fm);
                    tma_load_2d(wb + STAGE_A + STAGE_BG, &tmapW1, lk * BK, I + bn, fm);
                }
                if (k + 1 < num_k) {
                    int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1;
                    mbarrier_wait(to_smem_addr(&full_mbar[ns]), np);
                    uint32_t nrb = smem_base + ns * STAGE_BYTES;
                    #pragma unroll
                    for (int m = 0; m < WMT; ++m)
                        ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                    nrb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, ac));
                    #pragma unroll
                    for (int n = 0; n < WNT2; ++n) {
                        int r = warp_n * WARP_N + n * 16 + br;
                        ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<64>(r, bc));
                        ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<64>(r, bc));
                    }
                }
            } else {
                int cb = (ks + 1) * 2;
                #pragma unroll
                for (int m = 0; m < WMT; ++m)
                    ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                rb + swizzle_k<64>(warp_m * WARP_M + m * 16 + ar, cb + ac));
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    int r = warp_n * WARP_N + n * 16 + br;
                    ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<64>(r, cb + bc));
                    ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<64>(r, cb + bc));
                }
            }
            #pragma unroll
            for (int m = 0; m < WMT; ++m) {
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
                    mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
                    mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
                    mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
                }
            }
        }
    }

    __syncthreads();
    constexpr int EPI_STRIDE = 72;
    __nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem);
    int gi = lane >> 2, ti = lane & 3;
    #pragma unroll
    for (int m = 0; m < WMT; ++m) {
        int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
        if (r0 < valid_m) {
            #pragma unroll
            for (int n = 0; n < WNT; ++n) {
                int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                *reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) =
                    __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1]));
            }
        }
        if (r1 < valid_m) {
            #pragma unroll
            for (int n = 0; n < WNT; ++n) {
                int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                *reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) =
                    __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3]));
            }
        }
    }
    __syncthreads();
    int er = tid >> 3, ec = tid & 7;
    for (int r = er; r < valid_m; r += 32) {
        *reinterpret_cast<uint4*>(h_out + (size_t)(m_start + r) * I + bn + ec * 8) =
            *reinterpret_cast<const uint4*>(smem_h + r * EPI_STRIDE + ec * 8);
    }
}

// ============================================================================
// Regimes 2 & 3 Routed GEMM1 (1-Barrier Hybrid cp.async A + 3D-TMA w1 + SwiGLU)
// Single __syncthreads() per K-step + L2::evict_first when nm == 1!
// Supports:
//   BM=64,  BK=64, WM=2, WN=4, STAGES=4 (Regime 2: T=512, 1000 -> ~1650 GB/s!)
//   BM=128, BK=64, WM=4, WN=2, STAGES=3 (Regime 3 bucket A: T=4096, 4127 -> ~1626 GB/s!)
//   BM=192, BK=32, WM=4, WN=2, STAGES=4 (Regime 3 bucket C: T=4096, 4127 -> ~1549 GB/s!)
// ============================================================================

template <int BM, int BK, int WM, int WN, int STAGES>
__global__ __launch_bounds__(256, 1)
void routed_gemm1_hybrid_tma_kernel(
    const __nv_bfloat16* __restrict__ x,
    const __grid_constant__ CUtensorMap tmapW1,
    __nv_bfloat16* __restrict__ h_out,
    const int32_t* __restrict__ expert_offsets,
    const int32_t* __restrict__ emto,
    const int32_t* __restrict__ m2e,
    const int32_t* __restrict__ sorted_token_ids,
    int E, int H, int I
) {
    constexpr int BN_HALF = 64;
    constexpr int WARP_M = BM / WM;
    constexpr int WARP_N = BN_HALF / WN;
    constexpr int WMT = WARP_M / 16;
    constexpr int WNT2 = WARP_N / 16;
    constexpr int WNT = WARP_N / 8;
    constexpr int KSTEPS = BK / 16;
    constexpr int CHUNKS_K = BK / 8;
    constexpr int RPS = 256 / CHUNKS_K;
    constexpr int A_ITERS = BM / RPS;
    constexpr int STAGE_A = BM * BK * 2;
    constexpr int STAGE_BG = BN_HALF * BK * 2;
    constexpr int STAGE_B_BYTES = 2 * STAGE_BG;
    constexpr int STAGE_BYTES = STAGE_A + STAGE_B_BYTES;

    int num_n = I / BN_HALF;
    int total_mt = emto[E];
    int tile_id = blockIdx.x;
    if (tile_id >= total_mt * num_n) return;

    int e = m2e[tile_id / num_n];
    int e_mt0 = emto[e];
    int nm = emto[e + 1] - e_mt0;
    int local = tile_id - e_mt0 * num_n;
    int n_tile = local / nm;
    int m_local = local % nm;
    int e_tok0 = expert_offsets[e];
    int e_cnt = expert_offsets[e + 1] - e_tok0;
    int m_start = e_tok0 + m_local * BM;
    int valid_m = min(BM, e_cnt - m_local * BM);
    int bn = n_tile * BN_HALF;

    extern __shared__ __align__(128) char smem[];
    uint32_t smem_base = to_smem_addr(smem);
    uint64_t* full_mbar = reinterpret_cast<uint64_t*>(smem + STAGES * STAGE_BYTES);

    int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
    int warp_m = wid / WN, warp_n = wid % WN;

    uint64_t b_policy;
    if (nm == 1) asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(b_policy));
    else         asm volatile("createpolicy.fractional.L2::evict_normal.b64 %0, 1.0;" : "=l"(b_policy));

    if (tid == 0) {
        #pragma unroll
        for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1);
    }
    __syncthreads();

    int lr = tid / CHUNKS_K, lc = tid % CHUNKS_K;
    int swz_a[A_ITERS], valid_a[A_ITERS];
    const char* gA[A_ITERS];
    #pragma unroll
    for (int i = 0; i < A_ITERS; ++i) {
        int r = lr + i * RPS;
        swz_a[i] = swizzle_k<BK>(r, lc);
        if (r < valid_m) {
            gA[i] = reinterpret_cast<const char*>(x + (size_t)sorted_token_ids[m_start + r] * H) + lc * 16;
            valid_a[i] = 16;
        } else {
            gA[i] = reinterpret_cast<const char*>(x);
            valid_a[i] = 0;
        }
    }

    int num_k = H / BK;
    #pragma unroll
    for (int s = 0; s < STAGES - 1; ++s) {
        if (s < num_k) {
            uint32_t sb = smem_base + s * STAGE_BYTES;
            size_t koff = (size_t)s * BK * 2;
            #pragma unroll
            for (int i = 0; i < A_ITERS; ++i) cp_async_16(sb + swz_a[i], gA[i] + koff, valid_a[i]);
            if (tid == 0) {
                uint32_t fm = to_smem_addr(&full_mbar[s]);
                mbarrier_arrive_expect_tx(fm, STAGE_B_BYTES);
                tma_load_3d_hint(sb + STAGE_A, &tmapW1, s * BK, bn, e, fm, b_policy);
                tma_load_3d_hint(sb + STAGE_A + STAGE_BG, &tmapW1, s * BK, I + bn, e, fm, b_policy);
            }
        }
        cp_async_commit();
    }

    float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4];
    #pragma unroll
    for (int m = 0; m < WMT; ++m)
        #pragma unroll
        for (int n = 0; n < WNT; ++n) {
            acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f;
            acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f;
        }

    int ar = lane & 15, ac = lane >> 4;
    int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
    uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4];

    cp_async_wait<STAGES - 2>();
    mbarrier_wait(to_smem_addr(&full_mbar[0]), 0);
    __syncthreads();
    #pragma unroll
    for (int m = 0; m < WMT; ++m)
        ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
                    smem_base + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
    #pragma unroll
    for (int n = 0; n < WNT2; ++n) {
        int r = warp_n * WARP_N + n * 16 + br;
        ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], smem_base + STAGE_A + swizzle_k<BK>(r, bc));
        ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], smem_base + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
    }

    for (int k = 0; k < num_k; ++k) {
        int lk = k + STAGES - 1;
        uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
        #pragma unroll
        for (int ks = 0; ks < KSTEPS; ++ks) {
            int c = ks & 1, nx = c ^ 1;
            if (ks == KSTEPS - 1) {
                // Previous iter's __syncthreads() already guaranteed stage (k-1)%STAGES == lk%STAGES is consumed!
                if (lk < num_k) {
                    int ws = lk % STAGES;
                    uint32_t wb = smem_base + ws * STAGE_BYTES;
                    size_t koff = (size_t)lk * BK * 2;
                    #pragma unroll
                    for (int i = 0; i < A_ITERS; ++i) cp_async_16(wb + swz_a[i], gA[i] + koff, valid_a[i]);
                    if (tid == 0) {
                        uint32_t fm = to_smem_addr(&full_mbar[ws]);
                        mbarrier_arrive_expect_tx(fm, STAGE_B_BYTES);
                        tma_load_3d_hint(wb + STAGE_A, &tmapW1, lk * BK, bn, e, fm, b_policy);
                        tma_load_3d_hint(wb + STAGE_A + STAGE_BG, &tmapW1, lk * BK, I + bn, e, fm, b_policy);
                    }
                }
                cp_async_commit();
                if (k + 1 < num_k) {
                    int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1;
                    cp_async_wait<STAGES - 2>();
                    mbarrier_wait(to_smem_addr(&full_mbar[ns]), np);
                    __syncthreads();
                    uint32_t nrb = smem_base + ns * STAGE_BYTES;
                    #pragma unroll
                    for (int m = 0; m < WMT; ++m)
                        ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                    nrb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
                    #pragma unroll
                    for (int n = 0; n < WNT2; ++n) {
                        int r = warp_n * WARP_N + n * 16 + br;
                        ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<BK>(r, bc));
                        ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
                    }
                }
            } else {
                int cb = (ks + 1) * 2;
                #pragma unroll
                for (int m = 0; m < WMT; ++m)
                    ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, cb + ac));
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    int r = warp_n * WARP_N + n * 16 + br;
                    ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<BK>(r, cb + bc));
                    ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, cb + bc));
                }
            }
            #pragma unroll
            for (int m = 0; m < WMT; ++m) {
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
                    mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
                    mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
                    mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
                }
            }
        }
    }

    cp_async_wait<0>();
    __syncthreads();
    constexpr int EPI_STRIDE = 72;
    __nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem);
    int gi = lane >> 2, ti = lane & 3;
    #pragma unroll
    for (int m = 0; m < WMT; ++m) {
        int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
        if (r0 < valid_m) {
            #pragma unroll
            for (int n = 0; n < WNT; ++n) {
                int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                *reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) =
                    __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1]));
            }
        }
        if (r1 < valid_m) {
            #pragma unroll
            for (int n = 0; n < WNT; ++n) {
                int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                *reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) =
                    __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3]));
            }
        }
    }
    __syncthreads();
    int er = tid >> 3, ec = tid & 7;
    for (int r = er; r < valid_m; r += 32) {
        *reinterpret_cast<uint4*>(h_out + (size_t)(m_start + r) * I + bn + ec * 8) =
            *reinterpret_cast<const uint4*>(smem_h + r * EPI_STRIDE + ec * 8);
    }
}

// ============================================================================
// Regime 4 Routed GEMM1 (Zero-Branch Pipelined cp.async + SwiGLU)
// Supports:
//   BM=128, BK=64, WM=4, WN=2, STAGES=3 (Regime 4 bucket A)
//   BM=96,  BK=64, WM=2, WN=4, STAGES=3 (Regime 4 bucket B)
//   BM=192, BK=32, WM=4, WN=2, STAGES=4 (Regime 4 bucket C)
// ============================================================================

template <int BM, int BK, int WM, int WN, int STAGES>
__global__ __launch_bounds__(256, 1)
void routed_gemm1_cpasync_kernel(
    const __nv_bfloat16* __restrict__ x,
    const __nv_bfloat16* __restrict__ w1,
    __nv_bfloat16* __restrict__ h_out,
    const int32_t* __restrict__ expert_offsets,
    const int32_t* __restrict__ emto,
    const int32_t* __restrict__ m2e,
    const int32_t* __restrict__ sorted_token_ids,
    int E, int H, int I
) {
    constexpr int BN_HALF = 64;
    constexpr int WARP_M = BM / WM;
    constexpr int WARP_N = BN_HALF / WN;
    constexpr int WMT = WARP_M / 16;
    constexpr int WNT2 = WARP_N / 16;
    constexpr int WNT = WARP_N / 8;
    constexpr int KSTEPS = BK / 16;
    constexpr int CHUNKS_K = BK / 8;
    constexpr int RPS = 256 / CHUNKS_K;
    constexpr int A_ITERS = BM / RPS;
    constexpr int B_ITERS = BN_HALF / RPS;
    constexpr int STAGE_A = BM * BK * 2;
    constexpr int STAGE_BG = BN_HALF * BK * 2;
    constexpr int STAGE_BYTES = STAGE_A + 2 * STAGE_BG;

    int num_n = I / BN_HALF;
    int total_mt = emto[E];
    int tile_id = blockIdx.x;
    if (tile_id >= total_mt * num_n) return;

    int e = m2e[tile_id / num_n];
    int e_mt0 = emto[e];
    int nm = emto[e + 1] - e_mt0;
    int local = tile_id - e_mt0 * num_n;
    int n_tile = local / nm;
    int m_local = local % nm;
    int e_tok0 = expert_offsets[e];
    int e_cnt = expert_offsets[e + 1] - e_tok0;
    int m_start = e_tok0 + m_local * BM;
    int valid_m = min(BM, e_cnt - m_local * BM);
    int bn = n_tile * BN_HALF;

    extern __shared__ __align__(128) char smem[];
    uint32_t smem_base = to_smem_addr(smem);
    int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
    int warp_m = wid / WN, warp_n = wid % WN;

    int lr = tid / CHUNKS_K, lc = tid % CHUNKS_K;
    int swz_a[A_ITERS], valid_a[A_ITERS];
    const char* gA[A_ITERS];
    #pragma unroll
    for (int i = 0; i < A_ITERS; ++i) {
        int r = lr + i * RPS;
        swz_a[i] = swizzle_k<BK>(r, lc);
        if (r < valid_m) {
            gA[i] = reinterpret_cast<const char*>(x + (size_t)sorted_token_ids[m_start + r] * H) + lc * 16;
            valid_a[i] = 16;
        } else {
            gA[i] = reinterpret_cast<const char*>(x);
            valid_a[i] = 0;
        }
    }

    int swz_bg[B_ITERS], swz_bu[B_ITERS];
    const char* gBg[B_ITERS];
    const char* gBu[B_ITERS];
    const __nv_bfloat16* w1_e = w1 + (size_t)e * (2 * I) * H;
    #pragma unroll
    for (int i = 0; i < B_ITERS; ++i) {
        int r = lr + i * RPS;
        swz_bg[i] = STAGE_A + swizzle_k<BK>(r, lc);
        swz_bu[i] = STAGE_A + STAGE_BG + swizzle_k<BK>(r, lc);
        gBg[i] = reinterpret_cast<const char*>(w1_e + (size_t)(bn + r) * H) + lc * 16;
        gBu[i] = reinterpret_cast<const char*>(w1_e + (size_t)(I + bn + r) * H) + lc * 16;
    }

    int num_k = H / BK;
    #pragma unroll
    for (int s = 0; s < STAGES - 1; ++s) {
        if (s < num_k) {
            uint32_t sb = smem_base + s * STAGE_BYTES;
            size_t koff = (size_t)s * BK * 2;
            #pragma unroll
            for (int i = 0; i < A_ITERS; ++i) cp_async_16(sb + swz_a[i], gA[i] + koff, valid_a[i]);
            #pragma unroll
            for (int i = 0; i < B_ITERS; ++i) {
                cp_async_16_full(sb + swz_bg[i], gBg[i] + koff);
                cp_async_16_full(sb + swz_bu[i], gBu[i] + koff);
            }
        }
        cp_async_commit();
    }

    float acc_g[WMT][WNT][4], acc_u[WMT][WNT][4];
    #pragma unroll
    for (int m = 0; m < WMT; ++m)
        #pragma unroll
        for (int n = 0; n < WNT; ++n) {
            acc_g[m][n][0] = acc_g[m][n][1] = acc_g[m][n][2] = acc_g[m][n][3] = 0.0f;
            acc_u[m][n][0] = acc_u[m][n][1] = acc_u[m][n][2] = acc_u[m][n][3] = 0.0f;
        }

    int ar = lane & 15, ac = lane >> 4;
    int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
    uint32_t aR[2][WMT][4], bgR[2][WNT2][4], buR[2][WNT2][4];

    cp_async_wait<STAGES - 2>();
    __syncthreads();
    #pragma unroll
    for (int m = 0; m < WMT; ++m)
        ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
                    smem_base + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
    #pragma unroll
    for (int n = 0; n < WNT2; ++n) {
        int r = warp_n * WARP_N + n * 16 + br;
        ldmatrix_x4(bgR[0][n][0], bgR[0][n][1], bgR[0][n][2], bgR[0][n][3], smem_base + STAGE_A + swizzle_k<BK>(r, bc));
        ldmatrix_x4(buR[0][n][0], buR[0][n][1], buR[0][n][2], buR[0][n][3], smem_base + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
    }

    for (int k = 0; k < num_k; ++k) {
        int lk = k + STAGES - 1;
        uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
        #pragma unroll
        for (int ks = 0; ks < KSTEPS; ++ks) {
            int c = ks & 1, nx = c ^ 1;
            if (ks == KSTEPS - 1) {
                if (lk < num_k) {
                    uint32_t wb = smem_base + (lk % STAGES) * STAGE_BYTES;
                    size_t koff = (size_t)lk * BK * 2;
                    #pragma unroll
                    for (int i = 0; i < A_ITERS; ++i) cp_async_16(wb + swz_a[i], gA[i] + koff, valid_a[i]);
                    #pragma unroll
                    for (int i = 0; i < B_ITERS; ++i) {
                        cp_async_16_full(wb + swz_bg[i], gBg[i] + koff);
                        cp_async_16_full(wb + swz_bu[i], gBu[i] + koff);
                    }
                }
                cp_async_commit();
                cp_async_wait<STAGES - 2>();
                __syncthreads();
                if (k + 1 < num_k) {
                    uint32_t nrb = smem_base + ((k + 1) % STAGES) * STAGE_BYTES;
                    #pragma unroll
                    for (int m = 0; m < WMT; ++m)
                        ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                    nrb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
                    #pragma unroll
                    for (int n = 0; n < WNT2; ++n) {
                        int r = warp_n * WARP_N + n * 16 + br;
                        ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], nrb + STAGE_A + swizzle_k<BK>(r, bc));
                        ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], nrb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, bc));
                    }
                }
            } else {
                int cb = (ks + 1) * 2;
                #pragma unroll
                for (int m = 0; m < WMT; ++m)
                    ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, cb + ac));
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    int r = warp_n * WARP_N + n * 16 + br;
                    ldmatrix_x4(bgR[nx][n][0], bgR[nx][n][1], bgR[nx][n][2], bgR[nx][n][3], rb + STAGE_A + swizzle_k<BK>(r, cb + bc));
                    ldmatrix_x4(buR[nx][n][0], buR[nx][n][1], buR[nx][n][2], buR[nx][n][3], rb + STAGE_A + STAGE_BG + swizzle_k<BK>(r, cb + bc));
                }
            }
            if constexpr (WMT > WNT2) {
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    #pragma unroll
                    for (int m = 0; m < WMT; ++m) {
                        mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
                        mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
                        mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
                        mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
                    }
                }
            } else {
                #pragma unroll
                for (int m = 0; m < WMT; ++m) {
                    #pragma unroll
                    for (int n = 0; n < WNT2; ++n) {
                        mma_m16n8k16_bf16(acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][0], bgR[c][n][1], acc_g[m][n*2][0], acc_g[m][n*2][1], acc_g[m][n*2][2], acc_g[m][n*2][3]);
                        mma_m16n8k16_bf16(acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bgR[c][n][2], bgR[c][n][3], acc_g[m][n*2+1][0], acc_g[m][n*2+1][1], acc_g[m][n*2+1][2], acc_g[m][n*2+1][3]);
                        mma_m16n8k16_bf16(acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][0], buR[c][n][1], acc_u[m][n*2][0], acc_u[m][n*2][1], acc_u[m][n*2][2], acc_u[m][n*2][3]);
                        mma_m16n8k16_bf16(acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], buR[c][n][2], buR[c][n][3], acc_u[m][n*2+1][0], acc_u[m][n*2+1][1], acc_u[m][n*2+1][2], acc_u[m][n*2+1][3]);
                    }
                }
            }
        }
    }

    cp_async_wait<0>();
    __syncthreads();
    constexpr int EPI_STRIDE = 72;
    __nv_bfloat16* smem_h = reinterpret_cast<__nv_bfloat16*>(smem);
    int gi = lane >> 2, ti = lane & 3;
    #pragma unroll
    for (int m = 0; m < WMT; ++m) {
        int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
        if (r0 < valid_m) {
            #pragma unroll
            for (int n = 0; n < WNT; ++n) {
                int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                *reinterpret_cast<__nv_bfloat162*>(smem_h + r0 * EPI_STRIDE + c0) =
                    __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][0], acc_u[m][n][0]), silu_mul_f32(acc_g[m][n][1], acc_u[m][n][1]));
            }
        }
        if (r1 < valid_m) {
            #pragma unroll
            for (int n = 0; n < WNT; ++n) {
                int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                *reinterpret_cast<__nv_bfloat162*>(smem_h + r1 * EPI_STRIDE + c0) =
                    __floats2bfloat162_rn(silu_mul_f32(acc_g[m][n][2], acc_u[m][n][2]), silu_mul_f32(acc_g[m][n][3], acc_u[m][n][3]));
            }
        }
    }
    __syncthreads();
    int er = tid >> 3, ec = tid & 7;
    for (int r = er; r < valid_m; r += 32) {
        *reinterpret_cast<uint4*>(h_out + (size_t)(m_start + r) * I + bn + ec * 8) =
            *reinterpret_cast<const uint4*>(smem_h + r * EPI_STRIDE + ec * 8);
    }
}

// ============================================================================
// Pure TMA GEMM2 + Weighted Vectorized Global Reduction (or Direct Store)
// Supports:
//   Shared: BM=128 (4x2, BK=64, S=3), BM=64 (2x4, BK=64, S=4)
//   Routed: BM=64 (2x4, BK=64, S=4), BM=128 (4x2, BK=64, S=3),
//           BM=96 (2x4, BK=64, S=3), BM=192 (4x2, BK=32, S=4)
// ============================================================================

template <int BM, int BK, int WM, int WN, int STAGES, bool IS_SHARED, bool DIRECT_STORE>
__global__ __launch_bounds__(256, 1)
void gemm2_tma_kernel(
    const __grid_constant__ CUtensorMap tmapH,
    const __grid_constant__ CUtensorMap tmapW2,
    float* __restrict__ out_fp32,
    const int32_t* __restrict__ expert_offsets,
    const int32_t* __restrict__ emto,
    const int32_t* __restrict__ m2e,
    const int32_t* __restrict__ sorted_token_ids,
    const float* __restrict__ sorted_weights,
    int E_or_T, int H, int I
) {
    constexpr int BN = 128;
    constexpr int WARP_M = BM / WM;
    constexpr int WARP_N = BN / WN;
    constexpr int WMT = WARP_M / 16;
    constexpr int WNT2 = WARP_N / 16;
    constexpr int WNT = WARP_N / 8;
    constexpr int KSTEPS = BK / 16;
    constexpr int STAGE_A = BM * BK * 2;
    constexpr int STAGE_B = BN * BK * 2;
    constexpr int STAGE_BYTES = STAGE_A + STAGE_B;

    int num_n = H / BN;
    int tile_id = blockIdx.x;
    int e = 0, nm = 2, m_start = 0, valid_m = 0, bn = 0;

    if constexpr (IS_SHARED) {
        int m_tile = tile_id / num_n;
        int n_tile = tile_id % num_n;
        m_start = m_tile * BM;
        valid_m = min(BM, E_or_T - m_start);
        bn = n_tile * BN;
        if (valid_m <= 0) return;
    } else {
        int total_mt = emto[E_or_T];
        if (tile_id >= total_mt * num_n) return;
        e = m2e[tile_id / num_n];
        int e_mt0 = emto[e];
        nm = emto[e + 1] - e_mt0;
        int local = tile_id - e_mt0 * num_n;
        int n_tile = local / nm;
        int m_local = local % nm;
        int e_tok0 = expert_offsets[e];
        int e_cnt = expert_offsets[e + 1] - e_tok0;
        m_start = e_tok0 + m_local * BM;
        valid_m = min(BM, e_cnt - m_local * BM);
        bn = n_tile * BN;
    }

    extern __shared__ __align__(128) char smem[];
    uint32_t smem_base = to_smem_addr(smem);
    uint64_t* full_mbar = reinterpret_cast<uint64_t*>(smem + STAGES * STAGE_BYTES);

    int tid = threadIdx.x, wid = tid >> 5, lane = tid & 31;
    int warp_m = wid / WN, warp_n = wid % WN;

    uint64_t b_policy;
    if constexpr (!IS_SHARED) {
        if (nm == 1) asm volatile("createpolicy.fractional.L2::evict_first.b64 %0, 1.0;" : "=l"(b_policy));
        else         asm volatile("createpolicy.fractional.L2::evict_normal.b64 %0, 1.0;" : "=l"(b_policy));
    }

    if (tid == 0) {
        #pragma unroll
        for (int s = 0; s < STAGES; ++s) mbarrier_init(to_smem_addr(&full_mbar[s]), 1);
    }
    __syncthreads();

    int num_k = I / BK;
    if (tid == 0) {
        #pragma unroll
        for (int s = 0; s < STAGES - 1; ++s) {
            if (s < num_k) {
                uint32_t sb = smem_base + s * STAGE_BYTES;
                uint32_t fm = to_smem_addr(&full_mbar[s]);
                mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
                tma_load_2d(sb, &tmapH, s * BK, m_start, fm);
                if constexpr (IS_SHARED) tma_load_2d(sb + STAGE_A, &tmapW2, s * BK, bn, fm);
                else                     tma_load_3d_hint(sb + STAGE_A, &tmapW2, s * BK, bn, e, fm, b_policy);
            }
        }
    }

    float acc[WMT][WNT][4];
    #pragma unroll
    for (int m = 0; m < WMT; ++m)
        #pragma unroll
        for (int n = 0; n < WNT; ++n)
            acc[m][n][0] = acc[m][n][1] = acc[m][n][2] = acc[m][n][3] = 0.0f;

    int ar = lane & 15, ac = lane >> 4;
    int br = (lane & 7) + ((lane >> 4) << 3), bc = (lane >> 3) & 1;
    uint32_t aR[2][WMT][4], bR[2][WNT2][4];

    mbarrier_wait(to_smem_addr(&full_mbar[0]), 0);
    {
        uint32_t rb = smem_base;
        #pragma unroll
        for (int m = 0; m < WMT; ++m)
            ldmatrix_x4(aR[0][m][0], aR[0][m][1], aR[0][m][2], aR[0][m][3],
                        rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
        #pragma unroll
        for (int n = 0; n < WNT2; ++n)
            ldmatrix_x4(bR[0][n][0], bR[0][n][1], bR[0][n][2], bR[0][n][3],
                        rb + STAGE_A + swizzle_k<BK>(warp_n * WARP_N + n * 16 + br, bc));
    }

    for (int k = 0; k < num_k; ++k) {
        uint32_t rb = smem_base + (k % STAGES) * STAGE_BYTES;
        #pragma unroll
        for (int ks = 0; ks < KSTEPS; ++ks) {
            int c = ks & 1, nx = c ^ 1;
            if (ks == KSTEPS - 1) {
                __syncthreads();
                int lk = k + STAGES - 1;
                if (tid == 0 && lk < num_k) {
                    int ws = lk % STAGES;
                    uint32_t wb = smem_base + ws * STAGE_BYTES;
                    uint32_t fm = to_smem_addr(&full_mbar[ws]);
                    mbarrier_arrive_expect_tx(fm, STAGE_BYTES);
                    tma_load_2d(wb, &tmapH, lk * BK, m_start, fm);
                    if constexpr (IS_SHARED) tma_load_2d(wb + STAGE_A, &tmapW2, lk * BK, bn, fm);
                    else                     tma_load_3d_hint(wb + STAGE_A, &tmapW2, lk * BK, bn, e, fm, b_policy);
                }
                if (k + 1 < num_k) {
                    int ns = (k + 1) % STAGES, np = ((k + 1) / STAGES) & 1;
                    mbarrier_wait(to_smem_addr(&full_mbar[ns]), np);
                    uint32_t nrb = smem_base + ns * STAGE_BYTES;
                    #pragma unroll
                    for (int m = 0; m < WMT; ++m)
                        ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                    nrb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, ac));
                    #pragma unroll
                    for (int n = 0; n < WNT2; ++n)
                        ldmatrix_x4(bR[nx][n][0], bR[nx][n][1], bR[nx][n][2], bR[nx][n][3],
                                    nrb + STAGE_A + swizzle_k<BK>(warp_n * WARP_N + n * 16 + br, bc));
                }
            } else {
                int cb = (ks + 1) * 2;
                #pragma unroll
                for (int m = 0; m < WMT; ++m)
                    ldmatrix_x4(aR[nx][m][0], aR[nx][m][1], aR[nx][m][2], aR[nx][m][3],
                                rb + swizzle_k<BK>(warp_m * WARP_M + m * 16 + ar, cb + ac));
                #pragma unroll
                for (int n = 0; n < WNT2; ++n)
                    ldmatrix_x4(bR[nx][n][0], bR[nx][n][1], bR[nx][n][2], bR[nx][n][3],
                                rb + STAGE_A + swizzle_k<BK>(warp_n * WARP_N + n * 16 + br, cb + bc));
            }
            if constexpr (WMT > WNT2) {
                #pragma unroll
                for (int n = 0; n < WNT2; ++n) {
                    #pragma unroll
                    for (int m = 0; m < WMT; ++m) {
                        mma_m16n8k16_bf16(acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][0], bR[c][n][1], acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3]);
                        mma_m16n8k16_bf16(acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][2], bR[c][n][3], acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3]);
                    }
                }
            } else {
                #pragma unroll
                for (int m = 0; m < WMT; ++m) {
                    #pragma unroll
                    for (int n = 0; n < WNT2; ++n) {
                        mma_m16n8k16_bf16(acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][0], bR[c][n][1], acc[m][n*2][0], acc[m][n*2][1], acc[m][n*2][2], acc[m][n*2][3]);
                        mma_m16n8k16_bf16(acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3], aR[c][m][0], aR[c][m][1], aR[c][m][2], aR[c][m][3], bR[c][n][2], bR[c][n][3], acc[m][n*2+1][0], acc[m][n*2+1][1], acc[m][n*2+1][2], acc[m][n*2+1][3]);
                    }
                }
            }
        }
    }

    __syncthreads();
    constexpr int EPI_STRIDE = 136; // 128 + 8 floats (16B aligned for float4!)
    constexpr int EPI_ROWS = (BM <= 128) ? BM : (BM / 2);
    float* smem_f = reinterpret_cast<float*>(smem);
    int32_t* smem_tok = reinterpret_cast<int32_t*>(smem_f + EPI_ROWS * EPI_STRIDE);
    float* smem_wt = reinterpret_cast<float*>(smem_tok + BM);

    if (tid < valid_m) {
        if constexpr (IS_SHARED) {
            smem_tok[tid] = m_start + tid;
            smem_wt[tid] = 1.0f;
        } else {
            smem_tok[tid] = sorted_token_ids[m_start + tid];
            smem_wt[tid] = sorted_weights[m_start + tid];
        }
    }
    __syncthreads();

    int gi = lane >> 2, ti = lane & 3;
    if constexpr (BM <= 128) {
        #pragma unroll
        for (int m = 0; m < WMT; ++m) {
            int r0 = warp_m * WARP_M + m * 16 + gi, r1 = r0 + 8;
            if (r0 < valid_m) {
                float w0 = smem_wt[r0];
                #pragma unroll
                for (int n = 0; n < WNT; ++n) {
                    int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                    smem_f[r0 * EPI_STRIDE + c0]     = acc[m][n][0] * w0;
                    smem_f[r0 * EPI_STRIDE + c0 + 1] = acc[m][n][1] * w0;
                }
            }
            if (r1 < valid_m) {
                float w1 = smem_wt[r1];
                #pragma unroll
                for (int n = 0; n < WNT; ++n) {
                    int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                    smem_f[r1 * EPI_STRIDE + c0]     = acc[m][n][2] * w1;
                    smem_f[r1 * EPI_STRIDE + c0 + 1] = acc[m][n][3] * w1;
                }
            }
        }
        __syncthreads();
        for (int r = wid; r < valid_m; r += 8) {
            int tok = smem_tok[r];
            float4 v = *reinterpret_cast<const float4*>(smem_f + r * EPI_STRIDE + lane * 4);
            if constexpr (DIRECT_STORE) *reinterpret_cast<float4*>(out_fp32 + (size_t)tok * H + bn + lane * 4) = v;
            else                        red_global_add_v4_f32(out_fp32 + (size_t)tok * H + bn + lane * 4, v);
        }
    } else {
        // Two-pass epilogue for BM=192 (96 rows/pass -> 53KB <= 82KB!)
        #pragma unroll
        for (int pass = 0; pass < 2; ++pass) {
            int row_base = pass * EPI_ROWS;
            int pass_valid = min(EPI_ROWS, max(0, valid_m - row_base));
            if (pass_valid <= 0) break;
            if ((warp_m >> 1) == pass) {
                int local_wm = warp_m & 1;
                #pragma unroll
                for (int m = 0; m < WMT; ++m) {
                    int lr0 = local_wm * WARP_M + m * 16 + gi, lr1 = lr0 + 8;
                    int gr0 = row_base + lr0, gr1 = row_base + lr1;
                    if (gr0 < valid_m) {
                        float w0 = smem_wt[gr0];
                        #pragma unroll
                        for (int n = 0; n < WNT; ++n) {
                            int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                            smem_f[lr0 * EPI_STRIDE + c0]     = acc[m][n][0] * w0;
                            smem_f[lr0 * EPI_STRIDE + c0 + 1] = acc[m][n][1] * w0;
                        }
                    }
                    if (gr1 < valid_m) {
                        float w1 = smem_wt[gr1];
                        #pragma unroll
                        for (int n = 0; n < WNT; ++n) {
                            int c0 = warp_n * WARP_N + n * 8 + ti * 2;
                            smem_f[lr1 * EPI_STRIDE + c0]     = acc[m][n][2] * w1;
                            smem_f[lr1 * EPI_STRIDE + c0 + 1] = acc[m][n][3] * w1;
                        }
                    }
                }
            }
            __syncthreads();
            for (int r = wid; r < pass_valid; r += 8) {
                int tok = smem_tok[row_base + r];
                float4 v = *reinterpret_cast<const float4*>(smem_f + r * EPI_STRIDE + lane * 4);
                if constexpr (DIRECT_STORE) *reinterpret_cast<float4*>(out_fp32 + (size_t)tok * H + bn + lane * 4) = v;
                else                        red_global_add_v4_f32(out_fp32 + (size_t)tok * H + bn + lane * 4, v);
            }
            if (pass == 0) __syncthreads();
        }
    }
}

// ============================================================================
// Vectorized FP32 -> BF16 Cast
// ============================================================================

__global__ void fp32_to_bf16_vec8_kernel(
    const float* __restrict__ src,
    __nv_bfloat16* __restrict__ dst,
    size_t n_vec8
) {
    size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
    size_t stride = (size_t)gridDim.x * blockDim.x;
    const float4* src4 = reinterpret_cast<const float4*>(src);
    uint4* dst8 = reinterpret_cast<uint4*>(dst);
    for (size_t i = idx; i < n_vec8; i += stride) {
        float4 a = src4[i * 2];
        float4 b = src4[i * 2 + 1];
        __nv_bfloat162 r0 = __floats2bfloat162_rn(a.x, a.y);
        __nv_bfloat162 r1 = __floats2bfloat162_rn(a.z, a.w);
        __nv_bfloat162 r2 = __floats2bfloat162_rn(b.x, b.y);
        __nv_bfloat162 r3 = __floats2bfloat162_rn(b.z, b.w);
        uint4 packed;
        packed.x = *reinterpret_cast<uint32_t*>(&r0);
        packed.y = *reinterpret_cast<uint32_t*>(&r1);
        packed.z = *reinterpret_cast<uint32_t*>(&r2);
        packed.w = *reinterpret_cast<uint32_t*>(&r3);
        dst8[i] = packed;
    }
}

// ============================================================================
// TMA Helpers, Dual-Priority Streams, and O(1) Per-Device Workspace Cache
// ============================================================================

inline CUtensorMap make_tmap_2d(void* ptr, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols, bool swz64 = false) {
    CUtensorMap tmap;
    uint64_t dims[2] = {cols, rows};
    uint64_t strides[1] = {cols * 2};
    uint32_t box[2] = {box_cols, box_rows};
    uint32_t es[2] = {1, 1};
    CUresult res = cuTensorMapEncodeTiled(
        &tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 2, ptr, dims, strides, box, es,
        CU_TENSOR_MAP_INTERLEAVE_NONE,
        swz64 ? CU_TENSOR_MAP_SWIZZLE_64B : CU_TENSOR_MAP_SWIZZLE_128B,
        CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
    );
    TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 2D failed: ", (int)res);
    return tmap;
}

inline CUtensorMap make_tmap_3d(void* ptr, uint64_t E, uint64_t rows, uint64_t cols, uint32_t box_rows, uint32_t box_cols, bool swz64 = false) {
    CUtensorMap tmap;
    uint64_t dims[3] = {cols, rows, E};
    uint64_t strides[2] = {cols * 2, rows * cols * 2};
    uint32_t box[3] = {box_cols, box_rows, 1};
    uint32_t es[3] = {1, 1, 1};
    CUresult res = cuTensorMapEncodeTiled(
        &tmap, CU_TENSOR_MAP_DATA_TYPE_BFLOAT16, 3, ptr, dims, strides, box, es,
        CU_TENSOR_MAP_INTERLEAVE_NONE,
        swz64 ? CU_TENSOR_MAP_SWIZZLE_64B : CU_TENSOR_MAP_SWIZZLE_128B,
        CU_TENSOR_MAP_L2_PROMOTION_L2_128B, CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE
    );
    TORCH_CHECK(res == CUDA_SUCCESS, "cuTensorMapEncodeTiled 3D failed: ", (int)res);
    return tmap;
}

struct DeviceWorkspace {
    torch::Tensor out_fp32;
    torch::Tensor h_routed;
    torch::Tensor h_shared;
    torch::Tensor h_decode;
    torch::Tensor expert_counts;
    torch::Tensor expert_offsets;
    torch::Tensor expert_cursors;
    torch::Tensor emto_a, m2e_a;
    torch::Tensor emto_b, m2e_b;
    torch::Tensor emto_c, m2e_c;
    torch::Tensor sorted_token_ids;
    torch::Tensor sorted_weights;

    cudaStream_t shared_stream_high = nullptr;
    cudaStream_t shared_stream_low  = nullptr;
    cudaEvent_t ev_in_ready = nullptr;
    cudaEvent_t ev_sh_done  = nullptr;
    bool stream_init = false;

    static void ensure(torch::Tensor& t, int64_t numel, c10::ScalarType dtype, const c10::Device& dev) {
        if (!t.defined() || t.numel() < numel || t.scalar_type() != dtype) {
            t = at::empty({numel}, at::TensorOptions().dtype(dtype).device(dev));
        }
    }

    void init_streams() {
        if (!stream_init) {
            int low_p = 0, high_p = 0;
            cudaDeviceGetStreamPriorityRange(&low_p, &high_p);
            cudaStreamCreateWithPriority(&shared_stream_high, cudaStreamNonBlocking, high_p);
            cudaStreamCreateWithPriority(&shared_stream_low,  cudaStreamNonBlocking, low_p);
            cudaEventCreateWithFlags(&ev_in_ready, cudaEventDisableTiming);
            cudaEventCreateWithFlags(&ev_sh_done,  cudaEventDisableTiming);
            stream_init = true;
        }
    }

    void prepare_decode(const c10::Device& dev, int64_t n_h) {
        if (!h_decode.defined() || h_decode.numel() < n_h) {
            h_decode = at::empty({n_h}, at::TensorOptions().dtype(at::kBFloat16).device(dev));
        }
    }

    void prepare_general(const c10::Device& dev, int T, int E, int top_k, int H, int I) {
        int64_t total_tokens = (int64_t)T * top_k;
        int64_t max_mt = (total_tokens + 63) / 64 + E + 16;
        ensure(out_fp32, (int64_t)T * H, at::kFloat, dev);
        ensure(h_routed, total_tokens * I, at::kBFloat16, dev);
        ensure(h_shared, (int64_t)T * I, at::kBFloat16, dev);
        ensure(expert_counts, E, at::kInt, dev);
        ensure(expert_offsets, E + 1, at::kInt, dev);
        ensure(expert_cursors, E, at::kInt, dev);
        ensure(emto_a, E + 1, at::kInt, dev);
        ensure(m2e_a, max_mt, at::kInt, dev);
        ensure(emto_b, E + 1, at::kInt, dev);
        ensure(m2e_b, max_mt, at::kInt, dev);
        ensure(emto_c, E + 1, at::kInt, dev);
        ensure(m2e_c, max_mt, at::kInt, dev);
        ensure(sorted_token_ids, total_tokens, at::kInt, dev);
        ensure(sorted_weights, total_tokens, at::kFloat, dev);
    }
};

static DeviceWorkspace g_ws[16];
static bool g_smem_init = false;

inline void init_smem_attrs() {
    if (g_smem_init) return;
    constexpr int S96T = 98432;
    constexpr int S96  = 98304;
    constexpr int S84T = 86144;
    constexpr int S84  = 86016;
    constexpr int S82T = 82048;

    cudaFuncSetAttribute(shared_gemm1_tma_kernel<128, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(shared_gemm1_tma_kernel<64,  2, 4, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);

    cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<64,  64, 2, 4, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<128, 64, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(routed_gemm1_hybrid_tma_kernel<192, 32, 4, 2, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T);

    cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<128, 64, 4, 2, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96);
    cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<96,  64, 2, 4, 3>, cudaFuncAttributeMaxDynamicSharedMemorySize, S84);
    cudaFuncSetAttribute(routed_gemm1_cpasync_kernel<192, 32, 4, 2, 4>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T);

    cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, true,  true>,  cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, true,  false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(gemm2_tma_kernel<64,  64, 2, 4, 4, true,  true>,  cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(gemm2_tma_kernel<64,  64, 2, 4, 4, true,  false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);

    cudaFuncSetAttribute(gemm2_tma_kernel<64,  64, 2, 4, 4, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(gemm2_tma_kernel<128, 64, 4, 2, 3, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S96T);
    cudaFuncSetAttribute(gemm2_tma_kernel<96,  64, 2, 4, 3, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S84T);
    cudaFuncSetAttribute(gemm2_tma_kernel<192, 32, 4, 2, 4, false, false>, cudaFuncAttributeMaxDynamicSharedMemorySize, S82T);
    g_smem_init = true;
}

inline void launch_shared_experts(
    cudaStream_t stream,
    const torch::Tensor& x,
    const torch::Tensor& w1_shared,
    const torch::Tensor& w2_shared,
    DeviceWorkspace& ws,
    int T, int n_shared, int H, int I
) {
    int sBM = (T >= 128) ? 128 : 64;
    int s_m_tiles = (T + sBM - 1) / sBM;
    int num_n1 = I / 64, num_n2 = H / 128;
    constexpr int S96T = 98432;
    CUtensorMap tmapX = make_tmap_2d(x.data_ptr(), T, H, sBM, 64);
    CUtensorMap tmapHs = make_tmap_2d(ws.h_shared.data_ptr(), T, I, sBM, 64);

    for (int s = 0; s < n_shared; ++s) {
        void* w1s_ptr = reinterpret_cast<__nv_bfloat16*>(w1_shared.data_ptr()) + (size_t)s * (2 * I) * H;
        void* w2s_ptr = reinterpret_cast<__nv_bfloat16*>(w2_shared.data_ptr()) + (size_t)s * H * I;
        CUtensorMap tmapW1s = make_tmap_2d(w1s_ptr, 2 * I, H, 64, 64);
        CUtensorMap tmapW2s = make_tmap_2d(w2s_ptr, H, I, 128, 64);

        if (sBM == 128) {
            shared_gemm1_tma_kernel<128, 4, 2, 3><<<s_m_tiles * num_n1, 256, S96T, stream>>>(
                tmapX, tmapW1s, reinterpret_cast<__nv_bfloat16*>(ws.h_shared.data_ptr()), T, H, I);
            if (s == 0) {
                gemm2_tma_kernel<128, 64, 4, 2, 3, true, true><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
                    tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
                    nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
            } else {
                gemm2_tma_kernel<128, 64, 4, 2, 3, true, false><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
                    tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
                    nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
            }
        } else {
            shared_gemm1_tma_kernel<64, 2, 4, 4><<<s_m_tiles * num_n1, 256, S96T, stream>>>(
                tmapX, tmapW1s, reinterpret_cast<__nv_bfloat16*>(ws.h_shared.data_ptr()), T, H, I);
            if (s == 0) {
                gemm2_tma_kernel<64, 64, 2, 4, 4, true, true><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
                    tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
                    nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
            } else {
                gemm2_tma_kernel<64, 64, 2, 4, 4, true, false><<<s_m_tiles * num_n2, 256, S96T, stream>>>(
                    tmapHs, tmapW2s, ws.out_fp32.data_ptr<float>(),
                    nullptr, nullptr, nullptr, nullptr, nullptr, T, H, I);
            }
        }
    }
}

} // anonymous namespace

torch::Tensor moe_forward_cuda(
    torch::Tensor x,
    torch::Tensor expert_ids,
    torch::Tensor expert_weights,
    torch::Tensor w1_routed,
    torch::Tensor w2_routed,
    torch::Tensor w1_shared,
    torch::Tensor w2_shared
) {
    if (!x.is_contiguous()) x = x.contiguous();
    if (!expert_ids.is_contiguous()) expert_ids = expert_ids.contiguous();
    if (!expert_weights.is_contiguous()) expert_weights = expert_weights.contiguous();

    int dev_idx = x.device().index();
    c10::cuda::CUDAGuard device_guard(x.device());
    cudaStream_t stream = at::cuda::getCurrentCUDAStream();

    int T = x.size(0);
    int H = x.size(1);
    int E = w1_routed.size(0);
    int I = w2_routed.size(2);
    int top_k = expert_ids.size(1);
    int n_shared = w1_shared.size(0);

    auto out_bf16 = at::empty({T, H}, x.options());
    DeviceWorkspace& ws = g_ws[dev_idx & 15];

    // ------------------------------------------------------------------------
    // Regime 1: Decode / Tiny-T (T <= 4)
    // ------------------------------------------------------------------------
    if (T <= 4) {
        int S = n_shared + top_k;
        ws.prepare_decode(x.device(), (int64_t)T * S * I);

        decode_gemv1_splitwarp_128t_kernel<<<T * S * I, 128, 0, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            expert_ids.data_ptr<int64_t>(),
            reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w1_shared.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(ws.h_decode.data_ptr()),
            T, top_k, n_shared, H, I
        );

        decode_gemv2_4w_1row_kernel<<<T * H, 128, 0, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(ws.h_decode.data_ptr()),
            expert_ids.data_ptr<int64_t>(),
            reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w2_routed.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w2_shared.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()),
            T, top_k, n_shared, H, I
        );
        return out_bf16;
    }

    // ------------------------------------------------------------------------
    // Regimes 2, 3, 4: General-T Grouped TMA MoE (Zero CPU Sync)
    // ------------------------------------------------------------------------
    init_smem_attrs();
    ws.init_streams();
    ws.prepare_general(x.device(), T, E, top_k, H, I);

    int total_tokens = T * top_k;
    int num_n1 = I / 64;
    int num_n2 = H / 128;

    // Regime selection based on average tokens per expert (general across all T, E, top_k)
    bool is_small  = (total_tokens <= E * 48);                // Regime 2: BM=64 (e.g. T=512, 1000)
    bool is_medium = (!is_small && total_tokens <= E * 180);  // Regime 3: [BM=128, BM=192] (e.g. T=4096, 4127)
    // else: Regime 4: Universal 3-bucket [BM=128, BM=96, BM=192] (e.g. T=8192)

    bool overlap_shared = (n_shared > 0) && (is_small || is_medium);

    if (n_shared == 0) {
        cudaMemsetAsync(ws.out_fp32.data_ptr(), 0, (size_t)T * H * sizeof(float), stream);
    } else if (overlap_shared) {
        // Record entry event so shared_stream starts cleanly once inputs are ready,
        // AFTER routing + routed_gemm1 are enqueued on main stream!
        cudaEventRecord(ws.ev_in_ready, stream);
    } else {
        // Compute-bound Regime 4: run shared sequentially first (zero stream contention, zero memset)
        launch_shared_experts(stream, x, w1_shared, w2_shared, ws, T, n_shared, H, I);
    }

    // 1. GPU Routing on main stream (Zero CPU Sync)
    cudaMemsetAsync(ws.expert_counts.data_ptr(), 0, (size_t)E * sizeof(int32_t), stream);

    int hist_blocks = min(256, (total_tokens + 255) / 256);
    moe_histogram_kernel<<<hist_blocks, 256, E * sizeof(int32_t), stream>>>(
        expert_ids.data_ptr<int64_t>(), ws.expert_counts.data_ptr<int32_t>(), total_tokens, E);

    if (is_small) {
        moe_prefix_and_schedule_kernel<0><<<1, 256, 0, stream>>>(
            ws.expert_counts.data_ptr<int32_t>(), ws.expert_offsets.data_ptr<int32_t>(),
            ws.expert_cursors.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), nullptr, nullptr, nullptr, nullptr, E);
    } else if (is_medium) {
        moe_prefix_and_schedule_kernel<1><<<1, 256, 0, stream>>>(
            ws.expert_counts.data_ptr<int32_t>(), ws.expert_offsets.data_ptr<int32_t>(),
            ws.expert_cursors.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), nullptr, nullptr,
            ws.emto_c.data_ptr<int32_t>(), ws.m2e_c.data_ptr<int32_t>(), E);
    } else {
        moe_prefix_and_schedule_kernel<2><<<1, 256, 0, stream>>>(
            ws.expert_counts.data_ptr<int32_t>(), ws.expert_offsets.data_ptr<int32_t>(),
            ws.expert_cursors.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.emto_b.data_ptr<int32_t>(),
            ws.m2e_b.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
            ws.m2e_c.data_ptr<int32_t>(), E);
    }

    int scatter_blocks = min(512, (total_tokens + 255) / 256);
    moe_scatter_kernel<<<scatter_blocks, 256, 0, stream>>>(
        expert_ids.data_ptr<int64_t>(),
        reinterpret_cast<const __nv_bfloat16*>(expert_weights.data_ptr()),
        ws.expert_cursors.data_ptr<int32_t>(),
        ws.sorted_token_ids.data_ptr<int32_t>(),
        ws.sorted_weights.data_ptr<float>(),
        total_tokens, top_k);

    // 2. Routed GEMM1 + Overlapped Shared + Routed GEMM2
    constexpr int S96T = 98432, S96 = 98304, S84T = 86144, S84 = 86016, S82T = 82048;

    if (is_small) {
        int max_mt = (total_tokens + 63) / 64 + E;
        CUtensorMap tmapW1 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 64, false);
        CUtensorMap tmapW2 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false);
        CUtensorMap tmapHr = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 64, 64, false);

        routed_gemm1_hybrid_tma_kernel<64, 64, 2, 4, 4><<<max_mt * num_n1, 256, S96T, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            tmapW1, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);

        if (overlap_shared) {
            cudaStreamWaitEvent(ws.shared_stream_high, ws.ev_in_ready, 0);
            launch_shared_experts(ws.shared_stream_high, x, w1_shared, w2_shared, ws, T, n_shared, H, I);
            cudaEventRecord(ws.ev_sh_done, ws.shared_stream_high);
            cudaStreamWaitEvent(stream, ws.ev_sh_done, 0);
        }

        gemm2_tma_kernel<64, 64, 2, 4, 4, false, false><<<max_mt * num_n2, 256, S96T, stream>>>(
            tmapHr, tmapW2, ws.out_fp32.data_ptr<float>(),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
            ws.sorted_weights.data_ptr<float>(), E, H, I);
    } else if (is_medium) {
        // Regime 3: Disjoint [BM=128 (BK=64), BM=192 (BK=32)] with 1-barrier hybrid 3D-TMA (evict_first)
        int max_mt128 = (total_tokens + 127) / 128 + E;
        int max_mt192 = min(E, total_tokens / 129) + 1;

        CUtensorMap tmapW1_64 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 64, false);
        CUtensorMap tmapW1_32 = make_tmap_3d(w1_routed.data_ptr(), E, 2 * I, H, 64, 32, true);
        CUtensorMap tmapW2_64 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false);
        CUtensorMap tmapW2_32 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 32, true);
        CUtensorMap tmapH128  = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 128, 64, false);
        CUtensorMap tmapH192  = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 192, 32, true);

        routed_gemm1_hybrid_tma_kernel<128, 64, 4, 2, 3><<<max_mt128 * num_n1, 256, S96T, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            tmapW1_64, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);

        routed_gemm1_hybrid_tma_kernel<192, 32, 4, 2, 4><<<max_mt192 * num_n1, 256, S82T, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            tmapW1_32, reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
            ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);

        if (overlap_shared) {
            cudaStreamWaitEvent(ws.shared_stream_low, ws.ev_in_ready, 0);
            launch_shared_experts(ws.shared_stream_low, x, w1_shared, w2_shared, ws, T, n_shared, H, I);
            cudaEventRecord(ws.ev_sh_done, ws.shared_stream_low);
            cudaStreamWaitEvent(stream, ws.ev_sh_done, 0);
        }

        gemm2_tma_kernel<128, 64, 4, 2, 3, false, false><<<max_mt128 * num_n2, 256, S96T, stream>>>(
            tmapH128, tmapW2_64, ws.out_fp32.data_ptr<float>(),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
            ws.sorted_weights.data_ptr<float>(), E, H, I);

        gemm2_tma_kernel<192, 32, 4, 2, 4, false, false><<<max_mt192 * num_n2, 256, S82T, stream>>>(
            tmapH192, tmapW2_32, ws.out_fp32.data_ptr<float>(),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
            ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
            ws.sorted_weights.data_ptr<float>(), E, H, I);
    } else {
        // Regime 4: Universal 3-bucket [BM=128, BM=96, BM=192] (ZERO CPU sync)
        int max_mt128 = (total_tokens + 127) / 128 + E;
        int max_mt96  = (total_tokens + 95) / 96 + E;
        int max_mt192 = total_tokens / 129 + 1;

        CUtensorMap tmapW2_64 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 64, false);
        CUtensorMap tmapW2_32 = make_tmap_3d(w2_routed.data_ptr(), E, H, I, 128, 32, true);
        CUtensorMap tmapH128  = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 128, 64, false);
        CUtensorMap tmapH96   = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 96,  64, false);
        CUtensorMap tmapH192  = make_tmap_2d(ws.h_routed.data_ptr(), total_tokens, I, 192, 32, true);

        routed_gemm1_cpasync_kernel<128, 64, 4, 2, 3><<<max_mt128 * num_n1, 256, S96, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);

        routed_gemm1_cpasync_kernel<96, 64, 2, 4, 3><<<max_mt96 * num_n1, 256, S84, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_b.data_ptr<int32_t>(),
            ws.m2e_b.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);

        routed_gemm1_cpasync_kernel<192, 32, 4, 2, 4><<<max_mt192 * num_n1, 256, S82T, stream>>>(
            reinterpret_cast<const __nv_bfloat16*>(x.data_ptr()),
            reinterpret_cast<const __nv_bfloat16*>(w1_routed.data_ptr()),
            reinterpret_cast<__nv_bfloat16*>(ws.h_routed.data_ptr()),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
            ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(), E, H, I);

        gemm2_tma_kernel<128, 64, 4, 2, 3, false, false><<<max_mt128 * num_n2, 256, S96T, stream>>>(
            tmapH128, tmapW2_64, ws.out_fp32.data_ptr<float>(),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_a.data_ptr<int32_t>(),
            ws.m2e_a.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
            ws.sorted_weights.data_ptr<float>(), E, H, I);

        gemm2_tma_kernel<96, 64, 2, 4, 3, false, false><<<max_mt96 * num_n2, 256, S84T, stream>>>(
            tmapH96, tmapW2_64, ws.out_fp32.data_ptr<float>(),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_b.data_ptr<int32_t>(),
            ws.m2e_b.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
            ws.sorted_weights.data_ptr<float>(), E, H, I);

        gemm2_tma_kernel<192, 32, 4, 2, 4, false, false><<<max_mt192 * num_n2, 256, S82T, stream>>>(
            tmapH192, tmapW2_32, ws.out_fp32.data_ptr<float>(),
            ws.expert_offsets.data_ptr<int32_t>(), ws.emto_c.data_ptr<int32_t>(),
            ws.m2e_c.data_ptr<int32_t>(), ws.sorted_token_ids.data_ptr<int32_t>(),
            ws.sorted_weights.data_ptr<float>(), E, H, I);
    }

    // 3. Vectorized FP32 -> BF16 cast
    size_t n_vec8 = ((size_t)T * H) / 8;
    int cast_blocks = (int)min((size_t)(188 * 8), (n_vec8 + 255) / 256);
    fp32_to_bf16_vec8_kernel<<<cast_blocks, 256, 0, stream>>>(
        ws.out_fp32.data_ptr<float>(),
        reinterpret_cast<__nv_bfloat16*>(out_bf16.data_ptr()),
        n_vec8);

    return out_bf16;
}
'''

_CPP_SRC = r'''
torch::Tensor moe_forward_cuda(
    torch::Tensor x,
    torch::Tensor expert_ids,
    torch::Tensor expert_weights,
    torch::Tensor w1_routed,
    torch::Tensor w2_routed,
    torch::Tensor w1_shared,
    torch::Tensor w2_shared
);
'''

_EXT = None


def _get_ext():
    global _EXT
    if _EXT is None:
        src_hash = hashlib.md5(_CUDA_SRC.encode()).hexdigest()[:10]
        _EXT = load_inline(
            name=f"glm52_fused_moe_sm120_v4_{src_hash}",
            cpp_sources=_CPP_SRC,
            cuda_sources=_CUDA_SRC,
            functions=["moe_forward_cuda"],
            extra_cuda_cflags=["-O3", "-arch=sm_120a"],
            extra_ldflags=["-lcuda"],
            verbose=False,
        )
    return _EXT


class Model(nn.Module):
    def __init__(
        self,
        T: int,
        E: int,
        top_k: int,
        n_shared: int,
        H: int,
        I: int,
    ):
        super().__init__()
        self.T, self.E, self.top_k = T, E, top_k
        self.n_shared, self.H, self.I = n_shared, H, I
        self.w1_routed = nn.Parameter(torch.empty(E, 2 * I, H, dtype=torch.bfloat16))
        self.w2_routed = nn.Parameter(torch.empty(E, H, I, dtype=torch.bfloat16))
        self.w1_shared = nn.Parameter(torch.empty(n_shared, 2 * I, H, dtype=torch.bfloat16))
        self.w2_shared = nn.Parameter(torch.empty(n_shared, H, I, dtype=torch.bfloat16))
        for p in self.parameters():
            nn.init.normal_(p, std=0.02)
        self._fn = _get_ext().moe_forward_cuda

    def forward(
        self,
        x: torch.Tensor,
        expert_ids: torch.Tensor,
        expert_weights: torch.Tensor,
    ) -> torch.Tensor:
        return self._fn(
            x,
            expert_ids,
            expert_weights,
            self.w1_routed,
            self.w2_routed,
            self.w1_shared,
            self.w2_shared,
        )

20261119_123612_agy_gemini-4-neon-xhigh_01_glm52_fused_moe