KernelBench hard · RTX PRO 6000

W4A16 GEMM DeepSeek V4.1 Flash (1017)

25.1%geomean peak fraction across shapes

ctypes CUDA W4A16 GEMM that fuses the int4 unpack and dequant (sub/mul.rn.bf16x2 against a packed scale and 128+zero word, exact under the integral-zero contract) into mma m16n8k16 bf16. Weights are repacked once into fragment order and streamed from global with intra-CTA split-K; a per-shape table picks tile configs only, with a generic fallback.

harnessor-fableagent session3h 0mtotal wall3h 1mcheck3sbenchmark3soutput tokens450,142cost$33.90gpu-lock wait18mgpu-lock held20mregimememory

Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth

1×12288×40960.040 ms37.6%0.68 TB/s · 38% of 1.8 TB/s HBM · also 3 TFLOPS (1% of compute)
32×12288×40960.059 ms26.2%0.47 TB/s · 26% of 1.8 TB/s HBM · also 55 TFLOPS (11% of compute)
256×12288×40960.148 ms13.2%174 TFLOPS · 35% of 500 TF bf16 peak · also 0.24 TB/s (13% of HBM)
1×4096×40960.021 ms24.2%0.44 TB/s · 24% of 1.8 TB/s HBM · also 2 TFLOPS (0% of compute)
16×14336×40960.055 ms32.0%0.58 TB/s · 32% of 1.8 TB/s HBM · also 34 TFLOPS (7% of compute)

compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)

geomean(37.6% · 26.2% · 13.2% · 24.2% · 32.0%) = 25.1%

Kernel source (redacted)
"""Fused W4A16 weight-only quantized GEMM (AWQ/GPTQ asymmetric int4, group 128).

    y = x @ dequant(w_q, scales, zeros)

Design
------
The reference dequantises the whole weight matrix to bf16 and then calls matmul,
which moves 4x the bytes of the packed weights and materialises 25 MB of bf16.
Here unpack + dequant + GEMM run inside one CUDA kernel: the int4 weights are
read at 0.5 byte/element and fed straight into `mma.sync.m16n8k16` bf16
tensor-core ops, and are never written anywhere.

* Weights are repacked once (lazily, on the first forward after a weight load)
  into the exact per-lane fragment layout the kernel wants, so one warp load of
  128 B is one cache line of B fragments.
* Per group of 128 k the scale/zero-point pair is packed into one 32-bit word,
  `scale | (bf16(128 + zero) << 16)`.  Zero points are integral, so
  `bf16(128+v) - bf16(128+z)` is an exact integer and
  `(bf16(128+v) - bf16(128+z)) * scale` reproduces the reference rounding bit
  for bit; `sub.rn.bf16x2` / `mul.rn.bf16x2` dequantise two weights per
  instruction in the mma feed.
* This regime is memory bound (25 MB of weights, and M <= 32 for 3 of the 5
  scored shapes, against 128 MB of L2), so the kernel is organised for
  bandwidth, not FLOPs: each warp owns a 16-row x NJT*8-col output tile plus a
  slice of K (intra-CTA split-K), streams B fragments straight from global with
  no smem staging and no barriers in the main loop, and the per-warp partials
  are combined through shared memory at the end.  A 16 B weight load per lane
  (cfg 2) halves the issue pressure where the shapes get deeper.
* The host path is on the critical path of the benchmark (the timer starts
  before the kernel is launched), so it is one cached closure, one raw-stream
  lookup and one ctypes call into the launcher.
"""
from __future__ import annotations

import ctypes
import hashlib
import os
import subprocess
import tempfile

import torch
import torch.nn as nn

GROUP_SIZE = 128

_CUDA_SRC = r"""
// Fused W4A16 GEMM (AWQ/GPTQ asymmetric int4, group 128) for SM120 (Blackwell).
//
// One kernel: int4 -> bf16 dequant is fused into the mma feed, so the weights
// move as 4 bits and are never materialised.
//
// Weight packing (offline, Python):
//   Wpk[kt2][j][t][b] = w_q[16*kt2 + 4*b + (t%4)][8*j + (t/4)]      (uint8)
// For a fixed (kt2, j) there is one contiguous 128-byte block; lane t finds at
// offset t*4 the four bytes it needs to build the k16n8 B fragments of
// mma.sync.m16n8k16 (bf16) for k-tiles 2*kt2 and 2*kt2+1, so one warp load of
// 128 bytes is exactly one full cache line.
//
// SZ[g][n] = bits_bf16(scale) | (bits_bf16(128 + zero) << 16)
//   Zero points are integral 0..15 (AWQ/GPTQ contract), so bf16(128+v) is
//   exact, bf16(128+v) - bf16(128+z) is an exact integer, and
//       (bf16(128+v) - bf16(128+z)) * scale
//   reproduces the reference dequant rounding bit for bit.
//
// Work decomposition: three shapes are skinny (M <= 32) and the whole problem is
// 25 MB of weights against 128 MB of L2, so the kernel is bandwidth/latency
// bound, not FLOP bound.  Each warp owns a full 16-row x NJT*8-col output tile
// and a slice of K (intra-CTA split-K), streams its B fragments straight from
// global (no smem staging, no barriers in the main loop, so nothing serialises
// against the loads), and the per-warp partial sums are combined through smem
// at the end.  WK warps x WN n-groups gives enough concurrent warps per SM to
// cover DRAM latency without replicating the weight traffic.
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cstdint>

#define DEVICE_INLINE __device__ __forceinline__

namespace {

DEVICE_INLINE uint32_t deq_pair(uint32_t b, uint32_t cz2, uint32_t s2) {
  uint32_t v = (b & 0x0Fu) | ((b & 0xF0u) << 12) | 0x43004300u;
  uint32_t t, o;
  asm("sub.rn.bf16x2 %0, %1, %2;" : "=r"(t) : "r"(v), "r"(cz2));
  asm("mul.rn.bf16x2 %0, %1, %2;" : "=r"(o) : "r"(t), "r"(s2));
  return o;
}
DEVICE_INLINE void mma_bf16(float* c, const uint32_t* a, uint32_t b0, uint32_t b1) {
  asm volatile(
      "mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
      "{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n"
      : "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
      : "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1));
}
DEVICE_INLINE uint32_t ldg32(const void* p) {
  uint32_t v;
  asm volatile("ld.global.ca.b32 %0, [%1];\n" : "=r"(v) : "l"(p));
  return v;
}
DEVICE_INLINE void ldw128(const void* p, uint32_t* r) {
  asm volatile("ld.global.ca.v4.b32 {%0,%1,%2,%3}, [%4];\n"
               : "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3])
               : "l"(p));
}

DEVICE_INLINE uint32_t pack2(float lo, float hi) {
  __nv_bfloat162 v = __floats2bfloat162_rn(lo, hi);
  return *reinterpret_cast<uint32_t*>(&v);
}

// BM rows, BN cols, WK k-splits inside the CTA, WM x WN n/m warp grid, UNR unroll
template <int BM_, int BN_, int WK, int WM, int WN, bool V4 = false>
__global__ __launch_bounds__(WK * WM * WN * 32) void w4a16_ks(
    const uint32_t* __restrict__ W, const uint32_t* __restrict__ SZ,
    const __nv_bfloat16* __restrict__ X, __nv_bfloat16* __restrict__ OUT,
    int M, int N, int K) {
  constexpr int BM = BM_;
  constexpr int NB8 = BN_ / 8;
  constexpr int NJT = NB8 / WN;
  constexpr int NW = WK * WM * WN;

  // per-warp partials, reduced at the end; if the [WK][BM][BN] slab would not
  // fit in static smem, fall back to a single slab with shared-memory atomics
  constexpr bool BIGRED = (WK * BM_ * BN_ * 4 > 40000);
  constexpr int REDW = BIGRED ? BM_ * BN_ : WK * BM_ * BN_;
  __shared__ float red[REDW];

  const int nb8 = N >> 3;
  const int tid = threadIdx.x;
  const int lane = tid & 31;
  const int warp = tid >> 5;
  const int wn = warp % WN, wm = (warp / WN) % WM, wk = warp / (WN * WM);
  const int jbase = wn * NJT;
  const int n0 = blockIdx.x * BN_;
  const int m0 = blockIdx.y * BM;
  const int kchunks = K >> 7;                    // 128 k per chunk == one group
  const int kc0 = wk * (kchunks / WK), kc1 = (wk + 1) * (kchunks / WK);

  const int ar = lane >> 2, ac = (lane & 3) * 2;
  const int rowA = m0 + wm * 16 + ar;
  const int rA0 = min(rowA, M - 1), rA1 = min(rowA + 8, M - 1);
  const __nv_bfloat16* xb0 = X + (size_t)rA0 * K;
  const __nv_bfloat16* xb1 = X + (size_t)rA1 * K;

  float acc[NJT][4];
#pragma unroll
  for (int j = 0; j < NJT; j++)
#pragma unroll
    for (int i = 0; i < 4; i++) acc[j][i] = 0.f;

  // Two weight packings are supported:
  //   V4   [kt2][jg = n-tile/4][lane][jj]: a lane's four consecutive words are
  //        the four 8-column n-tiles of one group, so one 16 B load per lane
  //        covers 512 contiguous bytes of weight stream.
  //   else [kt2][n-tile][lane]: one 128 B line per n-tile, 32-bit loads.
  const uint32_t* wb = W + (size_t)(n0 >> 3) * 32 + jbase * 32 + lane;
  const uint32_t* wb4 = W + (size_t)((((n0 >> 3) + jbase) >> 2) * 128 + lane * 4);

  for (int c = kc0; c < kc1; c++) {
    uint32_t s2[NJT], cz2[NJT];
#pragma unroll
    for (int j = 0; j < NJT; j++) {
      int n = n0 + 8 * (jbase + j) + (lane >> 2);
      uint32_t sz = SZ[(size_t)c * N + n];
      uint32_t sv = sz & 0xFFFFu, zv = sz >> 16;
      s2[j] = sv | (sv << 16);
      cz2[j] = zv | (zv << 16);
    }
    const uint32_t* wc = wb + (size_t)c * 4 * nb8 * 32;
    const uint32_t* wc4 = wb4 + (size_t)c * 4 * (N >> 5) * 128;
    const __nv_bfloat16* p0 = xb0 + c * 128;
    const __nv_bfloat16* p1 = xb1 + c * 128;
#pragma unroll
    for (int q = 0; q < 4; q++) {
      const int kb = q * 32;
      uint32_t a0[4], a1[4];
      a0[0] = ldg32(p0 + kb + ac);
      a0[1] = ldg32(p1 + kb + ac);
      a0[2] = ldg32(p0 + kb + ac + 8);
      a0[3] = ldg32(p1 + kb + ac + 8);
      a1[0] = ldg32(p0 + kb + ac + 16);
      a1[1] = ldg32(p1 + kb + ac + 16);
      a1[2] = ldg32(p0 + kb + ac + 24);
      a1[3] = ldg32(p1 + kb + ac + 24);
      uint32_t w32[NJT];
      if constexpr (V4) {
        const uint32_t* wp4 = wc4 + (size_t)q * (N >> 5) * 128;
#pragma unroll
        for (int g = 0; g < NJT / 4; g++) ldw128(wp4 + g * 128, w32 + 4 * g);
      } else {
#pragma unroll
        for (int j = 0; j < NJT; j++) w32[j] = ldg32(wc + (size_t)(q * nb8 + j) * 32);
      }
#pragma unroll
      for (int j = 0; j < NJT; j++) {
        mma_bf16(acc[j], a0, deq_pair(w32[j] & 0xFFu, cz2[j], s2[j]),
                 deq_pair((w32[j] >> 8) & 0xFFu, cz2[j], s2[j]));
        mma_bf16(acc[j], a1, deq_pair((w32[j] >> 16) & 0xFFu, cz2[j], s2[j]),
                 deq_pair(w32[j] >> 24, cz2[j], s2[j]));
      }
    }
  }

  if (WK == 1) {
    const int r = lane >> 2;
#pragma unroll
    for (int j = 0; j < NJT; j++) {
      int ntile = n0 + 8 * (jbase + j) + 2 * (lane & 3);
      int mr = m0 + wm * 16 + r;
      if (mr < M) *reinterpret_cast<uint32_t*>(OUT + (size_t)mr * N + ntile) = pack2(acc[j][0], acc[j][1]);
      if (mr + 8 < M) *reinterpret_cast<uint32_t*>(OUT + (size_t)(mr + 8) * N + ntile) = pack2(acc[j][2], acc[j][3]);
    }
    return;
  }

  // publish partials: rows (wm*16 + ar) and (+8), cols 8*(jbase+j) + 2*(lane&3)
  constexpr int TOT = BM * BN_;
  if (BIGRED) {
    for (int idx = tid; idx < TOT; idx += NW * 32) red[idx] = 0.f;
    __syncthreads();
  }
  {
    int r0 = wm * 16 + ar;
#pragma unroll
    for (int j = 0; j < NJT; j++) {
      int cbase = 8 * (jbase + j) + 2 * (lane & 3);
      float* base = BIGRED ? red : red + wk * TOT;
      if (BIGRED) {
        atomicAdd(base + r0 * BN_ + cbase, acc[j][0]);
        atomicAdd(base + r0 * BN_ + cbase + 1, acc[j][1]);
        atomicAdd(base + (r0 + 8) * BN_ + cbase, acc[j][2]);
        atomicAdd(base + (r0 + 8) * BN_ + cbase + 1, acc[j][3]);
      } else {
        base[r0 * BN_ + cbase] = acc[j][0];
        base[r0 * BN_ + cbase + 1] = acc[j][1];
        base[(r0 + 8) * BN_ + cbase] = acc[j][2];
        base[(r0 + 8) * BN_ + cbase + 1] = acc[j][3];
      }
    }
  }
  __syncthreads();
  // reduce over WK, convert, store (all threads of the CTA cooperate)
  for (int idx = tid; idx < TOT; idx += NW * 32) {
    int r = idx / BN_, cc = idx % BN_;
    int mr = m0 + r;
    if (mr >= M) continue;
    float s;
    if (BIGRED) {
      s = red[idx];
    } else {
      s = red[idx];
#pragma unroll
      for (int k = 1; k < WK; k++) s += red[k * TOT + idx];
    }
    OUT[(size_t)mr * N + n0 + cc] = __float2bfloat16(s);
  }
}

}  // namespace

#define LAUNCH_KS4(BM_, BN_, WK_, WM_, WN_)                                                    \
  do {                                                                                        \
    dim3 grid(N / (BN_), (M + (BM_)-1) / (BM_), 1);                                            \
    w4a16_ks<BM_, BN_, WK_, WM_, WN_, true><<<grid, (WK_ * WM_ * WN_) * 32, 0, stream>>>(      \
        Wp, SZp, Xp, Op, M, N, K);                                                             \
  } while (0)

#define LAUNCH_KS(BM_, BN_, WK_, WM_, WN_)                                                     \
  do {                                                                                        \
    dim3 grid(N / (BN_), (M + (BM_)-1) / (BM_), 1);                                            \
    w4a16_ks<BM_, BN_, WK_, WM_, WN_><<<grid, (WK_ * WM_ * WN_) * 32, 0, stream>>>(            \
        Wp, SZp, Xp, Op, M, N, K);                                                             \
  } while (0)

extern "C" int w4a16_launch(int cfg, const void* W, const void* SZ, const void* X,
                            void* OUT, int M, int N, int K, cudaStream_t stream) {
  const uint32_t* Wp = (const uint32_t*)W;
  const uint32_t* SZp = (const uint32_t*)SZ;
  const __nv_bfloat16* Xp = (const __nv_bfloat16*)X;
  __nv_bfloat16* Op = (__nv_bfloat16*)OUT;
  switch (cfg) {
    case 0: LAUNCH_KS(16, 64, 8, 1, 2); break;      // skinny M, N%64,  K%1024
    case 1: LAUNCH_KS(16, 64, 4, 1, 1); break;      // M<=32,     N%64,  K%512
    case 2: LAUNCH_KS4(16, 128, 4, 1, 1); break;    // deep M,    N%128, K%512
    case 3: LAUNCH_KS(16, 32, 16, 1, 2); break;     // skinny M, N%32,  K%2048
    case 4: LAUNCH_KS(16, 128, 4, 1, 1); break;     // generic,  N%128, K%512
    case 5: LAUNCH_KS(16, 64, 1, 1, 1); break;      // generic,  N%64
    case 6: LAUNCH_KS(16, 32, 1, 1, 1); break;      // generic,  N%32
    case 7: LAUNCH_KS(32, 64, 4, 2, 1); break;      // generic,  N%64,  K%512
    case 8: LAUNCH_KS(16, 128, 1, 1, 1); break;     // generic,  N%128
    default: LAUNCH_KS(16, 64, 1, 1, 1);
  }
  return (int)cudaGetLastError();
}

"""

_lib = None


def _arch() -> str:
    major, minor = torch.cuda.get_device_capability()
    return f"sm_{major}{minor}a"


def _load_lib():
    global _lib
    if _lib is not None:
        return _lib
    tag = hashlib.sha1(_CUDA_SRC.encode()).hexdigest()[:16]
    cache = os.path.join(tempfile.gettempdir(), f"w4a16_{tag}")
    os.makedirs(cache, exist_ok=True)
    cu = os.path.join(cache, "k.cu")
    so = os.path.join(cache, "k.so")
    if not os.path.exists(so):
        with open(cu, "w") as f:
            f.write(_CUDA_SRC)
        base = ["nvcc", "-shared", "-Xcompiler", "-fPIC", "-O3", "-Xptxas", "-O3"]
        err = None
        for arch in (_arch(), _arch().rstrip("a"), "sm_90"):
            try:
                subprocess.run(base + ["-arch", arch, "-o", so, cu], check=True,
                               capture_output=True, text=True)
                err = None
                break
            except Exception as exc:  # noqa: BLE001
                err = exc
                if os.path.exists(so):
                    os.remove(so)
        if err is not None:
            raise RuntimeError(f"nvcc build failed: {err}")
    lib = ctypes.CDLL(so)
    lib.w4a16_launch.restype = ctypes.c_int
    lib.w4a16_launch.argtypes = [ctypes.c_int] + [ctypes.c_void_p] * 4 + [ctypes.c_int] * 3 + [ctypes.c_void_p]
    _lib = lib
    return lib


def _prepack_w(w_q: torch.Tensor, N: int, K: int) -> torch.Tensor:
    """(K//2, N) uint8 -> (K//32, N//8, 32, 4) uint8 in mma B-fragment order."""
    return w_q.view(K // 32, 4, 4, N // 8, 8).permute(0, 3, 4, 2, 1).contiguous()


def _prepack_w4(w_q: torch.Tensor, N: int, K: int) -> torch.Tensor:
    """Same data as (K//32, N//32, 32, 4) int32: a lane's four words are the four
    8-column n-tiles of one 32-column group, so a 16 B load covers 512 B."""
    p = _prepack_w(w_q, N, K)
    return p.view(K // 32, N // 32, 4, 8, 4, 4).permute(0, 1, 3, 4, 2, 5) \
            .contiguous().view(torch.int32).reshape(K // 32, N // 32, 32, 4)


def _prepack_sz(scales: torch.Tensor, zeros: torch.Tensor, N: int) -> torch.Tensor:
    """Pack (scale, 128 + zero) bf16 pairs into one int32 per (group, column)."""
    s_bits = scales.contiguous().view(torch.int16).to(torch.int32) & 0xFFFF
    cz = (128.0 + zeros.float()).to(torch.bfloat16).view(torch.int16).to(torch.int32) & 0xFFFF
    return (s_bits | (cz << 16)).contiguous()


# cfg -> (BN, WK, uses_uint4-packed weights); a cfg may only be launched where
# N % BN == 0 and (K // 128) % WK == 0.
_CFG_GEOM = {
    0: (64, 8, False),
    1: (64, 4, False),
    2: (128, 4, True),
    3: (32, 16, False),
    4: (128, 4, False),
    5: (64, 1, False),
    6: (32, 1, False),
    7: (64, 4, False),
    8: (128, 1, False),
}

_BY_SHAPE = {
    (1, 12288, 4096): 0,     # 16x64 tile,  8-way intra-CTA split-K
    (32, 12288, 4096): 1,    # 16x64 tile,  4-way split-K, one n-tile per warp
    (256, 12288, 4096): 2,   # 16x128 tile, 4-way split-K, 16 B weight loads
    (1, 4096, 4096): 3,      # 16x32 tile, 16-way split-K (small N: more CTAs)
    (16, 14336, 4096): 3,    # 16x32 tile, 16-way split-K
}

_CANDIDATES = {
    "skinny": [0, 3, 4, 5, 6],   # M <= 16
    "small": [1, 7, 5, 6],       # 16 < M <= 32
    "wide": [2, 4, 8, 5, 6],     # M > 32
}


def _pick_cfg(M: int, N: int, K: int) -> int:
    cfg = _BY_SHAPE.get((M, N, K))
    if cfg is not None:
        return cfg
    kc = K // 128
    group = "skinny" if M <= 16 else ("small" if M <= 32 else "wide")
    for cand in _CANDIDATES[group]:
        bn, wk, _ = _CFG_GEOM[cand]
        if N % bn == 0 and kc % wk == 0:
            return cand
    raise RuntimeError(f"no kernel config for M={M} N={N} K={K}")


try:  # cheapest way to read the current stream from Python
    _raw_stream = torch._C._cuda_getCurrentRawStream
except AttributeError:  # pragma: no cover
    _raw_stream = None


class _Launch:
    """Everything the steady-state forward needs, resolved once."""

    __slots__ = ("m", "dev", "run", "outs", "idx")

    def __init__(self, m, dev, run, outs):
        self.m = m
        self.dev = dev
        self.run = run
        self.outs = outs
        self.idx = 0


class Model(nn.Module):
    """y = x @ dequant(w_q, scales, zeros); int4 weights stay packed."""

    def __init__(self, M: int, N: int, K: int, group_size: int = GROUP_SIZE):
        super().__init__()
        assert K % group_size == 0, "K must be divisible by group_size"
        assert K % 2 == 0, "K must be even (int4 packing)"
        self.M, self.N, self.K = M, N, K
        self.group_size = group_size
        n_groups = K // group_size
        # Registered as buffers (not Parameters) so the reference state_dict
        # loads into them with strict=True.
        self.register_buffer("w_q", torch.zeros((K // 2, N), dtype=torch.uint8))
        self.register_buffer("scales", torch.zeros((n_groups, N), dtype=torch.bfloat16))
        self.register_buffer("zeros", torch.zeros((n_groups, N), dtype=torch.bfloat16))
        self._packed = None
        self._fast = None

    # -- repack caches are invalidated whenever the weights change or move ----
    def _apply(self, fn, recurse=True):
        self._packed = None
        self._fast = None
        return super()._apply(fn, recurse)

    def _load_from_state_dict(self, *args, **kwargs):
        self._packed = None
        self._fast = None
        return super()._load_from_state_dict(*args, **kwargs)

    def _pack(self, want4: bool):
        w_q, sc, zp = self.w_q, self.scales, self.zeros
        key = (w_q.data_ptr(), sc.data_ptr(), zp.data_ptr(), w_q.device, want4)
        if self._packed is not None and self._packed[0] == key:
            return self._packed[1], self._packed[2]
        w_c = w_q.contiguous()
        wp = _prepack_w4(w_c, self.N, self.K) if want4 else _prepack_w(w_c, self.N, self.K)
        sz = _prepack_sz(sc, zp, self.N)
        self._packed = (key, wp, sz)
        return wp, sz

    def _setup(self, x: torch.Tensor) -> _Launch:
        lib = _load_lib()
        M, N, K = x.shape[0], self.N, self.K
        cfg = _pick_cfg(M, N, K)
        wp, sz = self._pack(_CFG_GEOM[cfg][2])
        launch = lib.w4a16_launch
        wp_p, sz_p = wp.data_ptr(), sz.data_ptr()
        dev = x.device
        if _raw_stream is not None:
            dev_idx = dev.index or 0
            get_stream = lambda: _raw_stream(dev_idx)  # noqa: E731
        else:  # pragma: no cover
            get_stream = lambda: torch.cuda.current_stream(dev).cuda_stream  # noqa: E731

        def run(xx, out, _launch=launch, _cfg=cfg, _wp=wp_p, _sz=sz_p,
                _M=M, _N=N, _K=K, _get=get_stream):
            rc = _launch(_cfg, _wp, _sz, xx.data_ptr(), out.data_ptr(),
                         _M, _N, _K, _get())
            if rc != 0:
                raise RuntimeError(f"w4a16_launch failed: cudaError {rc}")

        outs = [torch.empty((M, N), dtype=torch.bfloat16, device=dev) for _ in range(2)]
        fast = _Launch(M, dev, run, outs)
        self._fast = fast
        return fast

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        fast = self._fast
        if fast is None or fast.m != x.shape[0] or fast.dev != x.device:
            fast = self._setup(x)
        if not x.is_contiguous():
            x = x.contiguous()
        i = fast.idx
        fast.idx = i ^ 1
        out = fast.outs[i]
        fast.run(x, out)
        return out


M = 1
N = 12288
K = 4096


def get_inputs():
    x = torch.randn(M, K, dtype=torch.bfloat16)
    return [x]


def get_init_inputs():
    return [M, N, K]

20261017_215437_or-fable_deepseek_deepseek-v4.1-flash-1017_07_w4a16_gemm