KernelBench hard · RTX PRO 6000
W4A16 GEMM DeepSeek V4.1 Flash (1017)
25.1%geomean peak fraction across shapes
ctypes CUDA W4A16 GEMM that fuses the int4 unpack and dequant (sub/mul.rn.bf16x2 against a packed scale and 128+zero word, exact under the integral-zero contract) into mma m16n8k16 bf16. Weights are repacked once into fragment order and streamed from global with intra-CTA split-K; a per-shape table picks tile configs only, with a generic fallback.
harnessor-fableagent session3h 0mtotal wall3h 1mcheck3sbenchmark3soutput tokens450,142cost$33.90gpu-lock wait18mgpu-lock held20mregimememory
Per-shape vs governing ceilingeach shape graded against whichever binds — bf16 compute or HBM bandwidth
1×12288×40960.040 ms37.6%0.68 TB/s · 38% of 1.8 TB/s HBM · also 3 TFLOPS (1% of compute)
32×12288×40960.059 ms26.2%0.47 TB/s · 26% of 1.8 TB/s HBM · also 55 TFLOPS (11% of compute)
256×12288×40960.148 ms13.2%174 TFLOPS · 35% of 500 TF bf16 peak · also 0.24 TB/s (13% of HBM)
1×4096×40960.021 ms24.2%0.44 TB/s · 24% of 1.8 TB/s HBM · also 2 TFLOPS (0% of compute)
16×14336×40960.055 ms32.0%0.58 TB/s · 32% of 1.8 TB/s HBM · also 34 TFLOPS (7% of compute)
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(37.6% · 26.2% · 13.2% · 24.2% · 32.0%) = 25.1%
Kernel source (redacted)
"""Fused W4A16 weight-only quantized GEMM (AWQ/GPTQ asymmetric int4, group 128).
y = x @ dequant(w_q, scales, zeros)
Design
------
The reference dequantises the whole weight matrix to bf16 and then calls matmul,
which moves 4x the bytes of the packed weights and materialises 25 MB of bf16.
Here unpack + dequant + GEMM run inside one CUDA kernel: the int4 weights are
read at 0.5 byte/element and fed straight into `mma.sync.m16n8k16` bf16
tensor-core ops, and are never written anywhere.
* Weights are repacked once (lazily, on the first forward after a weight load)
into the exact per-lane fragment layout the kernel wants, so one warp load of
128 B is one cache line of B fragments.
* Per group of 128 k the scale/zero-point pair is packed into one 32-bit word,
`scale | (bf16(128 + zero) << 16)`. Zero points are integral, so
`bf16(128+v) - bf16(128+z)` is an exact integer and
`(bf16(128+v) - bf16(128+z)) * scale` reproduces the reference rounding bit
for bit; `sub.rn.bf16x2` / `mul.rn.bf16x2` dequantise two weights per
instruction in the mma feed.
* This regime is memory bound (25 MB of weights, and M <= 32 for 3 of the 5
scored shapes, against 128 MB of L2), so the kernel is organised for
bandwidth, not FLOPs: each warp owns a 16-row x NJT*8-col output tile plus a
slice of K (intra-CTA split-K), streams B fragments straight from global with
no smem staging and no barriers in the main loop, and the per-warp partials
are combined through shared memory at the end. A 16 B weight load per lane
(cfg 2) halves the issue pressure where the shapes get deeper.
* The host path is on the critical path of the benchmark (the timer starts
before the kernel is launched), so it is one cached closure, one raw-stream
lookup and one ctypes call into the launcher.
"""
from __future__ import annotations
import ctypes
import hashlib
import os
import subprocess
import tempfile
import torch
import torch.nn as nn
GROUP_SIZE = 128
_CUDA_SRC = r"""
// Fused W4A16 GEMM (AWQ/GPTQ asymmetric int4, group 128) for SM120 (Blackwell).
//
// One kernel: int4 -> bf16 dequant is fused into the mma feed, so the weights
// move as 4 bits and are never materialised.
//
// Weight packing (offline, Python):
// Wpk[kt2][j][t][b] = w_q[16*kt2 + 4*b + (t%4)][8*j + (t/4)] (uint8)
// For a fixed (kt2, j) there is one contiguous 128-byte block; lane t finds at
// offset t*4 the four bytes it needs to build the k16n8 B fragments of
// mma.sync.m16n8k16 (bf16) for k-tiles 2*kt2 and 2*kt2+1, so one warp load of
// 128 bytes is exactly one full cache line.
//
// SZ[g][n] = bits_bf16(scale) | (bits_bf16(128 + zero) << 16)
// Zero points are integral 0..15 (AWQ/GPTQ contract), so bf16(128+v) is
// exact, bf16(128+v) - bf16(128+z) is an exact integer, and
// (bf16(128+v) - bf16(128+z)) * scale
// reproduces the reference dequant rounding bit for bit.
//
// Work decomposition: three shapes are skinny (M <= 32) and the whole problem is
// 25 MB of weights against 128 MB of L2, so the kernel is bandwidth/latency
// bound, not FLOP bound. Each warp owns a full 16-row x NJT*8-col output tile
// and a slice of K (intra-CTA split-K), streams its B fragments straight from
// global (no smem staging, no barriers in the main loop, so nothing serialises
// against the loads), and the per-warp partial sums are combined through smem
// at the end. WK warps x WN n-groups gives enough concurrent warps per SM to
// cover DRAM latency without replicating the weight traffic.
#include <cuda_runtime.h>
#include <cuda_bf16.h>
#include <cstdint>
#define DEVICE_INLINE __device__ __forceinline__
namespace {
DEVICE_INLINE uint32_t deq_pair(uint32_t b, uint32_t cz2, uint32_t s2) {
uint32_t v = (b & 0x0Fu) | ((b & 0xF0u) << 12) | 0x43004300u;
uint32_t t, o;
asm("sub.rn.bf16x2 %0, %1, %2;" : "=r"(t) : "r"(v), "r"(cz2));
asm("mul.rn.bf16x2 %0, %1, %2;" : "=r"(o) : "r"(t), "r"(s2));
return o;
}
DEVICE_INLINE void mma_bf16(float* c, const uint32_t* a, uint32_t b0, uint32_t b1) {
asm volatile(
"mma.sync.aligned.m16n8k16.row.col.f32.bf16.bf16.f32 "
"{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n"
: "+f"(c[0]), "+f"(c[1]), "+f"(c[2]), "+f"(c[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b0), "r"(b1));
}
DEVICE_INLINE uint32_t ldg32(const void* p) {
uint32_t v;
asm volatile("ld.global.ca.b32 %0, [%1];\n" : "=r"(v) : "l"(p));
return v;
}
DEVICE_INLINE void ldw128(const void* p, uint32_t* r) {
asm volatile("ld.global.ca.v4.b32 {%0,%1,%2,%3}, [%4];\n"
: "=r"(r[0]), "=r"(r[1]), "=r"(r[2]), "=r"(r[3])
: "l"(p));
}
DEVICE_INLINE uint32_t pack2(float lo, float hi) {
__nv_bfloat162 v = __floats2bfloat162_rn(lo, hi);
return *reinterpret_cast<uint32_t*>(&v);
}
// BM rows, BN cols, WK k-splits inside the CTA, WM x WN n/m warp grid, UNR unroll
template <int BM_, int BN_, int WK, int WM, int WN, bool V4 = false>
__global__ __launch_bounds__(WK * WM * WN * 32) void w4a16_ks(
const uint32_t* __restrict__ W, const uint32_t* __restrict__ SZ,
const __nv_bfloat16* __restrict__ X, __nv_bfloat16* __restrict__ OUT,
int M, int N, int K) {
constexpr int BM = BM_;
constexpr int NB8 = BN_ / 8;
constexpr int NJT = NB8 / WN;
constexpr int NW = WK * WM * WN;
// per-warp partials, reduced at the end; if the [WK][BM][BN] slab would not
// fit in static smem, fall back to a single slab with shared-memory atomics
constexpr bool BIGRED = (WK * BM_ * BN_ * 4 > 40000);
constexpr int REDW = BIGRED ? BM_ * BN_ : WK * BM_ * BN_;
__shared__ float red[REDW];
const int nb8 = N >> 3;
const int tid = threadIdx.x;
const int lane = tid & 31;
const int warp = tid >> 5;
const int wn = warp % WN, wm = (warp / WN) % WM, wk = warp / (WN * WM);
const int jbase = wn * NJT;
const int n0 = blockIdx.x * BN_;
const int m0 = blockIdx.y * BM;
const int kchunks = K >> 7; // 128 k per chunk == one group
const int kc0 = wk * (kchunks / WK), kc1 = (wk + 1) * (kchunks / WK);
const int ar = lane >> 2, ac = (lane & 3) * 2;
const int rowA = m0 + wm * 16 + ar;
const int rA0 = min(rowA, M - 1), rA1 = min(rowA + 8, M - 1);
const __nv_bfloat16* xb0 = X + (size_t)rA0 * K;
const __nv_bfloat16* xb1 = X + (size_t)rA1 * K;
float acc[NJT][4];
#pragma unroll
for (int j = 0; j < NJT; j++)
#pragma unroll
for (int i = 0; i < 4; i++) acc[j][i] = 0.f;
// Two weight packings are supported:
// V4 [kt2][jg = n-tile/4][lane][jj]: a lane's four consecutive words are
// the four 8-column n-tiles of one group, so one 16 B load per lane
// covers 512 contiguous bytes of weight stream.
// else [kt2][n-tile][lane]: one 128 B line per n-tile, 32-bit loads.
const uint32_t* wb = W + (size_t)(n0 >> 3) * 32 + jbase * 32 + lane;
const uint32_t* wb4 = W + (size_t)((((n0 >> 3) + jbase) >> 2) * 128 + lane * 4);
for (int c = kc0; c < kc1; c++) {
uint32_t s2[NJT], cz2[NJT];
#pragma unroll
for (int j = 0; j < NJT; j++) {
int n = n0 + 8 * (jbase + j) + (lane >> 2);
uint32_t sz = SZ[(size_t)c * N + n];
uint32_t sv = sz & 0xFFFFu, zv = sz >> 16;
s2[j] = sv | (sv << 16);
cz2[j] = zv | (zv << 16);
}
const uint32_t* wc = wb + (size_t)c * 4 * nb8 * 32;
const uint32_t* wc4 = wb4 + (size_t)c * 4 * (N >> 5) * 128;
const __nv_bfloat16* p0 = xb0 + c * 128;
const __nv_bfloat16* p1 = xb1 + c * 128;
#pragma unroll
for (int q = 0; q < 4; q++) {
const int kb = q * 32;
uint32_t a0[4], a1[4];
a0[0] = ldg32(p0 + kb + ac);
a0[1] = ldg32(p1 + kb + ac);
a0[2] = ldg32(p0 + kb + ac + 8);
a0[3] = ldg32(p1 + kb + ac + 8);
a1[0] = ldg32(p0 + kb + ac + 16);
a1[1] = ldg32(p1 + kb + ac + 16);
a1[2] = ldg32(p0 + kb + ac + 24);
a1[3] = ldg32(p1 + kb + ac + 24);
uint32_t w32[NJT];
if constexpr (V4) {
const uint32_t* wp4 = wc4 + (size_t)q * (N >> 5) * 128;
#pragma unroll
for (int g = 0; g < NJT / 4; g++) ldw128(wp4 + g * 128, w32 + 4 * g);
} else {
#pragma unroll
for (int j = 0; j < NJT; j++) w32[j] = ldg32(wc + (size_t)(q * nb8 + j) * 32);
}
#pragma unroll
for (int j = 0; j < NJT; j++) {
mma_bf16(acc[j], a0, deq_pair(w32[j] & 0xFFu, cz2[j], s2[j]),
deq_pair((w32[j] >> 8) & 0xFFu, cz2[j], s2[j]));
mma_bf16(acc[j], a1, deq_pair((w32[j] >> 16) & 0xFFu, cz2[j], s2[j]),
deq_pair(w32[j] >> 24, cz2[j], s2[j]));
}
}
}
if (WK == 1) {
const int r = lane >> 2;
#pragma unroll
for (int j = 0; j < NJT; j++) {
int ntile = n0 + 8 * (jbase + j) + 2 * (lane & 3);
int mr = m0 + wm * 16 + r;
if (mr < M) *reinterpret_cast<uint32_t*>(OUT + (size_t)mr * N + ntile) = pack2(acc[j][0], acc[j][1]);
if (mr + 8 < M) *reinterpret_cast<uint32_t*>(OUT + (size_t)(mr + 8) * N + ntile) = pack2(acc[j][2], acc[j][3]);
}
return;
}
// publish partials: rows (wm*16 + ar) and (+8), cols 8*(jbase+j) + 2*(lane&3)
constexpr int TOT = BM * BN_;
if (BIGRED) {
for (int idx = tid; idx < TOT; idx += NW * 32) red[idx] = 0.f;
__syncthreads();
}
{
int r0 = wm * 16 + ar;
#pragma unroll
for (int j = 0; j < NJT; j++) {
int cbase = 8 * (jbase + j) + 2 * (lane & 3);
float* base = BIGRED ? red : red + wk * TOT;
if (BIGRED) {
atomicAdd(base + r0 * BN_ + cbase, acc[j][0]);
atomicAdd(base + r0 * BN_ + cbase + 1, acc[j][1]);
atomicAdd(base + (r0 + 8) * BN_ + cbase, acc[j][2]);
atomicAdd(base + (r0 + 8) * BN_ + cbase + 1, acc[j][3]);
} else {
base[r0 * BN_ + cbase] = acc[j][0];
base[r0 * BN_ + cbase + 1] = acc[j][1];
base[(r0 + 8) * BN_ + cbase] = acc[j][2];
base[(r0 + 8) * BN_ + cbase + 1] = acc[j][3];
}
}
}
__syncthreads();
// reduce over WK, convert, store (all threads of the CTA cooperate)
for (int idx = tid; idx < TOT; idx += NW * 32) {
int r = idx / BN_, cc = idx % BN_;
int mr = m0 + r;
if (mr >= M) continue;
float s;
if (BIGRED) {
s = red[idx];
} else {
s = red[idx];
#pragma unroll
for (int k = 1; k < WK; k++) s += red[k * TOT + idx];
}
OUT[(size_t)mr * N + n0 + cc] = __float2bfloat16(s);
}
}
} // namespace
#define LAUNCH_KS4(BM_, BN_, WK_, WM_, WN_) \
do { \
dim3 grid(N / (BN_), (M + (BM_)-1) / (BM_), 1); \
w4a16_ks<BM_, BN_, WK_, WM_, WN_, true><<<grid, (WK_ * WM_ * WN_) * 32, 0, stream>>>( \
Wp, SZp, Xp, Op, M, N, K); \
} while (0)
#define LAUNCH_KS(BM_, BN_, WK_, WM_, WN_) \
do { \
dim3 grid(N / (BN_), (M + (BM_)-1) / (BM_), 1); \
w4a16_ks<BM_, BN_, WK_, WM_, WN_><<<grid, (WK_ * WM_ * WN_) * 32, 0, stream>>>( \
Wp, SZp, Xp, Op, M, N, K); \
} while (0)
extern "C" int w4a16_launch(int cfg, const void* W, const void* SZ, const void* X,
void* OUT, int M, int N, int K, cudaStream_t stream) {
const uint32_t* Wp = (const uint32_t*)W;
const uint32_t* SZp = (const uint32_t*)SZ;
const __nv_bfloat16* Xp = (const __nv_bfloat16*)X;
__nv_bfloat16* Op = (__nv_bfloat16*)OUT;
switch (cfg) {
case 0: LAUNCH_KS(16, 64, 8, 1, 2); break; // skinny M, N%64, K%1024
case 1: LAUNCH_KS(16, 64, 4, 1, 1); break; // M<=32, N%64, K%512
case 2: LAUNCH_KS4(16, 128, 4, 1, 1); break; // deep M, N%128, K%512
case 3: LAUNCH_KS(16, 32, 16, 1, 2); break; // skinny M, N%32, K%2048
case 4: LAUNCH_KS(16, 128, 4, 1, 1); break; // generic, N%128, K%512
case 5: LAUNCH_KS(16, 64, 1, 1, 1); break; // generic, N%64
case 6: LAUNCH_KS(16, 32, 1, 1, 1); break; // generic, N%32
case 7: LAUNCH_KS(32, 64, 4, 2, 1); break; // generic, N%64, K%512
case 8: LAUNCH_KS(16, 128, 1, 1, 1); break; // generic, N%128
default: LAUNCH_KS(16, 64, 1, 1, 1);
}
return (int)cudaGetLastError();
}
"""
_lib = None
def _arch() -> str:
major, minor = torch.cuda.get_device_capability()
return f"sm_{major}{minor}a"
def _load_lib():
global _lib
if _lib is not None:
return _lib
tag = hashlib.sha1(_CUDA_SRC.encode()).hexdigest()[:16]
cache = os.path.join(tempfile.gettempdir(), f"w4a16_{tag}")
os.makedirs(cache, exist_ok=True)
cu = os.path.join(cache, "k.cu")
so = os.path.join(cache, "k.so")
if not os.path.exists(so):
with open(cu, "w") as f:
f.write(_CUDA_SRC)
base = ["nvcc", "-shared", "-Xcompiler", "-fPIC", "-O3", "-Xptxas", "-O3"]
err = None
for arch in (_arch(), _arch().rstrip("a"), "sm_90"):
try:
subprocess.run(base + ["-arch", arch, "-o", so, cu], check=True,
capture_output=True, text=True)
err = None
break
except Exception as exc: # noqa: BLE001
err = exc
if os.path.exists(so):
os.remove(so)
if err is not None:
raise RuntimeError(f"nvcc build failed: {err}")
lib = ctypes.CDLL(so)
lib.w4a16_launch.restype = ctypes.c_int
lib.w4a16_launch.argtypes = [ctypes.c_int] + [ctypes.c_void_p] * 4 + [ctypes.c_int] * 3 + [ctypes.c_void_p]
_lib = lib
return lib
def _prepack_w(w_q: torch.Tensor, N: int, K: int) -> torch.Tensor:
"""(K//2, N) uint8 -> (K//32, N//8, 32, 4) uint8 in mma B-fragment order."""
return w_q.view(K // 32, 4, 4, N // 8, 8).permute(0, 3, 4, 2, 1).contiguous()
def _prepack_w4(w_q: torch.Tensor, N: int, K: int) -> torch.Tensor:
"""Same data as (K//32, N//32, 32, 4) int32: a lane's four words are the four
8-column n-tiles of one 32-column group, so a 16 B load covers 512 B."""
p = _prepack_w(w_q, N, K)
return p.view(K // 32, N // 32, 4, 8, 4, 4).permute(0, 1, 3, 4, 2, 5) \
.contiguous().view(torch.int32).reshape(K // 32, N // 32, 32, 4)
def _prepack_sz(scales: torch.Tensor, zeros: torch.Tensor, N: int) -> torch.Tensor:
"""Pack (scale, 128 + zero) bf16 pairs into one int32 per (group, column)."""
s_bits = scales.contiguous().view(torch.int16).to(torch.int32) & 0xFFFF
cz = (128.0 + zeros.float()).to(torch.bfloat16).view(torch.int16).to(torch.int32) & 0xFFFF
return (s_bits | (cz << 16)).contiguous()
# cfg -> (BN, WK, uses_uint4-packed weights); a cfg may only be launched where
# N % BN == 0 and (K // 128) % WK == 0.
_CFG_GEOM = {
0: (64, 8, False),
1: (64, 4, False),
2: (128, 4, True),
3: (32, 16, False),
4: (128, 4, False),
5: (64, 1, False),
6: (32, 1, False),
7: (64, 4, False),
8: (128, 1, False),
}
_BY_SHAPE = {
(1, 12288, 4096): 0, # 16x64 tile, 8-way intra-CTA split-K
(32, 12288, 4096): 1, # 16x64 tile, 4-way split-K, one n-tile per warp
(256, 12288, 4096): 2, # 16x128 tile, 4-way split-K, 16 B weight loads
(1, 4096, 4096): 3, # 16x32 tile, 16-way split-K (small N: more CTAs)
(16, 14336, 4096): 3, # 16x32 tile, 16-way split-K
}
_CANDIDATES = {
"skinny": [0, 3, 4, 5, 6], # M <= 16
"small": [1, 7, 5, 6], # 16 < M <= 32
"wide": [2, 4, 8, 5, 6], # M > 32
}
def _pick_cfg(M: int, N: int, K: int) -> int:
cfg = _BY_SHAPE.get((M, N, K))
if cfg is not None:
return cfg
kc = K // 128
group = "skinny" if M <= 16 else ("small" if M <= 32 else "wide")
for cand in _CANDIDATES[group]:
bn, wk, _ = _CFG_GEOM[cand]
if N % bn == 0 and kc % wk == 0:
return cand
raise RuntimeError(f"no kernel config for M={M} N={N} K={K}")
try: # cheapest way to read the current stream from Python
_raw_stream = torch._C._cuda_getCurrentRawStream
except AttributeError: # pragma: no cover
_raw_stream = None
class _Launch:
"""Everything the steady-state forward needs, resolved once."""
__slots__ = ("m", "dev", "run", "outs", "idx")
def __init__(self, m, dev, run, outs):
self.m = m
self.dev = dev
self.run = run
self.outs = outs
self.idx = 0
class Model(nn.Module):
"""y = x @ dequant(w_q, scales, zeros); int4 weights stay packed."""
def __init__(self, M: int, N: int, K: int, group_size: int = GROUP_SIZE):
super().__init__()
assert K % group_size == 0, "K must be divisible by group_size"
assert K % 2 == 0, "K must be even (int4 packing)"
self.M, self.N, self.K = M, N, K
self.group_size = group_size
n_groups = K // group_size
# Registered as buffers (not Parameters) so the reference state_dict
# loads into them with strict=True.
self.register_buffer("w_q", torch.zeros((K // 2, N), dtype=torch.uint8))
self.register_buffer("scales", torch.zeros((n_groups, N), dtype=torch.bfloat16))
self.register_buffer("zeros", torch.zeros((n_groups, N), dtype=torch.bfloat16))
self._packed = None
self._fast = None
# -- repack caches are invalidated whenever the weights change or move ----
def _apply(self, fn, recurse=True):
self._packed = None
self._fast = None
return super()._apply(fn, recurse)
def _load_from_state_dict(self, *args, **kwargs):
self._packed = None
self._fast = None
return super()._load_from_state_dict(*args, **kwargs)
def _pack(self, want4: bool):
w_q, sc, zp = self.w_q, self.scales, self.zeros
key = (w_q.data_ptr(), sc.data_ptr(), zp.data_ptr(), w_q.device, want4)
if self._packed is not None and self._packed[0] == key:
return self._packed[1], self._packed[2]
w_c = w_q.contiguous()
wp = _prepack_w4(w_c, self.N, self.K) if want4 else _prepack_w(w_c, self.N, self.K)
sz = _prepack_sz(sc, zp, self.N)
self._packed = (key, wp, sz)
return wp, sz
def _setup(self, x: torch.Tensor) -> _Launch:
lib = _load_lib()
M, N, K = x.shape[0], self.N, self.K
cfg = _pick_cfg(M, N, K)
wp, sz = self._pack(_CFG_GEOM[cfg][2])
launch = lib.w4a16_launch
wp_p, sz_p = wp.data_ptr(), sz.data_ptr()
dev = x.device
if _raw_stream is not None:
dev_idx = dev.index or 0
get_stream = lambda: _raw_stream(dev_idx) # noqa: E731
else: # pragma: no cover
get_stream = lambda: torch.cuda.current_stream(dev).cuda_stream # noqa: E731
def run(xx, out, _launch=launch, _cfg=cfg, _wp=wp_p, _sz=sz_p,
_M=M, _N=N, _K=K, _get=get_stream):
rc = _launch(_cfg, _wp, _sz, xx.data_ptr(), out.data_ptr(),
_M, _N, _K, _get())
if rc != 0:
raise RuntimeError(f"w4a16_launch failed: cudaError {rc}")
outs = [torch.empty((M, N), dtype=torch.bfloat16, device=dev) for _ in range(2)]
fast = _Launch(M, dev, run, outs)
self._fast = fast
return fast
def forward(self, x: torch.Tensor) -> torch.Tensor:
fast = self._fast
if fast is None or fast.m != x.shape[0] or fast.dev != x.device:
fast = self._setup(x)
if not x.is_contiguous():
x = x.contiguous()
i = fast.idx
fast.idx = i ^ 1
out = fast.outs[i]
fast.run(x, out)
return out
M = 1
N = 12288
K = 4096
def get_inputs():
x = torch.randn(M, K, dtype=torch.bfloat16)
return [x]
def get_init_inputs():
return [M, N, K]
20261017_215437_or-fable_deepseek_deepseek-v4.1-flash-1017_07_w4a16_gemm