KernelBench hard · RTX PRO 6000
FP8 GEMM DeepSeek V4.1 Flash (1017)
32.4%geomean peak fraction across shapes
manually audited: clean
Handwritten CUDA fp8 GEMM loaded through ctypes: real e4m3 x e4m3 mma.sync m16n8k32 with a cp.async multi-stage pipeline over swizzled shared memory and two tile configs (a skinny M<=64 path). The K=4127 shape gets a fused per-call pad kernel on x; the padded weight is cached and keyed on the weight's pointer and _version. No _scaled_mm and no Triton on the graded path.
harnessor-fableagent session2h 7mtotal wall2h 7mcheck8sbenchmark6soutput tokens411,719cost$34.52gpu-lock wait41mgpu-lock held27mregimecompute
Per-shape vs governing ceilingeach shape graded against whichever binds — fp8 compute or HBM bandwidth
4096×4096×40960.236 ms58.1%581 TFLOPS · 58% of 1,000 TF fp8 peak · also 0.28 TB/s (16% of HBM)
4096×4096×41270.259 ms53.5%535 TFLOPS · 53% of 1,000 TF fp8 peak · also 0.26 TB/s (14% of HBM)
32×8192×81920.080 ms5.3%0.85 TB/s · 47% of 1.8 TB/s HBM · also 53 TFLOPS (5% of compute)
4096×14336×40960.724 ms66.4%664 TFLOPS · 66% of 1,000 TF fp8 peak · also 0.27 TB/s (15% of HBM)
compute-bound memory-bound · bar + right column = official fraction of the ceiling (the geomean input)
geomean(58.1% · 53.5% · 5.3% · 66.4%) = 32.4%
Kernel source (redacted)
"""FP8 e4m3 GEMM: y = (x @ w.T) * weight_scale -> bf16.
Primary path: a hand-written CUDA kernel (compiled at first use with nvcc and
loaded through ctypes) that issues real fp8 x fp8 tensor-core MMAs
(mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32, fp32 accumulate) fed by a
multi-stage cp.async pipeline over xor-swizzled shared memory.
Fallback path: an equivalent Triton kernel (tl.dot on e4m3 operands), used only
if the CUDA toolchain is unavailable.
Fully custom, no vendor library calls.
"""
import ctypes
import hashlib
import os
import shutil
import subprocess
import tempfile
import torch
import torch.nn as nn
E4M3_MAX = 448.0
# --------------------------------------------------------------------------
# CUDA source (kernel + pad + launcher)
# --------------------------------------------------------------------------
_CUDA_SRC = r"""
#include <cstdint>
#include <cuda_bf16.h>
#define DEVI __device__ __forceinline__
DEVI uint32_t smem_addr(const void* p) { return (uint32_t)__cvta_generic_to_shared(p); }
// 16B async copy; src-size 0 zero-fills the smem destination (used for the
// out-of-range rows of an edge tile).
DEVI void cp_async16(uint32_t dst, const void* src, bool pred) {
int sz = pred ? 16 : 0;
asm volatile("cp.async.cg.shared.global [%0], [%1], 16, %2;\n" ::"r"(dst), "l"(src), "r"(sz));
}
DEVI void cp_commit() { asm volatile("cp.async.commit_group;\n"); }
template <int N>
DEVI void cp_wait() { asm volatile("cp.async.wait_group %0;\n" ::"n"(N)); }
DEVI void ldm_x4(uint32_t& r0, uint32_t& r1, uint32_t& r2, uint32_t& r3, uint32_t addr) {
asm volatile("ldmatrix.sync.aligned.m8n8.x4.shared.b16 {%0,%1,%2,%3}, [%4];\n"
: "=r"(r0), "=r"(r1), "=r"(r2), "=r"(r3)
: "r"(addr));
}
DEVI void mma_fp8(float* d, const uint32_t* a, const uint32_t* b) {
asm volatile(
"mma.sync.aligned.m16n8k32.row.col.f32.e4m3.e4m3.f32 "
"{%0,%1,%2,%3}, {%4,%5,%6,%7}, {%8,%9}, {%0,%1,%2,%3};\n"
: "+f"(d[0]), "+f"(d[1]), "+f"(d[2]), "+f"(d[3])
: "r"(a[0]), "r"(a[1]), "r"(a[2]), "r"(a[3]), "r"(b[0]), "r"(b[1]));
}
// The xor key must vary over a group of 8 consecutive rows so that one
// ldmatrix (8 rows x one 16B chunk) hits 8 distinct 16B bank groups. A row is
// BK bytes = NC chunks wide, i.e. NC bank groups, so rows advance by NC groups:
// with NC = 8 one row spans every group and the key is row&7; with NC = 4 a row
// only spans half of them and rows 0 and 2 collide, so the key follows row/2.
template <int BK>
DEVI int swz_row(int row) {
constexpr int NC = BK / 16;
return NC == 8 ? (row & 7) : (NC == 4 ? ((row >> 1) & 3) : (row & (NC - 1)));
}
template <int BK>
DEVI int swz_pre(int row) {
return row * BK + (swz_row<BK>(row) << 4);
}
// ---------------------------------------------------------------------------
// C[m,n] = sum_k A[m,k] * B[n,k] * scale[n] A,B fp8 e4m3, k-major
// ---------------------------------------------------------------------------
template <int BM, int BN, int BK, int WM, int WN, int STAGES, int GROUP_M, int MINB>
__global__ __launch_bounds__(WM* WN * 32, MINB) void fp8_gemm_kernel(
const uint8_t* __restrict__ A, const uint8_t* __restrict__ B,
__nv_bfloat16* __restrict__ C, const float* __restrict__ scale,
int M, int N, int K) {
constexpr int NWARP = WM * WN;
constexpr int NTHREADS = NWARP * 32;
constexpr int WTM = BM / WM;
constexpr int WTN = BN / WN;
constexpr int MT = WTM / 16;
constexpr int NT = WTN / 8;
constexpr int NC = BK / 16;
constexpr int CPT_A = BM * BK / 16 / NTHREADS;
constexpr int CPT_B = BN * BK / 16 / NTHREADS;
constexpr int NKA = BK / 32;
static_assert(BM * BK / 16 % NTHREADS == 0 && BN * BK / 16 % NTHREADS == 0, "chunk split");
static_assert(NT % 2 == 0, "NT even");
extern __shared__ uint8_t smem[];
uint8_t* sA = smem;
uint8_t* sB = smem + STAGES * BM * BK;
const int tid = threadIdx.x;
const int warp = tid / 32;
const int lane = tid % 32;
const int warp_m = warp / WN;
const int warp_n = warp % WN;
const int num_pid_m = (M + BM - 1) / BM;
const int num_pid_n = (N + BN - 1) / BN;
int pid = blockIdx.x;
int group_sz = GROUP_M * num_pid_n;
int gid = pid / group_sz;
int first_m = gid * GROUP_M;
int gsz = min(num_pid_m - first_m, GROUP_M);
int pid_m = first_m + ((pid % group_sz) % gsz);
int pid_n = (pid % group_sz) / gsz;
const int m0 = pid_m * BM;
const int n0 = pid_n * BN;
const int nk = K / BK;
// ---- global -> smem chunk mapping (constant per thread) ----
int awoff[CPT_A], boff[CPT_B];
size_t asrc[CPT_A], bsrc[CPT_B];
bool apred[CPT_A], bpred[CPT_B];
#pragma unroll
for (int i = 0; i < CPT_A; i++) {
int chunk = tid + i * NTHREADS;
int r = chunk / NC, c = chunk % NC;
awoff[i] = swz_pre<BK>(r) ^ (c << 4);
asrc[i] = (size_t)(m0 + r) * K + c * 16;
apred[i] = (m0 + r) < M;
}
#pragma unroll
for (int i = 0; i < CPT_B; i++) {
int chunk = tid + i * NTHREADS;
int r = chunk / NC, c = chunk % NC;
boff[i] = swz_pre<BK>(r) ^ (c << 4);
bsrc[i] = (size_t)(n0 + r) * K + c * 16;
bpred[i] = (n0 + r) < N;
}
// ---- ldmatrix offsets ----
// An mma A operand is (rows 0-7, k lo), (rows 8-15, k lo), (rows 0-7, k hi),
// (rows 8-15, k hi) -- already the order ldmatrix.x4 returns, so matrix t/8
// selects row += 8 when (t/8)&1 and k-half (t/8)/2.
// A B operand is (n 0-7, k lo), (n 0-7, k hi) for one mma and the same pair
// for n 8-15 for the next, so the mapping is swapped (row += 8 when (t/8)/2,
// k-half (t/8)&1). Matching these keeps the ldmatrix results in the operand
// quads mma wants; otherwise ptxas inserts four MOVs per fragment.
const int lane_row = lane & 7;
const int lane_mat = lane >> 3;
const int kx_a = ((lane >> 4) & 1) << 4;
const int kx_b = (lane_mat & 1) << 4;
int aoff[MT], boffl[NT / 2];
#pragma unroll
for (int mt = 0; mt < MT; mt++) {
int row = warp_m * WTM + mt * 16 + lane_row + (lane_mat & 1) * 8;
aoff[mt] = swz_pre<BK>(row);
}
#pragma unroll
for (int nt2 = 0; nt2 < NT / 2; nt2++) {
int row = warp_n * WTN + nt2 * 16 + lane_row + ((lane_mat >> 1) & 1) * 8;
boffl[nt2] = swz_pre<BK>(row);
}
float acc[MT][NT][4];
#pragma unroll
for (int i = 0; i < MT; i++)
#pragma unroll
for (int j = 0; j < NT; j++)
#pragma unroll
for (int q = 0; q < 4; q++) acc[i][j][q] = 0.f;
auto load_tile = [&](int stage, int kb, bool valid) {
uint32_t dA = smem_addr(sA + stage * BM * BK);
uint32_t dB = smem_addr(sB + stage * BN * BK);
const uint8_t* pa = A + (size_t)kb * BK;
const uint8_t* pb = B + (size_t)kb * BK;
#pragma unroll
for (int i = 0; i < CPT_A; i++) cp_async16(dA + awoff[i], pa + asrc[i], valid && apred[i]);
#pragma unroll
for (int i = 0; i < CPT_B; i++) cp_async16(dB + boff[i], pb + bsrc[i], valid && bpred[i]);
};
#pragma unroll
for (int s = 0; s < STAGES - 1; s++) {
load_tile(s, s, true);
cp_commit();
}
// Fragment registers, double buffered: the ldmatrix for k-step ks+1 is
// issued ahead of the mmas that consume k-step ks.
uint32_t fa[2][MT][4], fb[2][NT][2];
auto load_frags = [&](uint32_t baseA, uint32_t baseB, int ks, int buf) {
const uint32_t kxa = kx_a + (ks << 5);
const uint32_t kxb = kx_b + (ks << 5);
#pragma unroll
for (int mt = 0; mt < MT; mt++)
ldm_x4(fa[buf][mt][0], fa[buf][mt][1], fa[buf][mt][2], fa[buf][mt][3],
baseA + (aoff[mt] ^ kxa));
#pragma unroll
for (int nt2 = 0; nt2 < NT / 2; nt2++) {
ldm_x4(fb[buf][nt2 * 2][0], fb[buf][nt2 * 2][1],
fb[buf][nt2 * 2 + 1][0], fb[buf][nt2 * 2 + 1][1],
baseB + (boffl[nt2] ^ kxb));
}
};
// Prefetching across a k-block boundary reads the next smem stage, which is
// only resident if we hold one more group in flight (WAITN = STAGES-3).
constexpr bool XPREF = (STAGES >= 3);
constexpr int WAITN = XPREF ? (STAGES - 3) : (STAGES - 2);
int wr = 0;
int cur = 0;
if (XPREF) {
cp_wait<WAITN>();
__syncthreads();
load_frags(smem_addr(sA), smem_addr(sB), 0, 0);
}
for (int kb = 0; kb < nk; kb++) {
cp_wait<WAITN>();
__syncthreads();
int ws = (kb + STAGES - 1) % STAGES;
int wkb = kb + STAGES - 1;
load_tile(ws, wkb < nk ? wkb : 0, wkb < nk);
cp_commit();
const uint32_t baseA = smem_addr(sA + wr * BM * BK);
const uint32_t baseB = smem_addr(sB + wr * BN * BK);
const uint32_t nxtA = smem_addr(sA + ((wr + 1) % STAGES) * BM * BK);
const uint32_t nxtB = smem_addr(sB + ((wr + 1) % STAGES) * BN * BK);
if (!XPREF) load_frags(baseA, baseB, 0, cur);
#pragma unroll
for (int ks = 0; ks < NKA; ks++) {
int nbuf = cur ^ 1;
if (ks + 1 < NKA) {
load_frags(baseA, baseB, ks + 1, nbuf);
} else if (XPREF) {
load_frags(nxtA, nxtB, 0, nbuf);
}
#pragma unroll
for (int mt = 0; mt < MT; mt++)
#pragma unroll
for (int nt = 0; nt < NT; nt++) mma_fp8(acc[mt][nt], fa[cur][mt], fb[cur][nt]);
cur = nbuf;
}
wr = (wr + 1) % STAGES;
}
// ---- epilogue: per-output-channel dequant scale, bf16 store ----
int row0 = m0 + warp_m * WTM + (lane >> 2);
int col0 = n0 + warp_n * WTN + (lane & 3) * 2;
const bool nfull = (n0 + BN <= N); // false only for a partial last n-tile
#pragma unroll
for (int i = 0; i < MT; i++) {
int r0 = row0 + i * 16;
int r1 = r0 + 8;
bool p0 = r0 < M, p1 = r1 < M;
#pragma unroll
for (int j = 0; j < NT; j++) {
int col = col0 + j * 8;
if (nfull || (col + 1 < N)) { // guard both the scale read and the store
float s0 = scale[col];
float s1 = scale[col + 1];
__nv_bfloat162 o0 = __floats2bfloat162_rn(acc[i][j][0] * s0, acc[i][j][1] * s1);
__nv_bfloat162 o1 = __floats2bfloat162_rn(acc[i][j][2] * s0, acc[i][j][3] * s1);
if (p0) *(uint32_t*)(C + (size_t)r0 * N + col) = *(uint32_t*)&o0;
if (p1) *(uint32_t*)(C + (size_t)r1 * N + col) = *(uint32_t*)&o1;
}
}
}
}
// ---------------------------------------------------------------------------
// (M, K) fp8 -> (M, KP) fp8, zero padded. One block per row.
// Destination rows are 16B aligned (KP % 16 == 0) while a source row can begin
// at any byte offset, so each 16B destination block is assembled from the two
// aligned 16B source blocks that straddle it with word funnel shifts: fully
// coalesced 16B loads and 16B stores, no byte-granular traffic.
// ---------------------------------------------------------------------------
#define PAD_FS(a, b) __funnelshift_r((a), (b), bsh)
__global__ void fp8_pad_kernel(const uint8_t* __restrict__ src, uint8_t* __restrict__ dst,
int K, int KP, int nbytes) {
const int r = blockIdx.x;
const uint8_t* s0 = src + (size_t)r * K;
uint8_t* d0 = dst + (size_t)r * KP;
const uintptr_t base = (uintptr_t)s0 & ~(uintptr_t)15;
const int sh = (int)((uintptr_t)s0 - base); // row phase, 0..15
const int q = sh >> 2, bsh = (sh & 3) * 8;
const size_t base_off = (size_t)(base - (uintptr_t)src);
const int nb = KP >> 4;
for (int j = threadIdx.x; j < nb; j += blockDim.x) {
const int col = j << 4;
if (col >= K) { // padding region
*(uint4*)(d0 + col) = make_uint4(0, 0, 0, 0);
continue;
}
const int valid = K - col;
const size_t off = base_off + (size_t)col;
if (off + 32 > (size_t)nbytes) {
// Final block of the final row: the 32B window would read past the buffer,
// and re-basing the window would break the shift phase. Copy bytes.
uint8_t* dv = d0 + col;
for (int i = 0; i < 16; i++) dv[i] = (i < valid) ? s0[col + i] : (uint8_t)0;
continue;
}
const uint4 lo = *(const uint4*)(src + off);
const uint4 hi = *(const uint4*)(src + off + 16);
const uint32_t w0 = lo.x, w1 = lo.y, w2 = lo.z, w3 = lo.w;
const uint32_t w4 = hi.x, w5 = hi.y, w6 = hi.z, w7 = hi.w;
uint32_t a, b, c, d;
switch (q) {
case 0: a = PAD_FS(w0, w1); b = PAD_FS(w1, w2); c = PAD_FS(w2, w3); d = PAD_FS(w3, w4); break;
case 1: a = PAD_FS(w1, w2); b = PAD_FS(w2, w3); c = PAD_FS(w3, w4); d = PAD_FS(w4, w5); break;
case 2: a = PAD_FS(w2, w3); b = PAD_FS(w3, w4); c = PAD_FS(w4, w5); d = PAD_FS(w5, w6); break;
default: a = PAD_FS(w3, w4); b = PAD_FS(w4, w5); c = PAD_FS(w5, w6); d = PAD_FS(w6, w7); break;
}
if (valid < 16) { // last block of the row: keep 'valid' bytes, zero the rest
uint4 o = make_uint4(a, b, c, d);
uint8_t* dv = d0 + col;
const uint8_t* ov = (const uint8_t*)&o;
for (int i = 0; i < 16; i++) dv[i] = (i < valid) ? ov[i] : (uint8_t)0;
continue;
}
*(uint4*)(d0 + col) = make_uint4(a, b, c, d);
}
}
// ---------------------------------------------------------------------------
#define LAUNCH(BM, BN, BK, WM, WN, ST, GM, MB) \
{ \
constexpr int SM = (ST) * ((BM) + (BN)) * (BK); \
auto kfn = fp8_gemm_kernel<BM, BN, BK, WM, WN, ST, GM, MB>; \
static bool init_##BM##_##BN##_##BK##_##WM##_##WN##_##ST##_##GM##_##MB = false; \
if (!init_##BM##_##BN##_##BK##_##WM##_##WN##_##ST##_##GM##_##MB) { \
cudaFuncSetAttribute(kfn, cudaFuncAttributeMaxDynamicSharedMemorySize, SM); \
init_##BM##_##BN##_##BK##_##WM##_##WN##_##ST##_##GM##_##MB = true; \
} \
dim3 grid(((M + (BM) - 1) / (BM)) * ((N + (BN) - 1) / (BN))); \
kfn<<<grid, (WM) * (WN) * 32, SM, stream>>>( \
(const uint8_t*)Ause, (const uint8_t*)B, (__nv_bfloat16*)C, (const float*)S, \
M, N, K); \
}
extern "C" void fp8_gemm_forward(const void* A, const void* B, void* C, const void* S,
int M, int N, int K, int KP, void* As,
int cfg, cudaStream_t stream) {
const void* Ause = A;
if (KP != K) {
fp8_pad_kernel<<<M, 256, 0, stream>>>((const uint8_t*)A, (uint8_t*)As, K, KP, M * K);
K = KP;
Ause = As;
}
switch (cfg) {
case 0: LAUNCH(128, 256, 64, 2, 4, 4, 8, 1); break; // 8 warps, 64x64 tile
case 1: LAUNCH(32, 64, 128, 1, 2, 8, 1, 4); break; // skinny M
default: break;
}
}
extern "C" int fp8_gemm_smem(int cfg) {
switch (cfg) {
case 0: return 4 * (128 + 256) * 64;
case 1: return 8 * (32 + 64) * 128;
default: return -1;
}
}
"""
# --------------------------------------------------------------------------
# Build / load
# --------------------------------------------------------------------------
def _cc_arch():
cap = torch.cuda.get_device_capability()
arch = "sm_%d%d" % (cap[0], cap[1])
if cap[0] >= 9:
arch += "a"
return arch
def _build_lib():
nvcc = shutil.which("nvcc")
if nvcc is None:
cand = os.path.join(os.environ.get("CUDA_HOME", "/usr/local/cuda"), "bin", "nvcc")
nvcc = cand if os.path.exists(cand) else None
if nvcc is None:
return None
arch = _cc_arch()
tag = hashlib.sha256((_CUDA_SRC + arch).encode()).hexdigest()[:16]
so = os.path.join(tempfile.gettempdir(), "fp8gemm_%s.so" % tag)
if not os.path.exists(so):
with tempfile.TemporaryDirectory() as td:
cu = os.path.join(td, "k.cu")
with open(cu, "w") as f:
f.write(_CUDA_SRC)
tmp_so = so + ".%d.tmp" % os.getpid()
cmd = [nvcc, "-O3", "-arch", arch, "-std=c++17", "--shared",
"-Xcompiler", "-fPIC", "-o", tmp_so, cu]
r = subprocess.run(cmd, capture_output=True, timeout=600)
if r.returncode != 0:
return None
os.replace(tmp_so, so)
lib = ctypes.CDLL(so)
lib.fp8_gemm_forward.restype = None
lib.fp8_gemm_forward.argtypes = ([ctypes.c_void_p] * 4 + [ctypes.c_int] * 4 +
[ctypes.c_void_p, ctypes.c_int, ctypes.c_void_p])
return lib
_LIB = None
_LIB_DONE = False
# Fast, allocation-free stream lookup: the raw variant avoids building a Stream
# object (~2.3us) and still tracks the current stream, so a cached call is safe.
_RAW_STREAM = getattr(torch._C, "_cuda_getCurrentRawStream", None)
def _get_lib():
global _LIB, _LIB_DONE
if not _LIB_DONE:
_LIB_DONE = True
try:
_LIB = _build_lib()
except Exception:
_LIB = None
return _LIB
# --------------------------------------------------------------------------
# Triton fallback
# --------------------------------------------------------------------------
def _triton_fallback():
import triton
import triton.language as tl
@triton.jit
def gemm_kernel(a_ptr, b_ptr, c_ptr, s_ptr, M, N, K, stride_am, stride_bn,
stride_cm, BLOCK_M: tl.constexpr, BLOCK_N: tl.constexpr,
BLOCK_K: tl.constexpr, GROUP_M: tl.constexpr):
pid = tl.program_id(0)
num_pid_m = tl.cdiv(M, BLOCK_M)
num_pid_n = tl.cdiv(N, BLOCK_N)
num_pid_in_group = GROUP_M * num_pid_n
group_id = pid // num_pid_in_group
first_pid_m = group_id * GROUP_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_M)
pid_m = first_pid_m + ((pid % num_pid_in_group) % group_size_m)
pid_n = (pid % num_pid_in_group) // group_size_m
rm = pid_m * BLOCK_M + tl.arange(0, BLOCK_M)
rn = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
rk = tl.arange(0, BLOCK_K)
a_ptrs = a_ptr + rm[:, None] * stride_am + rk[None, :]
b_ptrs = b_ptr + rn[None, :] * stride_bn + rk[:, None]
acc = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
for _ in range(0, K // BLOCK_K):
a = tl.load(a_ptrs)
b = tl.load(b_ptrs)
acc = tl.dot(a, b, acc)
a_ptrs += BLOCK_K
b_ptrs += BLOCK_K
s = tl.load(s_ptr + rn)
acc = acc * s[None, :]
c_ptrs = c_ptr + rm[:, None] * stride_cm + rn[None, :]
tl.store(c_ptrs, acc.to(tl.bfloat16))
@triton.jit
def pad_kernel(src, dst, M, K, KP, BLOCK: tl.constexpr):
pid = tl.program_id(0)
npg = tl.num_programs(0)
rows_per = tl.cdiv(M, npg)
r0 = pid * rows_per
r1 = min(r0 + rows_per, M)
for r in range(r0, r1):
for c in range(0, KP, BLOCK):
offs = c + tl.arange(0, BLOCK)
v = tl.load(src + r * K + offs, mask=offs < K, other=0.0)
tl.store(dst + r * KP + offs, v, mask=offs < KP)
return gemm_kernel, pad_kernel
_TRITON = None
# --------------------------------------------------------------------------
# Model
# --------------------------------------------------------------------------
class Model(nn.Module):
def __init__(self, M: int, N: int, K: int):
super().__init__()
self.M, self.N, self.K = M, N, K
w = torch.empty(N, K, dtype=torch.bfloat16)
nn.init.normal_(w, std=0.02)
s = (w.float().abs().amax(dim=1, keepdim=True) / E4M3_MAX).clamp(min=1e-12)
w_fp8 = (w.float() / s).to(torch.float8_e4m3fn)
self.register_buffer("weight", w_fp8)
self.register_buffer("weight_scale", s.squeeze(1).to(torch.float32))
self._wpad = None
self._wkey = None
self._xpad = None
self._fast = None
# ---- padded weight (cached across calls; the weight never changes) ----
def _padded_weight(self, KP):
w = self.weight
key = (w.data_ptr(), w._version, KP)
if self._wkey != key:
if self._wpad is None or self._wpad.shape != (self.N, KP) or \
self._wpad.device != w.device:
self._wpad = torch.zeros((self.N, KP), device=w.device,
dtype=torch.float8_e4m3fn)
self._wpad.zero_()
# copy through uint8 views: plain strided memcpy, no fp8 kernels
self._wpad.view(torch.uint8)[:, :w.shape[1]].copy_(w.view(torch.uint8))
self._wkey = key
return self._wpad
def forward(self, x: torch.Tensor) -> torch.Tensor:
# Steady-state path: everything that depends only on the shapes (launcher,
# pointers, output buffer, tile config) is cached, so a call is one stream
# query plus one ctypes invocation. Invalidated when the weight or the
# weight_scale buffer is rewritten in place (numeric-stress rescales it).
f = self._fast
if (f is not None and f[0] == x.shape and f[1] == self.weight._version
and f[2] == self.weight_scale._version
and f[3] == self.weight.data_ptr() and x.is_contiguous()):
cfn, wsptr, out, outptr, bptr, KP, M, N, K, cfg, dev = f[4:]
xp = self._xpad if KP != K else x
cfn(x.data_ptr(), bptr, outptr, wsptr, M, N, K, KP, xp.data_ptr(), cfg,
_RAW_STREAM(dev) if _RAW_STREAM is not None
else torch.cuda.current_stream(dev).cuda_stream)
return out
return self._forward_build(x)
def _forward_build(self, x: torch.Tensor) -> torch.Tensor:
x = x.contiguous()
M, K = x.shape
N = self.N
dev = x.device
ws = self.weight_scale
cfg = 1 if M <= 64 else 0
bk = 128 if cfg == 1 else 64 # every kernel's BK must divide KP
lib = _get_lib()
if lib is not None:
if K % bk != 0:
KP = (K + bk - 1) // bk * bk
w = self._padded_weight(KP)
if self._xpad is None or self._xpad.shape != (M, KP) or \
self._xpad.device != dev:
self._xpad = torch.empty((M, KP), device=dev,
dtype=torch.float8_e4m3fn)
xp = self._xpad
else:
KP = K
w = self.weight
xp = x
out = torch.empty((M, N), device=dev, dtype=torch.bfloat16)
stream = torch.cuda.current_stream().cuda_stream
lib.fp8_gemm_forward(x.data_ptr(), w.data_ptr(), out.data_ptr(),
ws.data_ptr(), M, N, K, KP, xp.data_ptr(),
cfg, stream)
self._fast = (x.shape, self.weight._version, self.weight_scale._version,
self.weight.data_ptr(), lib.fp8_gemm_forward,
ws.data_ptr(), out, out.data_ptr(), w.data_ptr(),
KP, M, N, K, cfg, dev.index if dev.index is not None else 0)
return out
# ---- Triton fallback ----
global _TRITON
if _TRITON is None:
_TRITON = _triton_fallback()
gemm_kernel = _TRITON[0]
import triton
w = self.weight
if K % 128 != 0:
KP = (K + 127) // 128 * 128
w = self._padded_weight(KP)
if self._xpad is None or self._xpad.shape != (M, KP) or \
self._xpad.device != dev:
self._xpad = torch.zeros((M, KP), device=dev, dtype=torch.float8_e4m3fn)
_TRITON[1][(min(512, M),)](x, self._xpad, M, K, KP, BLOCK=1024)
x = self._xpad
K = KP
bm, bn, bkk = (32, 64, 128) if M <= 64 else (128, 256, 128)
out = torch.empty((M, N), device=dev, dtype=torch.bfloat16)
grid = (triton.cdiv(M, bm) * triton.cdiv(N, bn),)
gemm_kernel[grid](x, w, out, ws, M, N, K, x.stride(0), w.stride(0),
out.stride(0), BLOCK_M=bm, BLOCK_N=bn, BLOCK_K=bkk,
GROUP_M=1 if M <= 64 else 8,
num_warps=4 if M <= 64 else 8,
num_stages=4 if M <= 64 else 3)
return out
20261017_215207_or-fable_deepseek_deepseek-v4.1-flash-1017_01_fp8_gemm