MXFP8 decode GEMV on tcgen05: y[M, N] = x[M, K] @ W[N, K]^T for M <= 8, block-scaled E4M3 x E4M3.
The weight is the MMA's A operand (128 output features per CTA), the activation is B (8 token columns),
fp32 accumulation in TMEM. Both operands come in their MXFP8 storage: E4M3 values (K contiguous) and UE8M0
scales in the 128x4-swizzled layout, whose 512-byte atom per (128 rows, 128 K) is exactly the source of a
tcgen05.cp 32x128b.warpx4 scale copy, so both scale tensors are bulk-copied unchanged. For an activation of at
most 32 rows the atom of k-tile k sits at byte 512 k.
A cluster of SPLIT CTAs shares one 128-row weight tile; rank r streams the interleaved k-tiles r, r + SPLIT, ...
through a RING-stage ring. Before the grid dependency each CTA issues its first RING weight tiles (and their scale
atoms) and prefetches the rest into L2; after it, the activation slice of the CTA (values and scale atoms) lands in
one buffer. Rank r owns rows [r * 128 / SPLIT, (r + 1) * 128 / SPLIT) of the tile: the other ranks store their fp32
partials of those rows into its shared memory and arrive on its barrier (release, cluster scope); the owner adds the
SPLIT partials in rank order and stores bf16.
Warps: 0 weight TMA, 1 activation TMA, 2 TMEM allocation + scale copies + MMA, 3 idle, 4-7 epilogue (warp 4 + i
reads TMEM lanes 32 i .. 32 i + 31).
_absmax(acc, v, *, loc=None, ip=None)
max(acc, |v|).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _absmax(acc, v, *, loc=None, ip=None):
"""max(acc, |v|)."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
return cutlass.Float32(
_llvm.inline_asm(
_T.f32(), [cutlass.Float32(acc).ir_value(loc=loc, ip=ip), cutlass.Float32(v).ir_value(loc=loc, ip=ip)],
"{\n\t.reg .f32 a;\n\tabs.f32 a, $2;\n\tmax.f32 $0, $1, a;\n\t}", "=f,f,f", has_side_effects=False,
is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
|
_bf16_bits(x, *, loc=None, ip=None)
bf16(x) (RN) as the fp32 bit pattern of the rounded value.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _bf16_bits(x, *, loc=None, ip=None):
"""bf16(x) (RN) as the fp32 bit pattern of the rounded value."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_llvm.inline_asm(
_T.i32(), [cutlass.Float32(x).ir_value(loc=loc, ip=ip)],
"{\n\t.reg .b16 h, z;\n\tcvt.rn.bf16.f32 h, $1;\n\tmov.b16 z, 0;\n\tmov.b32 $0, {z, h};\n\t}", "=r,f",
has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
|
_bf16x2_word(lo, hi, *, loc=None, ip=None)
bf16(lo) | bf16(hi) << 16 (RN), with -0.0 halves turned into +0.0 (0x80000000 marks an empty exchange word).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _bf16x2_word(lo, hi, *, loc=None, ip=None):
"""bf16(lo) | bf16(hi) << 16 (RN), with -0.0 halves turned into +0.0 (0x80000000 marks an empty exchange word)."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_llvm.inline_asm(
_T.i32(), [cutlass.Float32(hi).ir_value(loc=loc, ip=ip), cutlass.Float32(lo).ir_value(loc=loc, ip=ip)],
"{\n\t.reg .b32 w, l, h;\n\t.reg .pred pl, ph;\n\tcvt.rn.bf16x2.f32 w, $1, $2;\n\t"
"and.b32 l, w, 0xFFFF;\n\tsetp.eq.u32 pl, l, 0x8000;\n\t@pl and.b32 w, w, 0xFFFF0000;\n\t"
"shr.u32 h, w, 16;\n\tsetp.eq.u32 ph, h, 0x8000;\n\t@ph and.b32 w, w, 0x0000FFFF;\n\t"
"mov.b32 $0, w;\n\t}", "=r,f,f",
has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
|
_din_load4(table, hidden, hidden_half, word_idx)
Four int32 words at word_idx of the hidden rows (hidden_half == 1) or the embedding table: one CTA-uniform
base-address select instead of a branch.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| def _din_load4(table, hidden, hidden_half, word_idx):
"""Four int32 words at word_idx of the hidden rows (hidden_half == 1) or the embedding table: one CTA-uniform
base-address select instead of a branch."""
addr = cutlass.select_(
hidden_half == cutlass.Int32(1),
hidden.data_ptr(0).toint(),
table.data_ptr(0).toint(),
)
ptr = cutlass.inttoptr(
addr + cutlass.Int64(word_idx) * cutlass.Int64(4),
mem_space=1,
dtype=cutlass.Int32,
)
return ptr.load(count=4, alignment=16)
|
_e4m3x4(v0, v1, v2, v3, *, loc=None, ip=None)
Four f32 to E4M3 (RN, satfinite), packed little-endian into one int32 word (v0 in the low byte).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _e4m3x4(v0, v1, v2, v3, *, loc=None, ip=None):
"""Four f32 to E4M3 (RN, satfinite), packed little-endian into one int32 word (v0 in the low byte)."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_llvm.inline_asm(
_T.i32(), [cutlass.Float32(v).ir_value(loc=loc, ip=ip) for v in (v0, v1, v2, v3)],
"{\n\t.reg .b16 lo, hi;\n\tcvt.rn.satfinite.e4m3x2.f32 lo, $2, $1;\n\t"
"cvt.rn.satfinite.e4m3x2.f32 hi, $4, $3;\n\tmov.b32 $0, {lo, hi};\n\t}", "=r,f,f,f,f",
has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
|
_ld_shared_v4_f32(smem_ptr, *, loc=None, ip=None)
ld.shared.v4.f32 of four fp32 at a 16-byte-aligned shared-memory pointer.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ld_shared_v4_f32(smem_ptr, *, loc=None, ip=None):
"""ld.shared.v4.f32 of four fp32 at a 16-byte-aligned shared-memory pointer."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _llvm.inline_asm(
_llvm.StructType.get_literal([_T.f32()] * 4), [smem_ptr.toint(loc=loc, ip=ip).ir_value()],
"ld.shared.v4.f32 {$0, $1, $2, $3}, [$4];", "=f,=f,=f,=f,r", has_side_effects=True,
is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
) # fmt: skip
return [
cutlass.Float32(_llvm.extractvalue(_T.f32(), out, [i], loc=loc, ip=ip))
for i in range(4)
]
|
_mapa_u32(smem_ptr, peer, *, loc=None, ip=None)
The shared::cluster address of this CTA's shared-memory location in cluster CTA peer.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _mapa_u32(smem_ptr, peer, *, loc=None, ip=None):
"""The shared::cluster address of this CTA's shared-memory location in cluster CTA `peer`."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_llvm.inline_asm(
_T.i32(), [smem_ptr.toint(loc=loc, ip=ip).ir_value(), cutlass.Int32(peer).ir_value(loc=loc, ip=ip)],
"mapa.shared::cluster.u32 $0, $1, $2;", "=r,r,r", has_side_effects=False,
is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
|
_p5_e8m0(x, *, loc=None, ip=None)
UE8M0 of x, rounded up, satfinite (trtllm mxfp8_quantize scale rule).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _p5_e8m0(x, *, loc=None, ip=None):
"""UE8M0 of x, rounded up, satfinite (trtllm mxfp8_quantize scale rule)."""
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_asm(
_T.i32(),
[cutlass.Float32(x)],
"{\n\t.reg .b16 h;\n\t.reg .b32 w;\n\tcvt.rp.satfinite.ue8m0x2.f32 h, 0f00000000, $1;\n\t"
"cvt.u32.u16 w, h;\n\tand.b32 $0, w, 255;\n\t}",
"=r,f",
side=False,
loc=loc,
ip=ip,
)
)
|
_p5_fix(addr, tag, w, active, *, loc=None, ip=None)
W was loaded from addr; while active and its tag is stale, re-poll (stragglers only).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _p5_fix(addr, tag, w, active, *, loc=None, ip=None):
"""W was loaded from addr; while active and its tag is stale, re-poll (stragglers only)."""
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_asm(
_T.i32(),
[
cutlass.Int64(addr),
cutlass.Int32(tag),
cutlass.Int32(w),
cutlass.Int32(active),
],
"{\n\t.reg .b32 w, g;\n\t.reg .pred p, a;\n\tmov.b32 w, $3;\n\tsetp.ne.u32 a, $4, 0;\n\t"
"@!a bra P5FD;\n\tP5FL:\n\tshr.u32 g, w, 16;\n\tsetp.ne.u32 p, g, $2;\n\t@!p bra P5FD;\n\t"
"ld.volatile.global.b32 w, [$1];\n\tbra P5FL;\n\tP5FD:\n\tmov.b32 $0, w;\n\t}",
"=r,l,r,r,r",
loc=loc,
ip=ip,
)
)
|
_p5_fix64(addr, tag, w, active, *, loc=None, ip=None)
Tagged rowsq word: re-poll while active and stale; returns the float payload.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _p5_fix64(addr, tag, w, active, *, loc=None, ip=None):
"""Tagged rowsq word: re-poll while active and stale; returns the float payload."""
from cutlass._mlir.extras import types as _T
return cutlass.Float32(
_asm(
_T.f32(),
[
cutlass.Int64(addr),
cutlass.Int32(tag),
cutlass.Int64(w),
cutlass.Int32(active),
],
"{\n\t.reg .b64 w;\n\t.reg .b32 lo, hi;\n\t.reg .pred p, a;\n\tmov.b64 w, $3;\n\t"
"setp.ne.u32 a, $4, 0;\n\tP5GL:\n\tmov.b64 {lo, hi}, w;\n\t@!a bra P5GD;\n\t"
"setp.ne.u32 p, hi, $2;\n\t@!p bra P5GD;\n\tld.relaxed.gpu.global.b64 w, [$1];\n\tbra P5GL;\n\t"
"P5GD:\n\tmov.b32 $0, lo;\n\t}",
"=f,l,r,l,r",
loc=loc,
ip=ip,
)
)
|
_p5_spin(addr, tag, *, loc=None, ip=None)
Poll a Lamport word until its tag (high 16 bits) matches; returns the word.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _p5_spin(addr, tag, *, loc=None, ip=None):
"""Poll a Lamport word until its tag (high 16 bits) matches; returns the word."""
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_asm(
_T.i32(),
[cutlass.Int64(addr), cutlass.Int32(tag)],
"{\n\t.reg .b32 w, g;\n\t.reg .pred p;\n\tP5SPIN:\n\tld.volatile.global.b32 w, [$1];\n\t"
"shr.u32 g, w, 16;\n\tsetp.ne.u32 p, g, $2;\n\t@p bra P5SPIN;\n\tmov.b32 $0, w;\n\t}",
"=r,l,r",
loc=loc,
ip=ip,
)
)
|
_p5_st_bf16(addr, x, *, loc=None, ip=None)
bf16(x) RN stored to addr (x is already bf16-exact here).
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _p5_st_bf16(addr, x, *, loc=None, ip=None):
"""bf16(x) RN stored to addr (x is already bf16-exact here)."""
_asm(
None,
[cutlass.Int64(addr), cutlass.Float32(x)],
"{\n\t.reg .b16 h;\n\tcvt.rn.bf16.f32 h, $1;\n\tst.global.b16 [$0], h;\n\t}",
"l,f",
loc=loc,
ip=ip,
)
|
_px_poll(addr, *, loc=None, ip=None)
Poll mode: a bf16-pair word of the decode output, re-polled while it is the 0xFFFFFFFF sentinel.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _px_poll(addr, *, loc=None, ip=None):
"""Poll mode: a bf16-pair word of the decode output, re-polled while it is the 0xFFFFFFFF sentinel."""
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_asm(
_T.i32(),
[cutlass.Int64(addr)],
"{\n\t.reg .b32 w;\n\t.reg .pred p;\n\tPXPL:\n\tld.volatile.global.b32 w, [$1];\n\t"
"setp.eq.u32 p, w, 0xFFFFFFFF;\n\t@p bra PXPL;\n\tmov.b32 $0, w;\n\t}",
"=r,l",
loc=loc,
ip=ip,
)
)
|
_ra3_bf4(p, *, loc=None, ip=None)
2 packed bf16 words -> 4 fp32 (exact)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra3_bf4(p, *, loc=None, ip=None):
"""2 packed bf16 words -> 4 fp32 (exact)"""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _asm(
_llvm.StructType.get_literal([_T.f32()] * 4),
[cutlass.Int32(p[0]), cutlass.Int32(p[1])],
"{\n\tshl.b32 $0, $4, 16;\n\tand.b32 $1, $4, 0xFFFF0000;\n\tshl.b32 $2, $5, 16;\n\t"
"and.b32 $3, $5, 0xFFFF0000;\n\t}",
"=f,=f,=f,=f,r,r",
loc=loc,
ip=ip,
) # side: no CSE across passes
return _f4(out, loc, ip)
|
_ra3_e4m3x4(w, *, loc=None, ip=None)
_ra_e4m3x4 kept per use (side effects: a CSE across the halves would hold 16 T fp32 live)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra3_e4m3x4(w, *, loc=None, ip=None):
"""_ra_e4m3x4 kept per use (side effects: a CSE across the halves would hold 16 T fp32 live)"""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _asm(
_llvm.StructType.get_literal([_T.f32()] * 4),
[cutlass.Int32(w)],
"{\n\t.reg .b16 lo, hi, a, b, c, d;\n\t.reg .b32 h0, h1;\n\tmov.b32 {lo, hi}, $4;\n\t"
"cvt.rn.f16x2.e4m3x2 h0, lo;\n\tcvt.rn.f16x2.e4m3x2 h1, hi;\n\tmov.b32 {a, b}, h0;\n\t"
"mov.b32 {c, d}, h1;\n\tcvt.f32.f16 $0, a;\n\tcvt.f32.f16 $1, b;\n\tcvt.f32.f16 $2, c;\n\t"
"cvt.f32.f16 $3, d;\n\t}",
"=f,=f,=f,=f,r",
loc=loc,
ip=ip,
)
return _f4(out, loc, ip)
|
_ra3_entry(tid, dst, mb_r, mb_a, src_r, src_a, *, loc=None, ip=None)
Thread 0: init both barriers, then W' (mb_r) and W'' (mb_a) by 16 KB bulk copies in one burst
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra3_entry(tid, dst, mb_r, mb_a, src_r, src_a, *, loc=None, ip=None):
"""Thread 0: init both barriers, then W' (mb_r) and W'' (mb_a) by 16 KB bulk copies in one burst"""
cpr = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{RA_R_OFF + 16384 * i}], "
f"[$4+{16384 * i}], 16384, [$2];\n"
for i in range(RA_R_BYTES // 16384)
)
cpa = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{16384 * i}], "
f"[$5+{16384 * i}], 16384, [$3];\n"
for i in range(RA_A_BYTES // 16384)
)
_asm(
None,
[
cutlass.Int32(tid),
cutlass.Int32(dst),
cutlass.Int32(mb_r),
cutlass.Int32(mb_a),
cutlass.Int64(src_r),
cutlass.Int64(src_a),
],
"{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p mbarrier.init.shared::cta.b64 [$2], 1;\n\t"
"@p mbarrier.init.shared::cta.b64 [$3], 1;\n\t@p fence.mbarrier_init.release.cluster;\n\t"
"@p fence.proxy.async.shared::cta;\n\t"
f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$2], {RA_R_BYTES};\n"
+ cpr
+ f"\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$3], {RA_A_BYTES};\n"
+ cpa
+ "\t}",
"r,r,r,r,l,l,~{memory}",
loc=loc,
ip=ip,
)
|
_ra3_ld2_cg(addr, *, loc=None, ip=None)
4 bf16 (8 B) at L2, kept packed
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra3_ld2_cg(addr, *, loc=None, ip=None):
"""4 bf16 (8 B) at L2, kept packed"""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _asm(
_llvm.StructType.get_literal([_T.i32()] * 2),
[cutlass.Int64(addr)],
"ld.global.cg.v2.b32 {$0, $1}, [$2];",
"=r,=r,l",
loc=loc,
ip=ip,
)
return [
cutlass.Int32(_llvm.extractvalue(_T.i32(), out, [i], loc=loc, ip=ip))
for i in range(2)
]
|
_ra3_wred(v, lane)
Warp reduce-scatter of len(v) (power of 2, <= 32) per-lane values: returns the lane's sum of output
lane >> (5 - log2 len(v)) over all 32 lanes (halving exchange, then plain butterflies)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| def _ra3_wred(v, lane):
"""Warp reduce-scatter of len(v) (power of 2, <= 32) per-lane values: returns the lane's sum of output
lane >> (5 - log2 len(v)) over all 32 lanes (halving exchange, then plain butterflies)"""
d = 16
while len(v) > 1:
half = len(v) // 2
up = (lane & cutlass.Int32(d)) != cutlass.Int32(0)
nv = []
for i in range(half):
snd = cutlass.select_(up, v[i], v[i + half])
kp = cutlass.select_(up, v[i + half], v[i])
nv.append(_ra_f("add.rn", kp, _ra_bfly(snd, d)))
v = nv
d //= 2
x = v[0]
while d >= 1:
x = _ra_f("add.rn", x, _ra_bfly(x, d))
d //= 2
return x
|
_ra_e4m3x4(w, *, loc=None, ip=None)
4 packed E4M3 (byte 0 first) -> 4 fp32 (exact)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_e4m3x4(w, *, loc=None, ip=None):
"""4 packed E4M3 (byte 0 first) -> 4 fp32 (exact)"""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _asm(
_llvm.StructType.get_literal([_T.f32()] * 4),
[cutlass.Int32(w)],
"{\n\t.reg .b16 lo, hi, a, b, c, d;\n\t.reg .b32 h0, h1;\n\tmov.b32 {lo, hi}, $4;\n\t"
"cvt.rn.f16x2.e4m3x2 h0, lo;\n\tcvt.rn.f16x2.e4m3x2 h1, hi;\n\tmov.b32 {a, b}, h0;\n\t"
"mov.b32 {c, d}, h1;\n\tcvt.f32.f16 $0, a;\n\tcvt.f32.f16 $1, b;\n\tcvt.f32.f16 $2, c;\n\t"
"cvt.f32.f16 $3, d;\n\t}",
"=f,=f,=f,=f,r",
side=False,
loc=loc,
ip=ip,
)
return _f4(out, loc, ip)
|
_ra_entry(tid, mb0, mb1, dst, src_a, src_r, *, loc=None, ip=None)
Thread 0: init both W'' barriers, the first half (experts 0-3, 64 KB) by 4 bulk copies on mb0; L2 prefetch of the
CTA's r blob
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_entry(tid, mb0, mb1, dst, src_a, src_r, *, loc=None, ip=None):
"""Thread 0: init both W'' barriers, the first half (experts 0-3, 64 KB) by 4 bulk copies on mb0; L2 prefetch of the
CTA's r blob"""
cp = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$3+{16384 * i}], "
f"[$4+{16384 * i}], 16384, [$1];\n"
for i in range(RA_A_BYTES // 2 // 16384)
)
_asm(
None,
[
cutlass.Int32(tid),
cutlass.Int32(mb0),
cutlass.Int32(mb1),
cutlass.Int32(dst),
cutlass.Int64(src_a),
cutlass.Int64(src_r),
],
"{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p mbarrier.init.shared::cta.b64 [$1], 1;\n\t"
"@p mbarrier.init.shared::cta.b64 [$2], 1;\n\t"
"@p fence.mbarrier_init.release.cluster;\n\t@p fence.proxy.async.shared::cta;\n\t"
f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], {RA_A_BYTES // 2};\n"
+ cp
+ f"\t@p cp.async.bulk.prefetch.L2.global [$5], {RA_R_BYTES};\n\t}}",
"r,r,r,r,l,l,~{memory}",
loc=loc,
ip=ip,
)
|
_ra_ld2bf_cg(addr, *, loc=None, ip=None)
4 bf16 at addr (8 B) as 4 fp32
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_ld2bf_cg(addr, *, loc=None, ip=None):
"""4 bf16 at addr (8 B) as 4 fp32"""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _asm(
_llvm.StructType.get_literal([_T.f32()] * 4),
[cutlass.Int64(addr)],
"{\n\t.reg .b32 a, b, c, d;\n\tld.global.cg.v2.b32 {a, c}, [$4];\n\tshl.b32 $0, a, 16;\n\t"
"and.b32 $1, a, 0xFFFF0000;\n\tshl.b32 $2, c, 16;\n\tand.b32 $3, c, 0xFFFF0000;\n\t}",
"=f,=f,=f,=f,l",
loc=loc,
ip=ip,
)
return _f4(out, loc, ip)
|
_ra_ld4b_cg(addr, *, loc=None, ip=None)
4 words at L2 (the decode's output: never an L1 copy)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_ld4b_cg(addr, *, loc=None, ip=None):
"""4 words at L2 (the decode's output: never an L1 copy)"""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
out = _asm(
_llvm.StructType.get_literal([_T.i32()] * 4),
[cutlass.Int64(addr)],
"ld.global.cg.v4.b32 {$0, $1, $2, $3}, [$4];",
"=r,=r,=r,=r,l",
loc=loc,
ip=ip,
)
return [
cutlass.Int32(_llvm.extractvalue(_T.i32(), out, [i], loc=loc, ip=ip))
for i in range(4)
]
|
_ra_load_a1(tid, mb1, dst, src_a, *, loc=None, ip=None)
Thread 0, after the CTA barrier that ends the generic-proxy use of the region: the second W'' half on mb1
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_load_a1(tid, mb1, dst, src_a, *, loc=None, ip=None):
"""Thread 0, after the CTA barrier that ends the generic-proxy use of the region: the second W'' half on mb1"""
cp = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$2+{16384 * i}], "
f"[$3+{16384 * i}], 16384, [$1];\n"
for i in range(RA_A_BYTES // 2 // 16384)
)
_asm(
None,
[
cutlass.Int32(tid),
cutlass.Int32(mb1),
cutlass.Int32(dst),
cutlass.Int64(src_a),
],
"{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p fence.proxy.async.shared::cta;\n\t"
f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$1], {RA_A_BYTES // 2};\n"
+ cp
+ "\t}",
"r,r,r,l,~{memory}",
loc=loc,
ip=ip,
)
|
_ra_scale_cg(addr, *, loc=None, ip=None)
UE8M0 byte at addr -> fp32 2^(s - 127) (s = 0: 2^-127)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_scale_cg(addr, *, loc=None, ip=None):
"""UE8M0 byte at addr -> fp32 2^(s - 127) (s = 0: 2^-127)"""
from cutlass._mlir.extras import types as _T
return cutlass.Float32(
_asm(
_T.f32(),
[cutlass.Int64(addr)],
"{\n\t.reg .b32 s, b;\n\t.reg .pred z;\n\tld.global.cg.u8 s, [$1];\n\t"
"shl.b32 b, s, 23;\n\tsetp.eq.u32 z, s, 0;\n\tselp.b32 $0, 0x00400000, b, z;\n\t}",
"=f,l",
loc=loc,
ip=ip,
)
)
|
_ra_st_tag1(addr, tag, z, on, *, loc=None, ip=None)
TP4: one rx word (part 0 = a + r) of one (token, src, expert): tag << 32 | fp32, relaxed sys
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_st_tag1(addr, tag, z, on, *, loc=None, ip=None):
"""TP4: one rx word (part 0 = a + r) of one (token, src, expert): tag << 32 | fp32, relaxed sys"""
_asm(
None,
[
cutlass.Int64(addr),
cutlass.Int32(tag),
cutlass.Float32(z),
cutlass.Int32(on),
],
"{\n\t.reg .pred p;\n\t.reg .b32 f;\n\t.reg .b64 q;\n\tsetp.ne.u32 p, $3, 0;\n\tmov.b32 f, $2;\n\t"
"mov.b64 q, {f, $1};\n\t@p st.relaxed.sys.global.b64 [$0], q;\n\t}",
"l,r,f,r",
loc=loc,
ip=ip,
)
|
_ra_st_tag2(addr, tag, za, zr, on, *, loc=None, ip=None)
Rx words (part 0 = a, part 1 = r) of one (token, src, expert): tag << 32 | fp32, relaxed sys (peer memory)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _ra_st_tag2(addr, tag, za, zr, on, *, loc=None, ip=None):
"""Rx words (part 0 = a, part 1 = r) of one (token, src, expert): tag << 32 | fp32, relaxed sys (peer memory)"""
_asm(
None,
[
cutlass.Int64(addr),
cutlass.Int32(tag),
cutlass.Float32(za),
cutlass.Float32(zr),
cutlass.Int32(on),
],
"{\n\t.reg .pred p;\n\t.reg .b32 f, g;\n\t.reg .b64 q, q1;\n\tsetp.ne.u32 p, $4, 0;\n\tmov.b32 f, $2;\n\t"
"mov.b32 g, $3;\n\tmov.b64 q, {f, $1};\n\tmov.b64 q1, {g, $1};\n\t"
f"@p st.relaxed.sys.global.b64 [$0], q;\n\t@p st.relaxed.sys.global.b64 [$0+{8 * 128}], q1;\n\t}}",
"l,r,f,f,r",
loc=loc,
ip=ip,
)
|
_rc_entry(tid, dst, mb0, mb1, mb2, src_r, src_a, *, loc=None, ip=None)
Thread 0: init 3 barriers; W' (96 KB, mb0) at 0 and W'' experts 0..RC_A0-1 (mb1) after it, one burst
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _rc_entry(tid, dst, mb0, mb1, mb2, src_r, src_a, *, loc=None, ip=None):
"""Thread 0: init 3 barriers; W' (96 KB, mb0) at 0 and W'' experts 0..RC_A0-1 (mb1) after it, one burst"""
cpr = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{16384 * i}], "
f"[$5+{16384 * i}], 16384, [$2];\n"
for i in range(RA_R_BYTES // 16384)
)
cpa = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{RA_R_BYTES + 16384 * i}], "
f"[$6+{16384 * i}], 16384, [$3];\n"
for i in range(RC_A0 * RC_E // 16384)
)
_asm(
None,
[
cutlass.Int32(tid),
cutlass.Int32(dst),
cutlass.Int32(mb0),
cutlass.Int32(mb1),
cutlass.Int32(mb2),
cutlass.Int64(src_r),
cutlass.Int64(src_a),
],
"{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p mbarrier.init.shared::cta.b64 [$2], 1;\n\t"
"@p mbarrier.init.shared::cta.b64 [$3], 1;\n\t@p mbarrier.init.shared::cta.b64 [$4], 1;\n\t"
"@p fence.mbarrier_init.release.cluster;\n\t@p fence.proxy.async.shared::cta;\n\t"
f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$2], {RA_R_BYTES};\n"
+ cpr
+ f"\t@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$3], {RC_A0 * RC_E};\n"
+ cpa
+ "\t}",
"r,r,r,r,r,l,l,~{memory}",
loc=loc,
ip=ip,
)
|
_rc_load_a(tid, dst, mb2, src_a, *, loc=None, ip=None)
Thread 0, after the CTA barrier that ends the generic reads of W': W'' experts RC_A0..7 over it (mb2)
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _rc_load_a(tid, dst, mb2, src_a, *, loc=None, ip=None):
"""Thread 0, after the CTA barrier that ends the generic reads of W': W'' experts RC_A0..7 over it (mb2)"""
n = (RA_E - RC_A0) * RC_E
cp = "".join(
f"\t@p cp.async.bulk.shared::cluster.global.mbarrier::complete_tx::bytes [$1+{16384 * i}], "
f"[$3+{RC_A0 * RC_E + 16384 * i}], 16384, [$2];\n"
for i in range(n // 16384)
)
_asm(
None,
[
cutlass.Int32(tid),
cutlass.Int32(dst),
cutlass.Int32(mb2),
cutlass.Int64(src_a),
],
"{\n\t.reg .pred p;\n\tsetp.eq.u32 p, $0, 0;\n\t@p fence.proxy.async.shared::cta;\n\t"
f"@p mbarrier.arrive.expect_tx.shared::cta.b64 _, [$2], {n};\n" + cp + "\t}",
"r,r,r,l,~{memory}",
loc=loc,
ip=ip,
)
|
_rope_bits(c, a, s, b, *, loc=None, ip=None)
bf16(c * a + s * b) with separately rounded products (no FMA), as fp32 bits: the stock RoPE arithmetic.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _rope_bits(c, a, s, b, *, loc=None, ip=None):
"""bf16(c * a + s * b) with separately rounded products (no FMA), as fp32 bits: the stock RoPE arithmetic."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
return cutlass.Int32(
_llvm.inline_asm(
_T.i32(), [cutlass.Float32(v).ir_value(loc=loc, ip=ip) for v in (c, a, s, b)],
"{\n\t.reg .f32 p, q, r;\n\t.reg .b16 h, z;\n\tmul.rn.f32 p, $1, $2;\n\tmul.rn.f32 q, $3, $4;\n\t"
"add.rn.f32 r, p, q;\n\tcvt.rn.bf16.f32 h, r;\n\tmov.b16 z, 0;\n\tmov.b32 $0, {z, h};\n\t}", "=r,f,f,f,f",
has_side_effects=False, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
|
_st_async_v4(dst, w0, w1, w2, w3, mbar, *, loc=None, ip=None)
st.async of four int32 words to a shared::cluster address, completing mbar there by 16 bytes.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _st_async_v4(dst, w0, w1, w2, w3, mbar, *, loc=None, ip=None):
"""st.async of four int32 words to a shared::cluster address, completing `mbar` there by 16 bytes."""
from cutlass._mlir.dialects import llvm as _llvm
_llvm.inline_asm(
None,
[cutlass.Int32(dst).ir_value(loc=loc, ip=ip), cutlass.Int32(w0).ir_value(loc=loc, ip=ip),
cutlass.Int32(w1).ir_value(loc=loc, ip=ip), cutlass.Int32(w2).ir_value(loc=loc, ip=ip),
cutlass.Int32(w3).ir_value(loc=loc, ip=ip), cutlass.Int32(mbar).ir_value(loc=loc, ip=ip)],
"st.async.shared::cluster.mbarrier::complete_tx::bytes.v4.b32 [$0], {$1, $2, $3, $4}, [$5];", "r,r,r,r,r,r",
has_side_effects=True, is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
) # fmt: skip
|
_test_wait_cluster(mbar, parity, *, loc=None, ip=None)
mbarrier.test_wait.parity.acquire.cluster on a barrier of this CTA completed by other CTAs' st.async.
Source code in vllm/models/minimax_m3/nvidia/specialized_decode/kernels/o_proj/kernel.py
| @dsl_user_op
def _test_wait_cluster(mbar, parity, *, loc=None, ip=None):
"""mbarrier.test_wait.parity.acquire.cluster on a barrier of this CTA completed by other CTAs' st.async."""
from cutlass._mlir.dialects import llvm as _llvm
from cutlass._mlir.extras import types as _T
done = cutlass.Int32(
_llvm.inline_asm(
_T.i32(), [mbar.toint(loc=loc, ip=ip).ir_value(), cutlass.Int32(parity).ir_value(loc=loc, ip=ip)],
"{\n\t.reg .pred p;\n\tmbarrier.test_wait.parity.acquire.cluster.shared::cta.b64 p, [$1], $2;\n\t"
"selp.u32 $0, 1, 0, p;\n\t}", "=r,r,r,~{memory}", has_side_effects=True,
is_align_stack=False, asm_dialect=_llvm.AsmDialect.AD_ATT, loc=loc, ip=ip,
)
) # fmt: skip
return done != cutlass.Int32(0)
|