Skip to content

vllm.model_executor.layers.fused_moe.flashinfer_moe_ep

Classes:

Functions:

FlashInferMoeEp

Methods:

  • expert_weight_views –

    Every per-expert tensor the kernel reads: weights plus epilogue.

  • kernel_weights –

    Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
class FlashInferMoeEp:
    def __init__(
        self,
        moe: FusedMoEConfig,
        weights: FlashInferMoeEpWeights,
        epilogue: FlashInferMoeEpEpilogue | None = None,
        *,
        apply_topk_in_fc1: bool,
        block_fp8: bool = False,
    ) -> None:
        spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
        if spec.kernel == "deep_gemm":
            _expose_deep_gemm_to_flashinfer()
        api = _load_flashinfer_moe_ep_api()
        if epilogue is None:
            epilogue = FlashInferMoeEpEpilogue()
        ep_group = get_ep_group()
        bootstrap = api.BootstrapConfig(
            world_size=ep_group.world_size,
            rank=ep_group.rank_in_group,
            process_group=ep_group.device_group,
            device=torch.accelerator.current_device_index(),
        )
        fleet_params = api.FleetParams(
            num_experts=moe.num_experts,
            max_tokens_per_rank=moe.max_num_tokens,
            token_hidden_size=moe.hidden_dim,
        )
        sm107 = is_sm107_device()
        if sm107 and moe.routing_method is RoutingMethodType.DeepseekV4:
            # The DeepSeek-V4 router can return non-finite scores on valid routes,
            # which FlashInfer 0.7.3's SM107 route check turns into a device assert.
            os.environ.setdefault(_SM107_ROUTE_CHECK_ENV, "0")
        if block_fp8:
            megakernel = api.Sm90PullFp8MegaMoeConfig(
                intermediate_size=moe.intermediate_size,
                top_k=moe.experts_per_token,
                fp8_scale_mode="blockwise",
                gate_up_clamp=moe.swiglu_limit,
            )
        elif spec.kernel == "cutedsl":
            config_cls = (
                api.Sm107Nvfp4MegaMoeConfig if sm107 else api.Nvfp4CutedslMegaMoeConfig
            )
            # In-kernel fc2 reduce keeps its default (off).
            megakernel = config_cls(
                intermediate_size=moe.intermediate_size,
                top_k=moe.experts_per_token,
                gate_up_clamp=moe.swiglu_limit,
                fast_math=True,
                apply_routing_weights_before_fc2=apply_topk_in_fc1,
                combine_dtype="bf16",
                input_norm_const=epilogue.input_norm_const,
                fc1_alpha=None,
                fc2_alpha=None,
                fc1_norm_const=None,
                knobs=None,
            )
        else:
            megakernel = api.DeepGemmMegaMoeConfig(
                intermediate_size=moe.intermediate_size,
                top_k=moe.experts_per_token,
                activation_clamp=moe.swiglu_limit,
                fast_math=True,
            )
        if block_fp8:
            # The oracle already put w13 in the kernel's row order
            # (sm90_fp8_block_moe_ep_weights); the kernel reads (E, K, N) views
            # with K contiguous.
            assert weights.w13_scale is not None and weights.w2_scale is not None
            weight_pack = None
            backend = api.MegaConfig(
                megakernel=megakernel,
                quantize_input=True,
                preprocess_weights=False,
                transformed_weights=(
                    (weights.w13.transpose(1, 2), weights.w13_scale, None, None),
                    (weights.w2.transpose(1, 2), weights.w2_scale, None, None),
                ),
            )
        else:
            weight_pack = api.MoEWeightPack(
                w13=weights.w13,
                w2=weights.w2,
                w13_scale=weights.w13_scale,
                w2_scale=weights.w2_scale,
            )
            backend = api.MegaConfig(
                megakernel=megakernel,
                quantize_input=True,
                preprocess_weights=True,
            )

        self._moe_ep_tensors_cls = api.MoEEpTensors
        self._mega_layer: Any | None = api.MoEEpMegaLayer(
            bootstrap,
            fleet_params,
            weight_pack,
            backend,
        )
        # Created collectively, in the same order on every EP rank.
        self._workspaces: list[tuple[int, Any]] = []
        if block_fp8:
            self._workspaces = [
                (capacity, self._mega_layer.create_workspace(capacity))
                for capacity in _SM90_WORKSPACE_CAPACITIES
                if capacity < moe.max_num_tokens
            ]
        self._device = weights.w13.device
        self._hidden_size = moe.hidden_dim
        self._max_num_tokens = moe.max_num_tokens
        self._num_experts = moe.num_experts
        self._top_k = moe.experts_per_token
        self._epilogue: FlashInferMoeEpEpilogue | None = epilogue
        # SM107 uses FlashInfer's owned output, which reduces only the live rows.
        self._return_workspace_view = (
            moe.routing_method is RoutingMethodType.DeepseekV4 and not sm107
        )
        self._block_fp8 = block_fp8
        logger.info_once("FlashInfer MoE-EP megakernel: %s", type(megakernel).__name__)

    @property
    def can_overlap_shared_experts(self) -> bool:
        return False

    @property
    def output_is_reduced(self) -> bool:
        return True

    @property
    def topk_indices_dtype(self) -> torch.dtype:
        return torch.int32

    @property
    def is_monolithic(self) -> bool:
        return False

    def _tensors(
        self,
        hidden_states: torch.Tensor,
        topk_ids: torch.Tensor,
        topk_weights: torch.Tensor,
    ) -> Any:
        epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
        return self._moe_ep_tensors_cls(
            hidden_states=hidden_states,
            topk_ids=topk_ids,
            topk_weights=topk_weights,
            fc1_alpha=epilogue.fc1_alpha,
            fc2_alpha=epilogue.fc2_alpha,
            fc1_norm_const=epilogue.fc1_norm_const,
        )

    def __call__(
        self,
        hidden_states: torch.Tensor,
        topk_ids: torch.Tensor,
        topk_weights: torch.Tensor,
    ) -> torch.Tensor:
        if self._mega_layer is None:
            raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
        if hidden_states.shape[0] > self._max_num_tokens:
            raise ValueError(
                f"FlashInfer MoE-EP got {hidden_states.shape[0]} tokens, but "
                f"the workspace supports at most {self._max_num_tokens}"
            )
        tensors = self._tensors(hidden_states, topk_ids, topk_weights)
        if self._return_workspace_view and getattr(
            self._mega_layer, "supports_output_view", False
        ):
            return self._mega_layer.forward(tensors, return_workspace_view=True)
        return self._mega_layer.forward(tensors, workspace=self._workspace())

    def _workspace(self) -> Any | None:
        """The smallest workspace that fits the batch on every EP rank.

        All ranks must use the same buffer, so without the DP-wide token
        counts this falls back to the default workspace.
        """
        if not self._workspaces or not is_forward_context_available():
            return None
        dp_metadata = get_forward_context().dp_metadata
        if dp_metadata is None:
            return None
        num_tokens = int(dp_metadata.num_tokens_across_dp_cpu.max())
        for capacity, workspace in self._workspaces:
            if num_tokens <= capacity:
                return workspace
        return None

    @torch.inference_mode()
    def warmup(self) -> None:
        if self._mega_layer is None:
            return
        hidden_states = torch.zeros(
            1,
            self._hidden_size,
            dtype=torch.bfloat16,
            device=self._device,
        )
        topk_ids = (
            torch.arange(self._top_k, dtype=torch.int32, device=self._device)
            .mul_(self._num_experts // self._top_k)
            .view(1, self._top_k)
        )
        topk_weights = torch.full(
            (1, self._top_k),
            1.0 / self._top_k,
            dtype=torch.float32,
            device=self._device,
        )
        tensors = self._tensors(hidden_states, topk_ids, topk_weights)
        self._mega_layer.warmup(tensors)
        for _, workspace in self._workspaces:
            self._mega_layer.warmup(tensors, workspace=workspace)

    def kernel_weights(
        self,
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
        """Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.

        Contiguous per-expert tensors that alias the memory the megakernel
        reads, so a caller may permute experts in place (EPLB). One-byte FP4
        data and scales come back as uint8; block-FP8 weights keep FP8.
        """
        if self._mega_layer is None:
            raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
        # The SM90 FP8 legs carry two (unused in blockwise mode) dequant scales.
        (fc1_weight, fc1_sf, *_), (fc2_weight, fc2_sf, *_) = (
            self._mega_layer._transformed
        )
        # Block-FP8 weights keep their dtype, so the layer holds the kernel's
        # FP8 runtime layout that preprocessed-weight loading rebuilds from.
        as_bytes = not self._block_fp8
        return (
            _expert_major_bytes(fc1_weight, as_bytes),
            _expert_major_bytes(fc1_sf, as_bytes),
            _expert_major_bytes(fc2_weight, as_bytes),
            _expert_major_bytes(fc2_sf, as_bytes),
        )

    def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
        """Every per-expert tensor the kernel reads: weights plus epilogue."""
        epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
        vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
        return (*self.kernel_weights(), *(v for v in vectors if v is not None))

    def destroy(self) -> None:
        mega_layer = self._mega_layer
        if mega_layer is None:
            return
        mega_layer.destroy()
        self._mega_layer = None
        self._workspaces = []
        self._epilogue = None

    def __del__(self) -> None:
        with contextlib.suppress(Exception):
            self.destroy()

_workspace()

The smallest workspace that fits the batch on every EP rank.

All ranks must use the same buffer, so without the DP-wide token counts this falls back to the default workspace.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
def _workspace(self) -> Any | None:
    """The smallest workspace that fits the batch on every EP rank.

    All ranks must use the same buffer, so without the DP-wide token
    counts this falls back to the default workspace.
    """
    if not self._workspaces or not is_forward_context_available():
        return None
    dp_metadata = get_forward_context().dp_metadata
    if dp_metadata is None:
        return None
    num_tokens = int(dp_metadata.num_tokens_across_dp_cpu.max())
    for capacity, workspace in self._workspaces:
        if num_tokens <= capacity:
            return workspace
    return None

expert_weight_views()

Every per-expert tensor the kernel reads: weights plus epilogue.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
def expert_weight_views(self) -> tuple[torch.Tensor, ...]:
    """Every per-expert tensor the kernel reads: weights plus epilogue."""
    epilogue = cast(FlashInferMoeEpEpilogue, self._epilogue)
    vectors = (epilogue.fc1_alpha, epilogue.fc2_alpha, epilogue.fc1_norm_const)
    return (*self.kernel_weights(), *(v for v in vectors if v is not None))

kernel_weights()

Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.

Contiguous per-expert tensors that alias the memory the megakernel reads, so a caller may permute experts in place (EPLB). One-byte FP4 data and scales come back as uint8; block-FP8 weights keep FP8.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
def kernel_weights(
    self,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
    """Kernel-resident fc1 weight, fc1 scales, fc2 weight, fc2 scales.

    Contiguous per-expert tensors that alias the memory the megakernel
    reads, so a caller may permute experts in place (EPLB). One-byte FP4
    data and scales come back as uint8; block-FP8 weights keep FP8.
    """
    if self._mega_layer is None:
        raise RuntimeError("FlashInfer MoE-EP layer was destroyed")
    # The SM90 FP8 legs carry two (unused in blockwise mode) dequant scales.
    (fc1_weight, fc1_sf, *_), (fc2_weight, fc2_sf, *_) = (
        self._mega_layer._transformed
    )
    # Block-FP8 weights keep their dtype, so the layer holds the kernel's
    # FP8 runtime layout that preprocessed-weight loading rebuilds from.
    as_bytes = not self._block_fp8
    return (
        _expert_major_bytes(fc1_weight, as_bytes),
        _expert_major_bytes(fc1_sf, as_bytes),
        _expert_major_bytes(fc2_weight, as_bytes),
        _expert_major_bytes(fc2_sf, as_bytes),
    )

_expert_major_bytes(tensor, as_bytes=True)

Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8 unless as_bytes is False.

The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
def _expert_major_bytes(tensor: torch.Tensor, as_bytes: bool = True) -> torch.Tensor:
    """Contiguous storage of a per-expert kernel tensor, 1-byte dtypes as uint8
    unless `as_bytes` is False.

    The CuTeDSL kernel keeps (E, K, N) transpose views over (E, N, K) storage.
    """
    if not tensor.is_contiguous():
        tensor = tensor.transpose(1, 2)
    if not tensor.is_contiguous():
        raise ValueError(f"expert tensor is not expert-major: {tensor.stride()}")
    if as_bytes and tensor.element_size() == 1:
        return tensor.view(torch.uint8)
    return tensor

flashinfer_moe_ep_unsupported_reasons(moe, weight_key, activation_key)

Deployment constraints of the megakernel beyond the framework's checks.

activation_key is None for MXFP4 checkpoints (the kernel quantizes bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
def flashinfer_moe_ep_unsupported_reasons(
    moe: FusedMoEConfig,
    weight_key: QuantKey | None,
    activation_key: QuantKey | None,
) -> tuple[str, ...]:
    """Deployment constraints of the megakernel beyond the framework's checks.

    `activation_key` is `None` for MXFP4 checkpoints (the kernel quantizes
    bf16 activations itself) but marks a W4A16 recipe for NVFP4 weights.
    """
    spec = flashinfer_moe_ep_backend_spec(moe.moe_backend)
    unsupported: list[str] = []
    weight_format = _WEIGHT_FORMATS.get(weight_key)
    if weight_format is None:
        unsupported.append(f"weight scheme {weight_key}")
    elif weight_format not in spec.weight_formats:
        unsupported.append(f"{weight_format.upper()} weights")
    elif _current_capability() not in _FORMAT_CAPABILITIES[weight_format]:
        unsupported.append(f"{weight_format.upper()} weights on this GPU")
    if weight_key == kNvfp4Static and activation_key is None:
        unsupported.append("A16 activations with NVFP4 weights")
    if weight_key == kFp8Static128BlockSym and activation_key != kFp8Dynamic128Sym:
        unsupported.append(f"activation scheme {activation_key} with block FP8")
    if moe.skip_final_all_reduce:
        unsupported.append("skip_final_all_reduce")
    if moe.in_dtype != torch.bfloat16:
        unsupported.append(f"activation dtype {moe.in_dtype}")
    if moe.has_bias:
        unsupported.append("expert bias")
    if moe.swiglu_alpha is not None or moe.swiglu_beta is not None:
        unsupported.append("custom SwiGLU alpha or beta")
    if (
        spec.kernel == "deep_gemm"
        and moe.routing_method is not RoutingMethodType.DeepseekV4
    ):
        unsupported.append(f"routing method {moe.routing_method.name}")

    vllm_config = get_current_vllm_config()
    if vllm_config.weight_transfer_config is not None:
        unsupported.append("runtime weight transfer")
    if vllm_config.parallel_config.enable_dbo:
        unsupported.append("dual batch overlap")
    if spec.kernel == "deep_gemm" and vllm_config.parallel_config.enable_eplb:
        unsupported.append("EPLB")
    elif is_sm107_device() and vllm_config.parallel_config.enable_eplb:
        unsupported.append("EPLB on SM107")
    return tuple(unsupported)

sm90_fp8_block_moe_ep_weights(w13, w13_scale)

Reorder block-FP8 w13 (gate rows, then up rows) for the SM90 kernel.

After the gate/up interleave a 128-row scale block holds 64 gate and 64 up rows from two checkpoint blocks. Both take the larger of the two scales and the other half is rescaled. Power-of-two scales make that an exponent shift, so only values that drop out of the E4M3 normal range round.

Parameters:

  • w13

    (Tensor) –

    (E, 2 * I, H) E4M3 weights.

  • w13_scale

    (Tensor) –

    (E, 2 * I / 128, H / 128) float32 dequantization scales.

Returns:

  • Tensor –

    The (E, 2 * I, H) weights in kernel row order and their

  • Tensor –

    (E, 2 * I / 128, H / 128) scales.

Source code in vllm/model_executor/layers/fused_moe/flashinfer_moe_ep.py
def sm90_fp8_block_moe_ep_weights(
    w13: torch.Tensor,
    w13_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Reorder block-FP8 `w13` (gate rows, then up rows) for the SM90 kernel.

    After the gate/up interleave a 128-row scale block holds 64 gate and 64 up
    rows from two checkpoint blocks. Both take the larger of the two scales and
    the other half is rescaled. Power-of-two scales make that an exponent
    shift, so only values that drop out of the E4M3 normal range round.

    Args:
        w13: `(E, 2 * I, H)` E4M3 weights.
        w13_scale: `(E, 2 * I / 128, H / 128)` float32 dequantization scales.

    Returns:
        The `(E, 2 * I, H)` weights in kernel row order and their
        `(E, 2 * I / 128, H / 128)` scales.

    """
    num_experts, two_i, hidden = w13.shape
    intermediate = two_i // 2
    num_blocks = intermediate // _FP8_BLOCK
    assert intermediate % _FP8_BLOCK == 0 and hidden % _FP8_BLOCK == 0, w13.shape
    assert w13_scale.shape == (num_experts, 2 * num_blocks, hidden // _FP8_BLOCK)
    gate_scale, up_scale = w13_scale.split(num_blocks, dim=1)
    shared_scale = torch.maximum(gate_scale, up_scale)
    out = torch.empty_like(w13)
    # One expert at a time bounds the float32 temporaries.
    for expert in range(num_experts):
        scale = torch.stack((gate_scale[expert], up_scale[expert]))
        ratio = (scale / shared_scale[expert]).repeat_interleave(_FP8_BLOCK, 1)
        ratio = ratio.repeat_interleave(_FP8_BLOCK, 2)
        halves = w13[expert].view(2, intermediate, hidden).float() * ratio
        out[expert] = (
            halves.to(w13.dtype)
            .view(2, intermediate // _SM90_GATE_UP_INTERLEAVE, -1, hidden)
            .transpose(0, 1)
            .reshape(two_i, hidden)
        )
    return out, shared_scale.repeat_interleave(2, dim=1).contiguous()