Skip to content

vllm.model_executor.layers.quantization.quark.quark_moe

Classes:

QuarkMoEMethod

Bases: FusedMoEMethodBase

Methods:

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
class QuarkMoEMethod(FusedMoEMethodBase):
    supported_activation_quant_keys: list[QuantKey | None] = []
    supported_weight_quant_keys: list[QuantKey] = []

    def __init__(
        self,
        moe: FusedMoEConfig,
        weight_quant_key: QuantKey,
        activation_quant_key: QuantKey | None,
    ):
        super().__init__(moe)
        if activation_quant_key not in self.supported_activation_quant_keys:
            raise ValueError(
                f"Unsupported activation quant key: {activation_quant_key}"
            )
        if weight_quant_key not in self.supported_weight_quant_keys:
            raise ValueError(f"Unsupported weight quant key: {weight_quant_key}")
        self.activation_quant_key = activation_quant_key
        self.weight_quant_key = weight_quant_key
        self.has_bias = self.moe.has_bias

    @staticmethod
    def get_moe_method_target(
        quant_config: "QuarkConfig",
        layer_type: type[torch.nn.Module],
        layer_name: str,
    ) -> tuple[
        QuantKey | None,
        QuantKey | None,
        type["QuarkMoEMethod"] | None,
    ]:
        """Return the quantization target for a routed-experts layer."""
        layer_quant_config = quant_config._find_matched_config(layer_name, layer_type)
        if layer_quant_config.get("output_tensors") or layer_quant_config.get("bias"):
            raise NotImplementedError(
                "Currently, Quark models with "
                "output_tensors and bias "
                "quantized are not supported"
            )
        weight_config: QuarkQTensorHint = layer_quant_config.get("weight")
        input_config: QuarkQTensorHint = layer_quant_config.get("input_tensors")

        if (
            isinstance(weight_config, list)
            and len(weight_config) > 1
            or isinstance(input_config, list)
            and len(input_config) > 1
        ):
            if match := quant_config._is_fp8_w4a8(weight_config, input_config):
                return (
                    match.weight_quant_key,
                    match.activation_quant_key,
                    QuarkW4A8Fp8MoEMethod,
                )
            if match := quant_config._is_nvfp4(weight_config, input_config):
                return (
                    match.weight_quant_key,
                    match.activation_quant_key,
                    QuarkNvfp4MoEMethod,
                )
            raise RuntimeError("Unsupported FusedMoe scheme")

        weight_config = quant_config._unwrap_single_quant_config(weight_config)
        input_config = quant_config._unwrap_single_quant_config(input_config)

        if match := quant_config._is_fp8_w8a8(weight_config, input_config):
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkW8A8Fp8MoEMethod,
            )
        if match := quant_config._is_w4a16_int4(weight_config, input_config):
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkW4A16Int4MoEMethod,
            )
        if match := quant_config._is_w_ocp_mx_a_x(
            weight_config, input_config, allow_static_fp8=True
        ):
            if match.weight_quant_key == kMxfp8Static:
                # MXFP8 experts run through the FP8 MoE kernels (AITER_MXFP8 on
                # gfx950), not the MXFP4/MXFP6 OCP MX path.
                normalized = ocp_mx_weight_dtype_and_rows(weight_config)
                if normalized is None or normalized[1] != 1:
                    raise NotImplementedError(
                        "Quark MXFP8 MoE experts support only the 1-D per_group "
                        "(group_size=32, e8m0) weight spelling, got "
                        f"{weight_config} for {layer_name}"
                    )
                return (
                    match.weight_quant_key,
                    match.activation_quant_key,
                    QuarkW8A8Mxfp8MoEMethod,
                )
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkOCP_MX_MoEMethod,
            )
        if match := quant_config._is_w8a8_int8(weight_config, input_config):
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkW8A8Int8MoEMethod,
            )
        raise RuntimeError("Unsupported FusedMoe scheme")

    @staticmethod
    def get_moe_method(
        quant_config: "QuarkConfig",  # type: ignore # noqa E501 # noqa F821
        module: RoutedExperts,
        method_cls: type["QuarkMoEMethod"],
        weight_quant_key: QuantKey | None,
        activation_quant_key: QuantKey | None,
        layer_name: str | None = None,
    ) -> "QuarkMoEMethod":
        if weight_quant_key is None:
            raise RuntimeError("Unsupported FusedMoe scheme")

        if method_cls is QuarkW4A8Fp8MoEMethod:
            return QuarkW4A8Fp8MoEMethod(module.moe_config, activation_quant_key)
        elif method_cls is QuarkNvfp4MoEMethod:
            return QuarkNvfp4MoEMethod(
                module.moe_config, quant_config, activation_quant_key
            )
        elif method_cls is QuarkW8A8Fp8MoEMethod:
            return QuarkW8A8Fp8MoEMethod(
                module.moe_config, weight_quant_key, activation_quant_key
            )
        elif method_cls is QuarkOCP_MX_MoEMethod:
            # All OCP MX schemes (W4A16, W4A8, etc.) handled by QuarkOCP_MX_MoEMethod
            # Backend selection happens inside via oracle
            return QuarkOCP_MX_MoEMethod(
                module.moe_config, weight_quant_key, activation_quant_key
            )
        elif method_cls is QuarkW8A8Mxfp8MoEMethod:
            return QuarkW8A8Mxfp8MoEMethod(
                module.moe_config, weight_quant_key, activation_quant_key
            )
        elif method_cls is QuarkW8A8Int8MoEMethod:
            return QuarkW8A8Int8MoEMethod(
                module.moe_config, weight_quant_key, activation_quant_key
            )
        elif method_cls is QuarkW4A16Int4MoEMethod:
            assert layer_name is not None
            layer_quant_config = quant_config._find_matched_config(
                layer_name, type(module)
            )
            weight_config = quant_config._unwrap_single_quant_config(
                layer_quant_config.get("weight")
            )
            if weight_config is None:
                raise RuntimeError(
                    f"Missing weight quantization config for layer {layer_name}"
                )
            return QuarkW4A16Int4MoEMethod(
                weight_quant_key,
                activation_quant_key,
                weight_config,
                quant_config.pack_method,
                module.moe_config,
                quant_config=quant_config,
            )
        else:
            raise RuntimeError("Unsupported FusedMoe scheme")

get_moe_method_target(quant_config, layer_type, layer_name) staticmethod

Return the quantization target for a routed-experts layer.

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
@staticmethod
def get_moe_method_target(
    quant_config: "QuarkConfig",
    layer_type: type[torch.nn.Module],
    layer_name: str,
) -> tuple[
    QuantKey | None,
    QuantKey | None,
    type["QuarkMoEMethod"] | None,
]:
    """Return the quantization target for a routed-experts layer."""
    layer_quant_config = quant_config._find_matched_config(layer_name, layer_type)
    if layer_quant_config.get("output_tensors") or layer_quant_config.get("bias"):
        raise NotImplementedError(
            "Currently, Quark models with "
            "output_tensors and bias "
            "quantized are not supported"
        )
    weight_config: QuarkQTensorHint = layer_quant_config.get("weight")
    input_config: QuarkQTensorHint = layer_quant_config.get("input_tensors")

    if (
        isinstance(weight_config, list)
        and len(weight_config) > 1
        or isinstance(input_config, list)
        and len(input_config) > 1
    ):
        if match := quant_config._is_fp8_w4a8(weight_config, input_config):
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkW4A8Fp8MoEMethod,
            )
        if match := quant_config._is_nvfp4(weight_config, input_config):
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkNvfp4MoEMethod,
            )
        raise RuntimeError("Unsupported FusedMoe scheme")

    weight_config = quant_config._unwrap_single_quant_config(weight_config)
    input_config = quant_config._unwrap_single_quant_config(input_config)

    if match := quant_config._is_fp8_w8a8(weight_config, input_config):
        return (
            match.weight_quant_key,
            match.activation_quant_key,
            QuarkW8A8Fp8MoEMethod,
        )
    if match := quant_config._is_w4a16_int4(weight_config, input_config):
        return (
            match.weight_quant_key,
            match.activation_quant_key,
            QuarkW4A16Int4MoEMethod,
        )
    if match := quant_config._is_w_ocp_mx_a_x(
        weight_config, input_config, allow_static_fp8=True
    ):
        if match.weight_quant_key == kMxfp8Static:
            # MXFP8 experts run through the FP8 MoE kernels (AITER_MXFP8 on
            # gfx950), not the MXFP4/MXFP6 OCP MX path.
            normalized = ocp_mx_weight_dtype_and_rows(weight_config)
            if normalized is None or normalized[1] != 1:
                raise NotImplementedError(
                    "Quark MXFP8 MoE experts support only the 1-D per_group "
                    "(group_size=32, e8m0) weight spelling, got "
                    f"{weight_config} for {layer_name}"
                )
            return (
                match.weight_quant_key,
                match.activation_quant_key,
                QuarkW8A8Mxfp8MoEMethod,
            )
        return (
            match.weight_quant_key,
            match.activation_quant_key,
            QuarkOCP_MX_MoEMethod,
        )
    if match := quant_config._is_w8a8_int8(weight_config, input_config):
        return (
            match.weight_quant_key,
            match.activation_quant_key,
            QuarkW8A8Int8MoEMethod,
        )
    raise RuntimeError("Unsupported FusedMoe scheme")

QuarkNvfp4MoEMethod

Bases: QuarkMoEMethod

Methods:

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
class QuarkNvfp4MoEMethod(QuarkMoEMethod):
    supported_activation_quant_keys = [kNvfp4Dynamic]
    supported_weight_quant_keys = [kNvfp4Static]

    def __init__(
        self,
        moe: FusedMoEConfig,
        quant_config: "QuarkConfig",  # type: ignore # noqa E501 # noqa F821
        activation_quant_key: QuantKey | None,
    ):
        super().__init__(moe, kNvfp4Static, activation_quant_key)
        self.quant_config = quant_config
        self.group_size = 16
        # Whether the selected backend reads the activation input scales.
        self.uses_input_scales = True

        # Select experts implementation.
        self.nvfp4_backend, self.experts_cls = select_nvfp4_moe_backend(
            config=self.moe,
            weight_key=self.weight_quant_key,
            activation_key=self.activation_quant_key,
        )

    def create_weights(
        self,
        layer: torch.nn.Module,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        layer.num_experts = num_experts
        layer.params_dtype = params_dtype
        layer.quant_config = self.quant_config
        weight_dtype = torch.uint8
        weight_scale_dtype = torch.float8_e4m3fn

        # GEMM 1 - w13 weight
        w13_weight = torch.nn.Parameter(
            torch.empty(
                num_experts,
                self.moe.w13_num_shards * intermediate_size_per_partition,
                # 2 fp4 items are packed in the input dimension
                hidden_size // 2,
                dtype=weight_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight", w13_weight)
        set_weight_attrs(w13_weight, extra_weight_attrs)

        # GEMM 2 - w2 weight
        w2_weight = torch.nn.Parameter(
            torch.empty(
                num_experts,
                hidden_size,
                # 2 fp4 items are packed in the input dimension
                intermediate_size_per_partition // 2,
                dtype=weight_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight", w2_weight)
        set_weight_attrs(w2_weight, extra_weight_attrs)

        # Weight scales (per-group FP8 scales)
        w13_weight_scale = torch.nn.Parameter(
            torch.empty(
                num_experts,
                self.moe.w13_num_shards * intermediate_size_per_partition,
                hidden_size // self.group_size,
                dtype=weight_scale_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight_scale", w13_weight_scale)
        extra_weight_attrs.update(
            {"quant_method": FusedMoeWeightScaleSupported.GROUP.value}
        )
        set_weight_attrs(w13_weight_scale, extra_weight_attrs)

        w2_weight_scale = torch.nn.Parameter(
            torch.empty(
                num_experts,
                hidden_size,
                intermediate_size_per_partition // self.group_size,
                dtype=weight_scale_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight_scale", w2_weight_scale)
        set_weight_attrs(w2_weight_scale, extra_weight_attrs)

        # Global weight scales (per-tensor FP32 scales)
        extra_weight_attrs.update(
            {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}
        )

        w13_weight_scale_2 = torch.nn.Parameter(
            torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight_scale_2", w13_weight_scale_2)
        set_weight_attrs(w13_weight_scale_2, extra_weight_attrs)

        w2_weight_scale_2 = torch.nn.Parameter(
            torch.empty(num_experts, dtype=torch.float32),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight_scale_2", w2_weight_scale_2)
        set_weight_attrs(w2_weight_scale_2, extra_weight_attrs)

        # Input global scales (per-tensor FP32 scales)
        w13_input_scale_2 = torch.nn.Parameter(
            torch.empty(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
            requires_grad=False,
        )
        layer.register_parameter("w13_input_scale_2", w13_input_scale_2)
        set_weight_attrs(w13_input_scale_2, extra_weight_attrs)

        w2_input_scale_2 = torch.nn.Parameter(
            torch.empty(num_experts, dtype=torch.float32),
            requires_grad=False,
        )
        layer.register_parameter("w2_input_scale_2", w2_input_scale_2)
        set_weight_attrs(w2_input_scale_2, extra_weight_attrs)

    def dequantize_weight(
        self, layer: torch.nn.Module
    ) -> Iterator[tuple[slice, torch.Tensor, torch.Tensor]]:
        """Yield `(experts, w13, w2)` dequantized Quark NVFP4 experts."""
        return dequantize_nvfp4_moe_weights(
            layer.w13_weight,
            layer.w13_weight_scale,
            layer.w13_weight_scale_2,
            layer.w2_weight,
            layer.w2_weight_scale,
            layer.w2_weight_scale_2,
            layer.params_dtype,
            group_size=self.group_size,
            invert_global_scales=False,
        )

    def process_weights_after_loading(self, layer: RoutedExperts) -> None:
        """Convert NVFP4 MoE weights into kernel format and setup the kernel."""
        # Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
        if self.moe.is_act_and_mul and not torch.allclose(
            layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
        ):
            logger.warning_once(
                "w1_weight_scale_2 must match w3_weight_scale_2. "
                "Accuracy may be affected."
            )

        w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()

        w2_weight_scale_2 = layer.w2_weight_scale_2

        (
            w13,
            w13_scale,
            w13_scale_2,
            a13_scale,
            w2,
            w2_scale,
            w2_scale_2,
            a2_scale,
        ) = convert_to_nvfp4_moe_kernel_format(
            nvfp4_backend=self.nvfp4_backend,
            layer=layer,
            w13=layer.w13_weight,
            w13_scale=layer.w13_weight_scale,
            w13_scale_2=w13_weight_scale_2,
            a13_scale=layer.w13_input_scale_2,
            w2=layer.w2_weight,
            w2_scale=layer.w2_weight_scale,
            w2_scale_2=w2_weight_scale_2,
            a2_scale=layer.w2_input_scale_2,
            is_act_and_mul=self.moe.is_act_and_mul,
        )

        replace_parameter(layer, "w13_weight", w13)
        replace_parameter(layer, "w13_weight_scale", w13_scale)
        replace_parameter(layer, "w13_weight_scale_2", w13_scale_2)

        replace_parameter(layer, "w2_weight", w2)
        replace_parameter(layer, "w2_weight_scale", w2_scale)
        replace_parameter(layer, "w2_weight_scale_2", w2_scale_2)
        # Weight-only backends (Marlin, Humming) return no input scales: the
        # loaded ones keep their data, retired, and the kernel config gets None.
        self.uses_input_scales = a13_scale is not None
        if self.uses_input_scales:
            replace_parameter(layer, "w13_input_scale_2", a13_scale)
            replace_parameter(layer, "w2_input_scale_2", a2_scale)
        else:
            retire_parameter(layer, "w13_input_scale_2")
            retire_parameter(layer, "w2_input_scale_2")

        # Setup modular kernel.
        self.moe_quant_config = self.get_fused_moe_quant_config(layer)
        if self.moe_quant_config:
            assert self.experts_cls is not None
            self.moe_kernel = make_nvfp4_moe_kernel(
                moe_quant_config=self.moe_quant_config,
                moe_config=self.moe,
                experts_cls=self.experts_cls,
                backend=self.nvfp4_backend,
                routing_tables=layer._expert_routing_tables(),
            )
            self.moe_kernel.fused_experts.keep_constants(layer)

    def get_fused_moe_quant_config(
        self, layer: torch.nn.Module
    ) -> FusedMoEQuantConfig | None:
        return make_nvfp4_moe_quant_config(
            backend=self.nvfp4_backend,
            w13_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            w13_scale_2=layer.w13_weight_scale_2,
            w2_scale_2=layer.w2_weight_scale_2,
            a13_scale=layer.w13_input_scale_2 if self.uses_input_scales else None,
            a2_scale=layer.w2_input_scale_2 if self.uses_input_scales else None,
            swiglu_limit=getattr(layer, "swiglu_limit", None),
            swiglu_alpha=getattr(layer, "swiglu_alpha", None),
            swiglu_beta=getattr(layer, "swiglu_beta", None),
            layer=layer,
        )

    def apply(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        shared_experts: SharedExperts | None,
        shared_experts_input: torch.Tensor | None,
    ) -> torch.Tensor | UnfinalizedMoEOutput:
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
            x,
            layer.w13_weight,
            layer.w2_weight,
            topk_weights,
            topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            shared_experts=shared_experts,
            shared_experts_input=shared_experts_input,
        )

dequantize_weight(layer)

Yield (experts, w13, w2) dequantized Quark NVFP4 experts.

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
def dequantize_weight(
    self, layer: torch.nn.Module
) -> Iterator[tuple[slice, torch.Tensor, torch.Tensor]]:
    """Yield `(experts, w13, w2)` dequantized Quark NVFP4 experts."""
    return dequantize_nvfp4_moe_weights(
        layer.w13_weight,
        layer.w13_weight_scale,
        layer.w13_weight_scale_2,
        layer.w2_weight,
        layer.w2_weight_scale,
        layer.w2_weight_scale_2,
        layer.params_dtype,
        group_size=self.group_size,
        invert_global_scales=False,
    )

process_weights_after_loading(layer)

Convert NVFP4 MoE weights into kernel format and setup the kernel.

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
def process_weights_after_loading(self, layer: RoutedExperts) -> None:
    """Convert NVFP4 MoE weights into kernel format and setup the kernel."""
    # Match existing NVFP4 MoE paths: fused w13 uses the w1 global scale.
    if self.moe.is_act_and_mul and not torch.allclose(
        layer.w13_weight_scale_2[:, 0], layer.w13_weight_scale_2[:, 1]
    ):
        logger.warning_once(
            "w1_weight_scale_2 must match w3_weight_scale_2. "
            "Accuracy may be affected."
        )

    w13_weight_scale_2 = layer.w13_weight_scale_2[:, 0].contiguous()

    w2_weight_scale_2 = layer.w2_weight_scale_2

    (
        w13,
        w13_scale,
        w13_scale_2,
        a13_scale,
        w2,
        w2_scale,
        w2_scale_2,
        a2_scale,
    ) = convert_to_nvfp4_moe_kernel_format(
        nvfp4_backend=self.nvfp4_backend,
        layer=layer,
        w13=layer.w13_weight,
        w13_scale=layer.w13_weight_scale,
        w13_scale_2=w13_weight_scale_2,
        a13_scale=layer.w13_input_scale_2,
        w2=layer.w2_weight,
        w2_scale=layer.w2_weight_scale,
        w2_scale_2=w2_weight_scale_2,
        a2_scale=layer.w2_input_scale_2,
        is_act_and_mul=self.moe.is_act_and_mul,
    )

    replace_parameter(layer, "w13_weight", w13)
    replace_parameter(layer, "w13_weight_scale", w13_scale)
    replace_parameter(layer, "w13_weight_scale_2", w13_scale_2)

    replace_parameter(layer, "w2_weight", w2)
    replace_parameter(layer, "w2_weight_scale", w2_scale)
    replace_parameter(layer, "w2_weight_scale_2", w2_scale_2)
    # Weight-only backends (Marlin, Humming) return no input scales: the
    # loaded ones keep their data, retired, and the kernel config gets None.
    self.uses_input_scales = a13_scale is not None
    if self.uses_input_scales:
        replace_parameter(layer, "w13_input_scale_2", a13_scale)
        replace_parameter(layer, "w2_input_scale_2", a2_scale)
    else:
        retire_parameter(layer, "w13_input_scale_2")
        retire_parameter(layer, "w2_input_scale_2")

    # Setup modular kernel.
    self.moe_quant_config = self.get_fused_moe_quant_config(layer)
    if self.moe_quant_config:
        assert self.experts_cls is not None
        self.moe_kernel = make_nvfp4_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            experts_cls=self.experts_cls,
            backend=self.nvfp4_backend,
            routing_tables=layer._expert_routing_tables(),
        )
        self.moe_kernel.fused_experts.keep_constants(layer)

QuarkOCP_MX_MoEMethod

Bases: QuarkMoEMethod

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
class QuarkOCP_MX_MoEMethod(QuarkMoEMethod):
    # MXFP8 experts are handled by QuarkW8A8Mxfp8MoEMethod; this path only
    # implements the packed MXFP4 / MXFP6 elements.
    supported_activation_quant_keys = [
        kMxfp4Dynamic,
        kMxfp6E3M2Dynamic,
        kMxfp6E2M3Dynamic,
        kFp8DynamicTensorSym,
        kFp8StaticTensorSym,
        None,
    ]
    supported_weight_quant_keys = [
        kMxfp4Static,
        kMxfp6E3M2Static,
        kMxfp6E2M3Static,
    ]

    def __init__(
        self,
        moe: FusedMoEConfig,
        weight_quant_key: QuantKey,
        activation_quant_key: QuantKey | None,
    ):
        super().__init__(moe, weight_quant_key, activation_quant_key)
        self.weight_dtype = next(
            dtype
            for dtype, quant_key in _WEIGHT_QUANT_KEY_MAP.items()
            if quant_key == weight_quant_key
        )
        if activation_quant_key in {kFp8DynamicTensorSym, kFp8StaticTensorSym}:
            self.input_dtype: str | None = "fp8"
        elif activation_quant_key is None:
            self.input_dtype = None
        else:
            self.input_dtype = next(
                dtype
                for dtype, quant_key in _ACTIVATION_QUANT_KEY_MAP.items()
                if quant_key == activation_quant_key
            )

        self.ocp_mx_scheme = OCP_MX_Scheme.from_quant_dtype(
            self.input_dtype, self.weight_dtype
        )

        if self.ocp_mx_scheme is None:
            raise ValueError(
                f"Unsupported OCP MX dtype combination for MoE: "
                f"input_dtype={self.input_dtype}, weight_dtype={self.weight_dtype}. "
                f"Supported MoE weight dtypes are mxfp4, mxfp6_e3m2, and mxfp6_e2m3."
            )

        # TODO(bowenbao): refactor and introduce backends for other OCP MX schemes,
        # use kernel abstraction for all OCP MX MOE implementations.
        self.mxfp4_backend: Mxfp4MoeBackend = Mxfp4MoeBackend.NONE
        self.experts_cls: type[mk.FusedMoEExperts] | None = None
        self.moe_kernel: mk.FusedMoEKernel | None = None

        # Used for triton kernel precision configs (W4A8, TRITON backends)
        self.w13_precision_config = None
        self.w2_precision_config = None

        self.static_input_scales = activation_quant_key == kFp8StaticTensorSym

        # Select backend based on OCP MX scheme
        if self.ocp_mx_scheme == "w_mxfp4":
            # W4A16: weight-only MXFP4
            self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(moe)
        elif self.ocp_mx_scheme == "w_mxfp4_a_fp8" and self.static_input_scales:
            # W4A8: MXFP4 weights + static FP8 activations
            self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(
                moe, activation_key=kFp8StaticTensorSym
            )
        elif self.ocp_mx_scheme == "w_mxfp4_a_mxfp4":
            # W4A4: MXFP4 weights + MXFP4 activations
            self.mxfp4_backend, self.experts_cls = select_mxfp4_moe_backend(
                moe, activation_key=kMxfp4Dynamic
            )

        # Validation for unsupported schemes
        if any(
            self.ocp_mx_scheme.endswith(a_scheme)
            for a_scheme in ["a_mxfp4", "a_mxfp6_e3m2", "a_mxfp6_e2m3"]
        ):
            if self.static_input_scales:
                raise NotImplementedError(
                    "QuarkOCP_MX_MoEMethod with static input scales is currently "
                    f"not implemented for OCP MX scheme {self.ocp_mx_scheme}. "
                    "Please open an issue."
                )
        elif self.ocp_mx_scheme.endswith("a_fp8") and not self.static_input_scales:
            raise NotImplementedError(
                "QuarkOCP_MX_MoEMethod with dynamic input scales is currently "
                f"not implemented for OCP MX scheme {self.ocp_mx_scheme}. "
                "Please open an issue."
            )

        self.model_type = getattr(
            get_current_vllm_config().model_config.hf_config, "model_type", None
        )

        # If no native backend available, use emulation.
        if self.mxfp4_backend is Mxfp4MoeBackend.NONE:
            self.mxfp4_backend = Mxfp4MoeBackend.EMULATION
            self.experts_cls = None

        if self.experts_cls is None:
            self.experts_cls = backend_to_kernel_cls(self.mxfp4_backend)[0]

        logger.info_once(
            f"Using {self.mxfp4_backend.value} backend for {self.ocp_mx_scheme}"
        )

    def maybe_roundup_sizes(
        self,
        hidden_size: int,
        intermediate_size_per_partition: int,
        act_dtype: torch.dtype,
        moe_parallel_config: FusedMoEParallelConfig,
    ) -> tuple[int, int]:
        hidden_size, intermediate_size_per_partition = super().maybe_roundup_sizes(
            hidden_size=hidden_size,
            intermediate_size_per_partition=intermediate_size_per_partition,
            act_dtype=act_dtype,
            moe_parallel_config=moe_parallel_config,
        )
        # Round per-partition sizes up to each backend's requirement. Emulation is
        # handled inside the helper too (OCP MX block alignment), so no special-case.
        if self.mxfp4_backend is not None:
            hidden_size, intermediate_size_per_partition = (
                mxfp4_round_up_hidden_size_and_intermediate_size(
                    self.mxfp4_backend, hidden_size, intermediate_size_per_partition
                )
            )
        return hidden_size, intermediate_size_per_partition

    def get_packed_dim(self, dim: int, quant_dtype: str):
        if quant_dtype == "mxfp4":
            assert dim % 2 == 0
            return dim // 2
        elif quant_dtype in {"mxfp6_e3m2", "mxfp6_e2m3"}:
            # FP6 packs 4 * 6 = 24 bits on 3 bytes.
            assert (dim * 3) % 4 == 0
            return (dim * 3) // 4
        else:
            raise NotImplementedError(
                f"Unsupported quant_dtype in QuarkOCP_MX_MoEMethod."
                f"get_packed_dim: {quant_dtype}. MXFP8 is not supported for "
                f"MoE experts."
            )

    def create_weights(
        self,
        layer: RoutedExperts,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        # Add the quantization method used (per tensor/grouped/channel)
        # to ensure the weight scales are loaded in properly
        extra_weight_attrs.update(
            {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}
        )

        params_dtype = torch.uint8

        # WEIGHTS
        w13_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                self.moe.w13_num_shards * intermediate_size_per_partition,
                self.get_packed_dim(hidden_size, self.weight_dtype),
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight", w13_weight)

        set_weight_attrs(w13_weight, extra_weight_attrs)

        w2_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
                self.get_packed_dim(intermediate_size_per_partition, self.weight_dtype),
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight", w2_weight)

        set_weight_attrs(w2_weight, extra_weight_attrs)

        # WEIGHT_SCALES
        w13_weight_scale = torch.nn.Parameter(
            torch.ones(
                num_experts,
                self.moe.w13_num_shards * intermediate_size_per_partition,
                hidden_size // OCP_MX_BLOCK_SIZE,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        w2_weight_scale = torch.nn.Parameter(
            torch.ones(
                num_experts,
                hidden_size,
                intermediate_size_per_partition // OCP_MX_BLOCK_SIZE,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        set_weight_attrs(w2_weight_scale, extra_weight_attrs)
        set_weight_attrs(w13_weight_scale, extra_weight_attrs)

        layer.register_parameter("w13_weight_scale", w13_weight_scale)
        layer.register_parameter("w2_weight_scale", w2_weight_scale)

        if self.has_bias:
            w13_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
                    self.moe.w13_num_shards * intermediate_size_per_partition,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w13_bias", w13_bias)
            set_weight_attrs(w13_bias, extra_weight_attrs)

            w2_bias = torch.nn.Parameter(
                torch.zeros(num_experts, hidden_size, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_bias", w2_bias)
            set_weight_attrs(w2_bias, extra_weight_attrs)
        else:
            layer.w13_bias, layer.w2_bias = None, None

        # INPUT_SCALES
        if self.static_input_scales:
            w13_input_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32), requires_grad=False
            )
            layer.register_parameter("w13_input_scale", w13_input_scale)
            set_weight_attrs(w13_input_scale, extra_weight_attrs)

            w2_input_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32), requires_grad=False
            )
            layer.register_parameter("w2_input_scale", w2_input_scale)
            set_weight_attrs(w2_input_scale, extra_weight_attrs)
        else:
            layer.w13_input_scale = None
            layer.w2_input_scale = None

    def process_weights_after_loading(self, layer):
        self._setup_kernel(layer)

    def _setup_kernel(self, layer: RoutedExperts):
        """Setup kernel using oracle functions for MXFP4 schemes (W4A16, W4A8)."""
        w13_bias = getattr(layer, "w13_bias", None)
        w2_bias = getattr(layer, "w2_bias", None)

        if self.model_type == "gpt_oss":
            converted_weights = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
                mxfp4_backend=self.mxfp4_backend,
                layer=layer,
                w13_weight=layer.w13_weight,
                w2_weight=layer.w2_weight,
                w13_weight_scale=layer.w13_weight_scale,
                w2_weight_scale=layer.w2_weight_scale,
                w13_bias=w13_bias,
                w2_bias=w2_bias,
                w13_input_scale=layer.w13_input_scale,
                w2_input_scale=layer.w2_input_scale,
            )
        else:
            converted_weights = convert_weight_to_mxfp4_moe_kernel_format(
                mxfp4_backend=self.mxfp4_backend,
                layer=layer,
                w13_weight=layer.w13_weight,
                w2_weight=layer.w2_weight,
                w13_weight_scale=layer.w13_weight_scale,
                w2_weight_scale=layer.w2_weight_scale,
                w13_bias=w13_bias,
                w2_bias=w2_bias,
                activation=self.moe.activation,
            )
        w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = converted_weights

        # Handle weight/scale assignment based on backend type
        if self.mxfp4_backend in TRITON_BACKENDS or self.mxfp4_backend in (
            Mxfp4MoeBackend.AITER_MXFP4_FP8,
        ):
            # Triton-based backends: w13/w2 are triton_kernels.tensor.Tensor
            # Store on layer for apply(), scales are PrecisionConfig
            layer.w13_weight_triton = w13
            layer.w2_weight_triton = w2
            self.w13_precision_config = w13_scale
            self.w2_precision_config = w2_scale
        else:
            # Standard backends: replace parameters
            replace_parameter(layer, "w13_weight", w13)
            replace_parameter(layer, "w2_weight", w2)
            replace_parameter(layer, "w13_weight_scale", w13_scale)
            replace_parameter(layer, "w2_weight_scale", w2_scale)

        if w13_bias is not None and w2_bias is not None:
            replace_parameter(layer, "w13_bias", w13_bias)
            replace_parameter(layer, "w2_bias", w2_bias)

        torch.accelerator.empty_cache()

        # Build quant config and kernel
        self.moe_quant_config = self.get_fused_moe_quant_config(layer)
        if self.moe_quant_config is not None and self.experts_cls is not None:
            self.moe_kernel = make_mxfp4_moe_kernel(
                moe_quant_config=self.moe_quant_config,
                moe_config=self.moe,
                mxfp4_backend=self.mxfp4_backend,
                experts_cls=self.experts_cls,
                routing_tables=layer._expert_routing_tables(),
            )
            self.moe_kernel.fused_experts.keep_constants(layer)
            self.moe_kernel.fused_experts.process_weights_after_loading(layer)

    def get_fused_moe_quant_config(
        self, layer: RoutedExperts
    ) -> FusedMoEQuantConfig | None:
        # For oracle-based backends (W4A16, W4A8), use make_mxfp4_moe_quant_config
        if self.mxfp4_backend not in (Mxfp4MoeBackend.NONE, Mxfp4MoeBackend.EMULATION):
            # Determine scale source based on backend type
            if self.mxfp4_backend in TRITON_BACKENDS or self.mxfp4_backend in (
                Mxfp4MoeBackend.AITER_MXFP4_FP8,
            ):
                w1_scale = self.w13_precision_config
                w2_scale = self.w2_precision_config
            else:
                w1_scale = layer.w13_weight_scale
                w2_scale = layer.w2_weight_scale

            return make_mxfp4_moe_quant_config(
                mxfp4_backend=self.mxfp4_backend,
                w1_scale=w1_scale,
                w2_scale=w2_scale,
                w1_bias=getattr(layer, "w13_bias", None),
                w2_bias=getattr(layer, "w2_bias", None),
                a1_scale=getattr(layer, "w13_input_scale", None),
                a2_scale=getattr(layer, "w2_input_scale", None),
                gemm1_alpha=getattr(layer, "swiglu_alpha", None),
                gemm1_beta=getattr(layer, "swiglu_beta", None),
                swiglu_limit=getattr(layer, "swiglu_limit", None),
                layer=layer,
            )

        # Emulation and other schemes
        if self.ocp_mx_scheme == "w_mxfp4":
            return mxfp4_w4a16_moe_quant_config(
                w1_scale=layer.w13_weight_scale,
                w2_scale=layer.w2_weight_scale,
                w1_bias=layer.w13_bias,
                w2_bias=layer.w2_bias,
            )
        elif self.ocp_mx_scheme == "w_mxfp4_a_fp8":
            return mxfp4_w4a8_moe_quant_config(
                w1_scale=layer.w13_weight_scale,
                w2_scale=layer.w2_weight_scale,
                a1_scale=layer.w13_input_scale,
                a2_scale=layer.w2_input_scale,
                w1_bias=layer.w13_bias,
                w2_bias=layer.w2_bias,
                block_shape=None,
            )
        elif self.ocp_mx_scheme in ["w_mxfp6_e3m2_a_fp8", "w_mxfp6_e2m3_a_fp8"]:
            raise NotImplementedError(
                "Currently there is no corresponding fused moe quant config configured "
                f"in vLLM for OCP MX scheme {self.ocp_mx_scheme}. Please open an issue."
            )
        else:
            assert self.input_dtype is not None
            return ocp_mx_moe_quant_config(
                quant_dtype=self.input_dtype,
                weight_dtype=self.weight_dtype,
                w1_scale=layer.w13_weight_scale,
                w2_scale=layer.w2_weight_scale,
                w1_bias=layer.w13_bias,
                w2_bias=layer.w2_bias,
                a1_scale=None,
                a2_scale=None,
                block_shape=None,
                gemm1_alpha=getattr(layer, "swiglu_alpha", None),
                gemm1_beta=getattr(layer, "swiglu_beta", None),
                gemm1_clamp_limit=getattr(layer, "swiglu_limit", None),
            )

    @property
    def supports_eplb(self) -> bool:
        # AITER shuffle keeps expert dim outermost, so EPLB row moves are layout-safe.
        return True

    @property
    def is_monolithic(self) -> bool:
        if self.moe_kernel is not None:
            return self.moe_kernel.is_monolithic
        return False

    def apply(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        shared_experts: SharedExperts | None,
        shared_experts_input: torch.Tensor | None,
    ) -> torch.Tensor | UnfinalizedMoEOutput:
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
            hidden_states=x,
            w1=triton_weight(layer, "w13_weight"),
            w2=triton_weight(layer, "w2_weight"),
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            expert_map=layer.expert_map,
            shared_experts_input=shared_experts_input,
        )

    def apply_monolithic(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        router_logits: torch.Tensor,
        input_ids: torch.Tensor | None = None,
    ) -> torch.Tensor | UnfinalizedMoEOutput:
        assert self.is_monolithic
        assert self.moe_kernel is not None
        return self.moe_kernel.apply_monolithic(
            hidden_states=x,
            w1=triton_weight(layer, "w13_weight"),
            w2=triton_weight(layer, "w2_weight"),
            router_logits=router_logits,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            num_expert_group=layer.num_expert_group,
            topk_group=layer.topk_group,
            e_score_correction_bias=layer.e_score_correction_bias,
            routed_scaling_factor=layer.routed_scaling_factor,
            routing_sink=layer.routing_sink,
        )

_setup_kernel(layer)

Setup kernel using oracle functions for MXFP4 schemes (W4A16, W4A8).

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
def _setup_kernel(self, layer: RoutedExperts):
    """Setup kernel using oracle functions for MXFP4 schemes (W4A16, W4A8)."""
    w13_bias = getattr(layer, "w13_bias", None)
    w2_bias = getattr(layer, "w2_bias", None)

    if self.model_type == "gpt_oss":
        converted_weights = convert_gpt_oss_weight_to_mxfp4_moe_kernel_format(
            mxfp4_backend=self.mxfp4_backend,
            layer=layer,
            w13_weight=layer.w13_weight,
            w2_weight=layer.w2_weight,
            w13_weight_scale=layer.w13_weight_scale,
            w2_weight_scale=layer.w2_weight_scale,
            w13_bias=w13_bias,
            w2_bias=w2_bias,
            w13_input_scale=layer.w13_input_scale,
            w2_input_scale=layer.w2_input_scale,
        )
    else:
        converted_weights = convert_weight_to_mxfp4_moe_kernel_format(
            mxfp4_backend=self.mxfp4_backend,
            layer=layer,
            w13_weight=layer.w13_weight,
            w2_weight=layer.w2_weight,
            w13_weight_scale=layer.w13_weight_scale,
            w2_weight_scale=layer.w2_weight_scale,
            w13_bias=w13_bias,
            w2_bias=w2_bias,
            activation=self.moe.activation,
        )
    w13, w2, w13_scale, w2_scale, w13_bias, w2_bias = converted_weights

    # Handle weight/scale assignment based on backend type
    if self.mxfp4_backend in TRITON_BACKENDS or self.mxfp4_backend in (
        Mxfp4MoeBackend.AITER_MXFP4_FP8,
    ):
        # Triton-based backends: w13/w2 are triton_kernels.tensor.Tensor
        # Store on layer for apply(), scales are PrecisionConfig
        layer.w13_weight_triton = w13
        layer.w2_weight_triton = w2
        self.w13_precision_config = w13_scale
        self.w2_precision_config = w2_scale
    else:
        # Standard backends: replace parameters
        replace_parameter(layer, "w13_weight", w13)
        replace_parameter(layer, "w2_weight", w2)
        replace_parameter(layer, "w13_weight_scale", w13_scale)
        replace_parameter(layer, "w2_weight_scale", w2_scale)

    if w13_bias is not None and w2_bias is not None:
        replace_parameter(layer, "w13_bias", w13_bias)
        replace_parameter(layer, "w2_bias", w2_bias)

    torch.accelerator.empty_cache()

    # Build quant config and kernel
    self.moe_quant_config = self.get_fused_moe_quant_config(layer)
    if self.moe_quant_config is not None and self.experts_cls is not None:
        self.moe_kernel = make_mxfp4_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            mxfp4_backend=self.mxfp4_backend,
            experts_cls=self.experts_cls,
            routing_tables=layer._expert_routing_tables(),
        )
        self.moe_kernel.fused_experts.keep_constants(layer)
        self.moe_kernel.fused_experts.process_weights_after_loading(layer)

QuarkW4A16Int4MoEMethod

Bases: QuarkMoEMethod

Quark packed INT4 weight-only MoE method.

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
class QuarkW4A16Int4MoEMethod(QuarkMoEMethod):
    """Quark packed INT4 weight-only MoE method."""

    supported_activation_quant_keys: list[QuantKey | None] = [None]
    supported_weight_quant_keys: list[QuantKey] = [
        kInt4Static,
        kInt4Static32,
        kInt4StaticAsym,
        kInt4Static32Asym,
    ]

    def __init__(
        self,
        weight_quant_key: QuantKey,
        activation_quant_key: QuantKey | None,
        weight_config: dict[str, Any],
        pack_method: str,
        moe: FusedMoEConfig,
        quant_config: "QuarkConfig | None" = None,  # type: ignore # noqa F821
    ):
        super().__init__(moe, weight_quant_key, activation_quant_key)

        self.quant_config = quant_config
        # group_size is read from the Quark config rather than weight_quant_key:
        # the int4 QuantKeys only encode group_size 32 or -1, but Quark int4
        # checkpoints commonly use group_size 128, which the QuantKey cannot
        # represent. Symmetry is unambiguous, so it comes from the key.
        self.group_size = parse_w4a16_int4_weight_config(weight_config)[0]
        self.is_symmetric = weight_quant_key.symmetric
        self.bit8_pack_factor = 2
        self.pack_reorder = pack_method == "reorder"

        self.use_wna16_backend = True
        self.wna16_backend, self.experts_cls = select_wna16_moe_backend(
            config=self.moe,
            weight_key=weight_quant_key,
            quant_config=self.quant_config,
            may_have_zp=not self.is_symmetric,
            may_have_bias=False,
        )

    def create_weights(
        self,
        layer: RoutedExperts,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        group_size = self.group_size
        group_size_div_factor = 1

        while intermediate_size_per_partition % group_size or hidden_size % group_size:
            group_size = group_size // 2
            group_size_div_factor *= 2
            assert group_size >= 32
        layer.group_size = group_size
        layer.group_size_div_factor = group_size_div_factor

        strategy = FusedMoeWeightScaleSupported.GROUP.value
        extra_weight_attrs.update({"quant_method": strategy, "is_transposed": False})

        assert "weight_loader" in extra_weight_attrs
        weight_loader = extra_weight_attrs["weight_loader"]
        extra_weight_attrs["weight_loader"] = self.get_weight_loader(
            layer, weight_loader
        )

        w13_weight = torch.nn.Parameter(
            torch.empty(
                num_experts,
                2 * intermediate_size_per_partition,
                hidden_size // self.bit8_pack_factor,
                dtype=torch.uint8,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight", w13_weight)
        set_weight_attrs(w13_weight, extra_weight_attrs)

        w2_weight = torch.nn.Parameter(
            torch.empty(
                num_experts,
                hidden_size,
                intermediate_size_per_partition // self.bit8_pack_factor,
                dtype=torch.uint8,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight", w2_weight)
        set_weight_attrs(w2_weight, extra_weight_attrs)

        w13_weight_scale = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                2 * intermediate_size_per_partition,
                hidden_size // group_size,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight_scale", w13_weight_scale)
        set_weight_attrs(w13_weight_scale, extra_weight_attrs)

        w2_weight_scale = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
                intermediate_size_per_partition // group_size,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight_scale", w2_weight_scale)
        set_weight_attrs(w2_weight_scale, extra_weight_attrs)

        w13_weight_zero_point = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                2 * intermediate_size_per_partition // self.bit8_pack_factor,
                hidden_size // group_size,
                dtype=torch.uint8,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point)
        set_weight_attrs(w13_weight_zero_point, extra_weight_attrs)

        w2_weight_zero_point = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size // self.bit8_pack_factor,
                intermediate_size_per_partition // group_size,
                dtype=torch.uint8,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point)
        set_weight_attrs(w2_weight_zero_point, extra_weight_attrs)

    def process_weights_after_loading(self, layer: RoutedExperts) -> None:
        w13_qzeros = layer.w13_weight_zero_point if not self.is_symmetric else None
        w2_qzeros = layer.w2_weight_zero_point if not self.is_symmetric else None

        converted = convert_to_wna16_moe_kernel_format(
            backend=self.wna16_backend,
            layer=layer,
            quant_config=self.quant_config,
            w13=layer.w13_weight,
            w2=layer.w2_weight,
            w13_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            w13_qzeros=w13_qzeros,
            w2_qzeros=w2_qzeros,
        )

        if converted is None:
            # Backend rewrote the layer's params in place (e.g. Humming).
            self._setup_kernel(layer)
            return

        (
            w13_qweight,
            w2_qweight,
            w13_scales,
            w2_scales,
            _,  # w13_qzeros
            _,  # w2_qzeros
            _,  # w13_bias
            _,  # w2_bias
        ) = converted

        replace_parameter(layer, "w13_weight", w13_qweight)
        replace_parameter(layer, "w2_weight", w2_qweight)
        replace_parameter(layer, "w13_weight_scale", w13_scales)
        replace_parameter(layer, "w2_weight_scale", w2_scales)

        self._setup_kernel(layer)

    def _setup_kernel(self, layer: RoutedExperts) -> None:
        assert self.experts_cls is not None
        self.moe_quant_config = self.get_fused_moe_quant_config(layer)
        assert self.moe_quant_config is not None
        self.moe_kernel = make_wna16_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            experts_cls=self.experts_cls,
            backend=self.wna16_backend,
            routing_tables=layer._expert_routing_tables(),
        )

    def get_fused_moe_quant_config(
        self, layer: RoutedExperts
    ) -> FusedMoEQuantConfig | None:
        w1_zp = layer.w13_weight_zero_point if not self.is_symmetric else None
        w2_zp = layer.w2_weight_zero_point if not self.is_symmetric else None
        return make_wna16_moe_quant_config(
            w1_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            w1_zp=w1_zp,
            w2_zp=w2_zp,
            group_size=layer.group_size,
            num_bits=4,
        )

    def apply(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        shared_experts: SharedExperts | None,
        shared_experts_input: torch.Tensor | None,
    ) -> torch.Tensor:
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            expert_map=layer.expert_map,
            shared_experts=shared_experts,
            shared_experts_input=shared_experts_input,
        )

    def get_weight_loader(self, layer: RoutedExperts, weight_loader):
        def convert_quark_tensor(tensor: torch.Tensor, tensor_type: str):
            size0 = tensor.size(0)
            tensor = tensor.view(torch.uint8)

            shifter = torch.tensor([0, 4], dtype=torch.uint8, device=tensor.device)
            tensor = (tensor[:, :, None] >> shifter) & 0xF

            if self.pack_reorder:
                reverse_awq_pack_order = [0, 4, 1, 5, 2, 6, 3, 7]
                tensor = tensor.view(-1, 8)[:, reverse_awq_pack_order]
            else:
                tensor = tensor.view(-1, 8)
            if self.is_symmetric:
                tensor = tensor ^ 0x8
            tensor = tensor.view(size0, -1)

            tensor = tensor.T.contiguous()
            if tensor_type == "weight":
                tensor = tensor[:, 1::2] * 16 + tensor[:, ::2]
            elif tensor_type == "zero_point":
                tensor = tensor[1::2, :] * 16 + tensor[::2, :]
            return tensor

        def quark_int4_weight_loader(
            param: torch.nn.Parameter,
            loaded_weight: torch.Tensor,
            weight_name: str,
            shard_id: str,
            expert_id: int,
            return_success: bool = False,
        ):
            loaded_weight = loaded_weight.to(param.data.device)
            shard_size = layer.intermediate_size_per_partition

            if weight_name.endswith("_weight"):
                loaded_weight = convert_quark_tensor(loaded_weight, "weight")
            elif weight_name.endswith("_weight_zero_point"):
                loaded_weight = convert_quark_tensor(loaded_weight, "zero_point")
            elif weight_name.endswith("_weight_scale"):
                loaded_weight = loaded_weight.T

            if layer.group_size_div_factor > 1 and (
                "weight_zero_point" in weight_name or "weight_scale" in weight_name
            ):
                loaded_weight = loaded_weight.repeat_interleave(
                    layer.group_size_div_factor, 1
                )

            if "w13_weight_zero_point" in weight_name:
                tensor = loaded_weight.view(
                    layer.moe_config.tp_size, -1, loaded_weight.size(1)
                )[layer.moe_config.tp_rank]
                if shard_id == "w1":
                    param.data[expert_id, : shard_size // 2] = tensor
                else:
                    param.data[expert_id, shard_size // 2 :] = tensor
                return True if return_success else None
            elif "w2_weight_zero_point" in weight_name:
                param.data[expert_id] = loaded_weight.view(
                    loaded_weight.size(0), layer.moe_config.tp_size, -1
                )[:, layer.moe_config.tp_rank]
                return True if return_success else None

            return weight_loader(
                param,
                loaded_weight,
                weight_name,
                shard_id,
                expert_id,
                return_success=return_success,
            )

        return quark_int4_weight_loader

QuarkW8A8Fp8MoEMethod

Bases: QuarkMoEMethod

Methods:

  • dequantize_weight –

    Yield (experts, w13, w2) dequantized block-FP8 experts, for online

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
 627
 628
 629
 630
 631
 632
 633
 634
 635
 636
 637
 638
 639
 640
 641
 642
 643
 644
 645
 646
 647
 648
 649
 650
 651
 652
 653
 654
 655
 656
 657
 658
 659
 660
 661
 662
 663
 664
 665
 666
 667
 668
 669
 670
 671
 672
 673
 674
 675
 676
 677
 678
 679
 680
 681
 682
 683
 684
 685
 686
 687
 688
 689
 690
 691
 692
 693
 694
 695
 696
 697
 698
 699
 700
 701
 702
 703
 704
 705
 706
 707
 708
 709
 710
 711
 712
 713
 714
 715
 716
 717
 718
 719
 720
 721
 722
 723
 724
 725
 726
 727
 728
 729
 730
 731
 732
 733
 734
 735
 736
 737
 738
 739
 740
 741
 742
 743
 744
 745
 746
 747
 748
 749
 750
 751
 752
 753
 754
 755
 756
 757
 758
 759
 760
 761
 762
 763
 764
 765
 766
 767
 768
 769
 770
 771
 772
 773
 774
 775
 776
 777
 778
 779
 780
 781
 782
 783
 784
 785
 786
 787
 788
 789
 790
 791
 792
 793
 794
 795
 796
 797
 798
 799
 800
 801
 802
 803
 804
 805
 806
 807
 808
 809
 810
 811
 812
 813
 814
 815
 816
 817
 818
 819
 820
 821
 822
 823
 824
 825
 826
 827
 828
 829
 830
 831
 832
 833
 834
 835
 836
 837
 838
 839
 840
 841
 842
 843
 844
 845
 846
 847
 848
 849
 850
 851
 852
 853
 854
 855
 856
 857
 858
 859
 860
 861
 862
 863
 864
 865
 866
 867
 868
 869
 870
 871
 872
 873
 874
 875
 876
 877
 878
 879
 880
 881
 882
 883
 884
 885
 886
 887
 888
 889
 890
 891
 892
 893
 894
 895
 896
 897
 898
 899
 900
 901
 902
 903
 904
 905
 906
 907
 908
 909
 910
 911
 912
 913
 914
 915
 916
 917
 918
 919
 920
 921
 922
 923
 924
 925
 926
 927
 928
 929
 930
 931
 932
 933
 934
 935
 936
 937
 938
 939
 940
 941
 942
 943
 944
 945
 946
 947
 948
 949
 950
 951
 952
 953
 954
 955
 956
 957
 958
 959
 960
 961
 962
 963
 964
 965
 966
 967
 968
 969
 970
 971
 972
 973
 974
 975
 976
 977
 978
 979
 980
 981
 982
 983
 984
 985
 986
 987
 988
 989
 990
 991
 992
 993
 994
 995
 996
 997
 998
 999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
class QuarkW8A8Fp8MoEMethod(QuarkMoEMethod):
    supported_activation_quant_keys = [
        kFp8StaticTensorSym,
        kFp8DynamicTensorSym,
        kFp8DynamicTokenSym,
        kFp8Dynamic128Sym,
    ]
    supported_weight_quant_keys = [
        kFp8StaticChannelSym,
        kFp8StaticTensorSym,
        kFp8Static128BlockSym,
    ]

    def __init__(
        self,
        moe: FusedMoEConfig,
        weight_quant_key: QuantKey,
        activation_quant_key: QuantKey | None,
    ):
        super().__init__(moe, weight_quant_key, activation_quant_key)
        self.weight_dtype = "fp8"
        if weight_quant_key == kFp8StaticChannelSym:
            self.weight_qscheme = "per_channel"
        elif weight_quant_key == kFp8Static128BlockSym:
            self.weight_qscheme = "per_block"
        else:
            self.weight_qscheme = "per_tensor"
        if activation_quant_key == kFp8DynamicTokenSym:
            self.input_qscheme = "per_channel"
        elif activation_quant_key == kFp8Dynamic128Sym:
            self.input_qscheme = "per_group"
        else:
            self.input_qscheme = "per_tensor"
        self.static_input_scales = activation_quant_key == kFp8StaticTensorSym
        per_tensor = (
            self.weight_qscheme == "per_tensor" and self.input_qscheme == "per_tensor"
        )
        per_channel = (
            self.weight_qscheme == "per_channel" and self.input_qscheme == "per_channel"
        )
        per_block = (
            self.weight_qscheme == "per_block" and self.input_qscheme == "per_group"
        )
        if not (per_tensor or per_channel or per_block):
            raise ValueError(
                "For FP8 Fused MoE layers, only per-tensor, per-channel and "
                "per-block scales for weights and activations are supported. "
                f"Found {self.weight_qscheme}, {self.input_qscheme}"
            )  # noqa E501

        # One scale per (block_n, block_k) tile of the weight, so the block
        # shape is carried by the weight key's group shape.
        self.weight_block_size: list[int] | None = (
            list(weight_quant_key.scale.group_shape) if per_block else None
        )
        self.block_quant = self.weight_block_size is not None

        if per_channel:
            self.act_quant_group_shape = GroupShape.PER_TOKEN
        elif per_block:
            assert self.weight_block_size is not None
            self.act_quant_group_shape = GroupShape(1, self.weight_block_size[1])
        else:
            self.act_quant_group_shape = GroupShape.PER_TENSOR

        self.fp8_backend, self.experts_cls = select_fp8_moe_backend(
            config=moe,
            weight_key=self.weight_quant_key,
            activation_key=self.activation_quant_key,
        )

        self.model_type = getattr(
            get_current_vllm_config().model_config.hf_config, "model_type", None
        )

    def create_weights(
        self,
        layer: RoutedExperts,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        layer.num_experts = num_experts
        layer.orig_dtype = params_dtype
        layer.weight_block_size = None
        params_dtype = torch.float8_e4m3fn

        if self.block_quant:
            assert self.weight_block_size is not None
            layer.weight_block_size = self.weight_block_size
            block_n, block_k = self.weight_block_size[0], self.weight_block_size[1]
            validate_fp8_block_shape_moe(
                intermediate_size_per_partition,
                self.weight_block_size,
            )

        # WEIGHTS
        w13_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                self.moe.w13_num_shards * intermediate_size_per_partition,
                hidden_size,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight", w13_weight)
        set_weight_attrs(w13_weight, extra_weight_attrs)

        w2_weight = torch.nn.Parameter(
            torch.zeros(
                num_experts,
                hidden_size,
                intermediate_size_per_partition,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight", w2_weight)
        set_weight_attrs(w2_weight, extra_weight_attrs)

        # WEIGHT_SCALES
        if self.weight_qscheme == "per_tensor":
            # Allocate 2 scales for w1 and w3 respectively.
            # They are combined to a single scale after weight loading.
            if self.model_type != "gpt_oss":
                w13_weight_scale = torch.nn.Parameter(
                    torch.ones(
                        num_experts, self.moe.w13_num_shards, dtype=torch.float32
                    ),
                    requires_grad=False,
                )
            else:
                # For gpt_oss, the w1(gate) & w3(up) are fused as one.
                # Therefore, only one weight scale for each expert.
                w13_weight_scale = torch.nn.Parameter(
                    torch.ones(num_experts, 1, dtype=torch.float32), requires_grad=False
                )
            layer.register_parameter("w13_weight_scale", w13_weight_scale)
            w2_weight_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32), requires_grad=False
            )
            layer.register_parameter("w2_weight_scale", w2_weight_scale)
            # Add PER-TENSOR quantization for RoutedExperts.weight_loader.
            extra_weight_attrs.update(
                {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}
            )
            set_weight_attrs(w13_weight_scale, extra_weight_attrs)
            set_weight_attrs(w2_weight_scale, extra_weight_attrs)
        elif self.weight_qscheme == "per_channel":
            # quark's scale is 1 dim.
            w13_weight_scale = torch.nn.Parameter(
                torch.ones(
                    num_experts,
                    self.moe.w13_num_shards * intermediate_size_per_partition,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w13_weight_scale", w13_weight_scale)
            w2_weight_scale = torch.nn.Parameter(
                torch.ones(num_experts, hidden_size, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_weight_scale", w2_weight_scale)
            # Add PER-CHANNEL quantization for RoutedExperts.weight_loader.
            extra_weight_attrs.update(
                {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value}
            )
            set_weight_attrs(w13_weight_scale, extra_weight_attrs)
            set_weight_attrs(w2_weight_scale, extra_weight_attrs)
        elif self.weight_qscheme == "per_block":
            # One scale per (block_n, block_k) tile of each expert's weight.
            w13_weight_scale = torch.nn.Parameter(
                torch.ones(
                    num_experts,
                    self.moe.w13_num_shards
                    * ((intermediate_size_per_partition + block_n - 1) // block_n),
                    (hidden_size + block_k - 1) // block_k,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w13_weight_scale", w13_weight_scale)
            w2_weight_scale = torch.nn.Parameter(
                torch.ones(
                    num_experts,
                    (hidden_size + block_n - 1) // block_n,
                    (intermediate_size_per_partition + block_k - 1) // block_k,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w2_weight_scale", w2_weight_scale)
            # Add PER-BLOCK quantization for RoutedExperts.weight_loader.
            extra_weight_attrs.update(
                {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}
            )
            set_weight_attrs(w13_weight_scale, extra_weight_attrs)
            set_weight_attrs(w2_weight_scale, extra_weight_attrs)

        # INPUT_SCALES
        if self.static_input_scales:
            w13_input_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32), requires_grad=False
            )
            layer.register_parameter("w13_input_scale", w13_input_scale)
            set_weight_attrs(w13_input_scale, extra_weight_attrs)

            w2_input_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32), requires_grad=False
            )
            layer.register_parameter("w2_input_scale", w2_input_scale)
            set_weight_attrs(w2_input_scale, extra_weight_attrs)
        else:
            layer.w13_input_scale = None
            layer.w2_input_scale = None

        if self.has_bias:
            w13_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
                    self.moe.w13_num_shards * intermediate_size_per_partition,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w13_bias", w13_bias)
            set_weight_attrs(w13_bias, extra_weight_attrs)

            w2_bias = torch.nn.Parameter(
                torch.zeros(num_experts, hidden_size, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_bias", w2_bias)
            set_weight_attrs(w2_bias, extra_weight_attrs)
        else:
            layer.w13_bias, layer.w2_bias = None, None

    def dequantize_weight(
        self, layer: torch.nn.Module
    ) -> Iterator[tuple[slice, torch.Tensor, torch.Tensor]]:
        """Yield `(experts, w13, w2)` dequantized block-FP8 experts, for online
        requantization. Other FP8 weight schemes are not supported."""
        return dequantize_block_fp8_moe_weights(
            layer.w13_weight,
            layer.w13_weight_scale,
            layer.w2_weight,
            layer.w2_weight_scale,
            num_w13_shards=self.moe.w13_num_shards,
            weight_block_size=self.weight_block_size,
            out_dtype=layer.orig_dtype,
        )

    def process_weights_after_loading(self, layer: RoutedExperts) -> None:
        # Fp8 moe kernels require a single activation scale.
        # We take the max of all the scales in case they differ.
        if self.static_input_scales:
            if layer.w13_input_scale is None or layer.w2_input_scale is None:
                raise ValueError(
                    "QuantConfig has static quantization, but found "
                    "activation scales are None."
                )
            if not all_close_1d(layer.w13_input_scale) or not all_close_1d(
                layer.w2_input_scale
            ):
                logger.warning_once(
                    "Found input_scales that are not equal for "
                    "fp8 MoE layer. Using the maximum across experts "
                    "for each layer. "
                )
            replace_parameter(layer, "w13_input_scale", layer.w13_input_scale.max())
            replace_parameter(layer, "w2_input_scale", layer.w2_input_scale.max())

        if current_platform.is_fp8_fnuz():
            # Normalize the weights and scales
            w13_weight, w13_weight_scale, w13_input_scale = (
                normalize_e4m3fn_to_e4m3fnuz(
                    layer.w13_weight, layer.w13_weight_scale, layer.w13_input_scale
                )
            )
            w2_weight, w2_weight_scale, w2_input_scale = normalize_e4m3fn_to_e4m3fnuz(
                layer.w2_weight, layer.w2_weight_scale, layer.w2_input_scale
            )
            # Reset the parameter
            replace_parameter(layer, "w13_weight", w13_weight)
            replace_parameter(layer, "w13_weight_scale", w13_weight_scale)
            if w13_input_scale is not None:
                replace_parameter(layer, "w13_input_scale", w13_input_scale)
            replace_parameter(layer, "w2_weight", w2_weight)
            replace_parameter(layer, "w2_weight_scale", w2_weight_scale)
            if w2_input_scale is not None:
                replace_parameter(layer, "w2_input_scale", w2_input_scale)

        # For per-tensor case, Fp8 moe kernel needs single weight scale
        # for w13 per expert. Use max then dequant and requant each expert.
        if self.weight_qscheme == "per_tensor":
            assert layer.w13_weight_scale is not None
            shard_size = layer.intermediate_size_per_partition
            max_w13_scales = layer.w13_weight_scale.max(dim=1).values

            # For gpt_oss, w1 and w3 are fused into a single combined
            # gate_up_proj tensor with size 2*intermediate_size_per_partition
            # and only one scale per expert.
            # Process the entire weight tensor as one shard.
            if self.model_type == "gpt_oss":
                for expert_id in range(layer.local_num_experts):
                    # Process all 2*intermediate_size_per_partition rows at once
                    dq_weight = per_tensor_dequantize(
                        layer.w13_weight[expert_id],
                        layer.w13_weight_scale[expert_id][0],
                    )
                    layer.w13_weight[expert_id], _ = ops.scaled_fp8_quant(
                        dq_weight, max_w13_scales[expert_id]
                    )
            else:
                # For non-gpt_oss, process w1 and w3 shards separately
                for expert_id in range(layer.local_num_experts):
                    start = 0
                    for shard_id in range(self.moe.w13_num_shards):
                        dq_weight = per_tensor_dequantize(
                            layer.w13_weight[expert_id][start : start + shard_size, :],
                            layer.w13_weight_scale[expert_id][shard_id],
                        )
                        (
                            layer.w13_weight[expert_id][start : start + shard_size, :],
                            _,
                        ) = ops.scaled_fp8_quant(dq_weight, max_w13_scales[expert_id])
                        start += shard_size

            replace_parameter(layer, "w13_weight_scale", max_w13_scales)

        # quark's scale is 1 dim.
        elif self.weight_qscheme == "per_channel":
            if self.act_quant_group_shape == GroupShape.PER_TOKEN:
                w13_weight_scale = layer.w13_weight_scale.unsqueeze(-1)
                replace_parameter(layer, "w13_weight_scale", w13_weight_scale)
                w2_weight_scale = layer.w2_weight_scale.unsqueeze(-1)
                replace_parameter(layer, "w2_weight_scale", w2_weight_scale)
        self._setup_kernel(layer)

    def _setup_kernel(self, layer: RoutedExperts) -> None:
        w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
            fp8_backend=self.fp8_backend,
            layer=layer,
            w13=layer.w13_weight,
            w2=layer.w2_weight,
            w13_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            w13_input_scale=layer.w13_input_scale,
            w2_input_scale=layer.w2_input_scale,
        )
        replace_parameter(layer, "w13_weight", w13)
        replace_parameter(layer, "w2_weight", w2)
        replace_parameter(layer, "w13_weight_scale", w13_scale)
        replace_parameter(layer, "w2_weight_scale", w2_scale)

        self.moe_quant_config = self.get_fused_moe_quant_config(layer)
        assert self.moe_quant_config is not None
        assert self.experts_cls is not None
        self.moe_kernel = make_fp8_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            fp8_backend=self.fp8_backend,
            experts_cls=self.experts_cls,
            routing_tables=layer._expert_routing_tables(),
        )
        self.moe_kernel.fused_experts.process_weights_after_loading(layer)
        self.moe_kernel.fused_experts.keep_constants(layer)

    def get_fused_moe_quant_config(self, layer: RoutedExperts) -> FusedMoEQuantConfig:
        return make_fp8_moe_quant_config(
            fp8_backend=self.fp8_backend,
            w1_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            a1_scale=layer.w13_input_scale,
            a2_scale=layer.w2_input_scale,
            w1_bias=getattr(layer, "w13_bias", None),
            w2_bias=getattr(layer, "w2_bias", None),
            block_shape=self.weight_block_size,
            per_act_token_quant=self.input_qscheme == "per_channel",
            per_out_ch_quant=self.weight_qscheme == "per_channel",
            swiglu_limit=getattr(layer, "swiglu_limit", None),
            layer=layer,
        )

    def apply(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        shared_experts: SharedExperts | None,
        shared_experts_input: torch.Tensor | None,
    ) -> torch.Tensor | UnfinalizedMoEOutput:
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            expert_map=layer.expert_map,
            shared_experts=shared_experts,
            shared_experts_input=shared_experts_input,
        )

dequantize_weight(layer)

Yield (experts, w13, w2) dequantized block-FP8 experts, for online requantization. Other FP8 weight schemes are not supported.

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
def dequantize_weight(
    self, layer: torch.nn.Module
) -> Iterator[tuple[slice, torch.Tensor, torch.Tensor]]:
    """Yield `(experts, w13, w2)` dequantized block-FP8 experts, for online
    requantization. Other FP8 weight schemes are not supported."""
    return dequantize_block_fp8_moe_weights(
        layer.w13_weight,
        layer.w13_weight_scale,
        layer.w2_weight,
        layer.w2_weight_scale,
        num_w13_shards=self.moe.w13_num_shards,
        weight_block_size=self.weight_block_size,
        out_dtype=layer.orig_dtype,
    )

QuarkW8A8Int8MoEMethod

Bases: QuarkMoEMethod

Quark W8A8 INT8 MoE method.

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
class QuarkW8A8Int8MoEMethod(QuarkMoEMethod):
    """Quark W8A8 INT8 MoE method."""

    supported_activation_quant_keys = [
        kInt8StaticTensorSym,
        kInt8StaticTensorAsym,
        kInt8DynamicTensorSym,
        kInt8DynamicTensorAsym,
        kInt8DynamicTokenSym,
        kInt8DynamicTokenAsym,
    ]
    supported_weight_quant_keys = [
        kInt8StaticChannelSym,
        kInt8StaticTensorSym,
    ]

    def __init__(
        self,
        moe: FusedMoEConfig,
        weight_quant_key: QuantKey,
        activation_quant_key: QuantKey | None,
    ):
        super().__init__(moe, weight_quant_key, activation_quant_key)

        # TODO: oracle needs to support asym int8.
        if self.activation_quant_key == kInt8StaticTensorAsym:
            self.activation_quant_key = kInt8StaticTensorSym
        if self.activation_quant_key == kInt8DynamicTensorAsym:
            self.activation_quant_key = kInt8DynamicTensorSym
        if self.activation_quant_key == kInt8DynamicTokenAsym:
            self.activation_quant_key = kInt8DynamicTokenSym

        self.weight_qscheme = (
            "per_channel" if weight_quant_key == kInt8StaticChannelSym else "per_tensor"
        )

        assert self.activation_quant_key is not None
        self.static_input_scales = self.activation_quant_key.scale.static

        self.moe_quant_config: FusedMoEQuantConfig | None = None
        self.moe_kernel: mk.FusedMoEKernel | None = None
        self.int8_backend: Int8MoeBackend | None = None
        self.experts_cls: type[mk.FusedMoEExperts] | None = None

        # Dynamic-activation INT8 MoE goes through the oracle + modular kernel.
        # The modular TritonExperts kernel consumes float activations and
        # quantizes them to int8 itself, so it cannot apply a loaded static
        # activation scale (this matches CompressedTensorsW8A8Int8MoEMethod).
        # TODO: Static-activation INT8 therefore stays on the legacy fused_experts
        # path (see apply()) for now, preserving pre-refactor behavior.
        # Needs to be migrated to expert backend.
        if not self.static_input_scales:
            # Map the Quark weight scheme to oracle quant keys. Per-channel
            # weights pair with dynamic per-token activations; per-tensor
            # weights with dynamic per-tensor activations.
            self.int8_backend, self.experts_cls = select_int8_moe_backend(
                config=moe,
                weight_key=self.weight_quant_key,
                activation_key=self.activation_quant_key,
            )

    def create_weights(
        self,
        layer: torch.nn.Module,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        layer.num_experts = num_experts
        layer.orig_dtype = params_dtype
        layer.weight_block_size = None
        params_dtype = torch.int8

        # WEIGHTS
        w13_weight = torch.nn.Parameter(
            torch.empty(
                num_experts,
                self.moe.w13_num_shards * intermediate_size_per_partition,
                hidden_size,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w13_weight", w13_weight)
        set_weight_attrs(w13_weight, extra_weight_attrs)

        w2_weight = torch.nn.Parameter(
            torch.empty(
                num_experts,
                hidden_size,
                intermediate_size_per_partition,
                dtype=params_dtype,
            ),
            requires_grad=False,
        )
        layer.register_parameter("w2_weight", w2_weight)
        set_weight_attrs(w2_weight, extra_weight_attrs)

        # WEIGHT_SCALES
        if self.weight_qscheme == "per_channel":
            w13_weight_scale = torch.nn.Parameter(
                torch.ones(
                    num_experts,
                    self.moe.w13_num_shards * intermediate_size_per_partition,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w13_weight_scale", w13_weight_scale)
            w2_weight_scale = torch.nn.Parameter(
                torch.ones(num_experts, hidden_size, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_weight_scale", w2_weight_scale)
            extra_weight_attrs.update(
                {"quant_method": FusedMoeWeightScaleSupported.CHANNEL.value}
            )
            set_weight_attrs(w13_weight_scale, extra_weight_attrs)
            set_weight_attrs(w2_weight_scale, extra_weight_attrs)
        else:
            # per-tensor: one scalar per expert (two for the fused w1/w3)
            w13_weight_scale = torch.nn.Parameter(
                torch.ones(num_experts, self.moe.w13_num_shards, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w13_weight_scale", w13_weight_scale)
            w2_weight_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_weight_scale", w2_weight_scale)
            extra_weight_attrs.update(
                {"quant_method": FusedMoeWeightScaleSupported.TENSOR.value}
            )
            set_weight_attrs(w13_weight_scale, extra_weight_attrs)
            set_weight_attrs(w2_weight_scale, extra_weight_attrs)

        # INPUT_SCALES
        if self.static_input_scales:
            # Static activations: the per-expert scales are loaded from the
            # checkpoint (used by the legacy fused_experts path).
            w13_input_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w13_input_scale", w13_input_scale)
            set_weight_attrs(w13_input_scale, extra_weight_attrs)

            w2_input_scale = torch.nn.Parameter(
                torch.ones(num_experts, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_input_scale", w2_input_scale)
            set_weight_attrs(w2_input_scale, extra_weight_attrs)
        else:
            # Dynamic activations are quantized in-kernel (no stored scale).
            layer.w13_input_scale = None
            layer.w2_input_scale = None

        # ZERO POINTS (loaded but discarded after loading; kernel uses symmetric)
        w13_input_zero_point = torch.nn.Parameter(
            torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
            requires_grad=False,
        )
        layer.register_parameter("w13_input_zero_point", w13_input_zero_point)
        set_weight_attrs(w13_input_zero_point, extra_weight_attrs)

        w2_input_zero_point = torch.nn.Parameter(
            torch.zeros(num_experts, dtype=torch.int8),
            requires_grad=False,
        )
        layer.register_parameter("w2_input_zero_point", w2_input_zero_point)
        set_weight_attrs(w2_input_zero_point, extra_weight_attrs)

        if self.weight_qscheme == "per_channel":
            w13_weight_zero_point = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
                    self.moe.w13_num_shards * intermediate_size_per_partition,
                    dtype=torch.int8,
                ),
                requires_grad=False,
            )
            w2_weight_zero_point = torch.nn.Parameter(
                torch.zeros(num_experts, hidden_size, dtype=torch.int8),
                requires_grad=False,
            )
        else:
            w13_weight_zero_point = torch.nn.Parameter(
                torch.zeros(num_experts, self.moe.w13_num_shards, dtype=torch.int8),
                requires_grad=False,
            )
            w2_weight_zero_point = torch.nn.Parameter(
                torch.zeros(num_experts, dtype=torch.int8),
                requires_grad=False,
            )
        layer.register_parameter("w13_weight_zero_point", w13_weight_zero_point)
        set_weight_attrs(w13_weight_zero_point, extra_weight_attrs)
        layer.register_parameter("w2_weight_zero_point", w2_weight_zero_point)
        set_weight_attrs(w2_weight_zero_point, extra_weight_attrs)

        # BIAS
        if self.has_bias:
            w13_bias = torch.nn.Parameter(
                torch.zeros(
                    num_experts,
                    self.moe.w13_num_shards * intermediate_size_per_partition,
                    dtype=torch.float32,
                ),
                requires_grad=False,
            )
            layer.register_parameter("w13_bias", w13_bias)
            set_weight_attrs(w13_bias, extra_weight_attrs)
            w2_bias = torch.nn.Parameter(
                torch.zeros(num_experts, hidden_size, dtype=torch.float32),
                requires_grad=False,
            )
            layer.register_parameter("w2_bias", w2_bias)
            set_weight_attrs(w2_bias, extra_weight_attrs)
        else:
            layer.w13_bias, layer.w2_bias = None, None

    def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
        # Loaded zero points stay registered; the INT8 fused MoE kernel ignores them.
        # For static input scales, collapse the per-expert scales to a single
        # value (the legacy fused_experts path expects one scale per layer).
        if self.static_input_scales:
            if layer.w13_input_scale is None or layer.w2_input_scale is None:
                raise ValueError(
                    "QuantConfig has static quantization, but found "
                    "activation scales are None."
                )
            if not all_close_1d(layer.w13_input_scale) or not all_close_1d(
                layer.w2_input_scale
            ):
                logger.warning_once(
                    "Found input_scales that are not equal for "
                    "INT8 MoE layer. Using the maximum across experts "
                    "for each layer."
                )
            replace_parameter(layer, "w13_input_scale", layer.w13_input_scale.max())
            replace_parameter(layer, "w2_input_scale", layer.w2_input_scale.max())

        # Per-channel scales: 2D [E, N] -> 3D [E, N, 1] for the int8 MoE kernel.
        if self.weight_qscheme == "per_channel":
            for attr in ("w13_weight_scale", "w2_weight_scale"):
                param = getattr(layer, attr, None)
                if param is not None and param.dim() == 2:
                    replace_parameter(
                        layer,
                        attr,
                        torch.nn.Parameter(
                            param.data.unsqueeze(-1).contiguous(),
                            requires_grad=False,
                        ),
                    )

        # For per-tensor weights, merge the w1/w3 scales into a single
        # per-expert scale (dequant -> requant at the max scale).
        if self.weight_qscheme == "per_tensor":
            assert layer.w13_weight_scale is not None
            shard_size = layer.intermediate_size_per_partition
            max_w13_scales = layer.w13_weight_scale.max(dim=1).values

            for expert_id in range(layer.local_num_experts):
                start = 0
                for shard_id in range(self.moe.w13_num_shards):
                    dq_weight = per_tensor_dequantize(
                        layer.w13_weight[expert_id][start : start + shard_size, :],
                        layer.w13_weight_scale[expert_id][shard_id],
                    )
                    layer.w13_weight[expert_id][start : start + shard_size, :], _, _ = (
                        ops.scaled_int8_quant(
                            dq_weight,
                            scale=max_w13_scales[expert_id],
                        )
                    )
                    start += shard_size

            replace_parameter(layer, "w13_weight_scale", max_w13_scales)

        # Dynamic activations run through the oracle's modular kernel; static
        # activations use the legacy fused_experts path in apply().
        if not self.static_input_scales:
            assert self.int8_backend is not None
            assert self.experts_cls is not None
            w13, w2 = convert_to_int8_moe_kernel_format(
                int8_backend=self.int8_backend,
                w13=layer.w13_weight,
                w2=layer.w2_weight,
                layer=layer,
                w13_scale=layer.w13_weight_scale,
            )
            replace_parameter(layer, "w13_weight", w13)
            replace_parameter(layer, "w2_weight", w2)

        self.moe_quant_config = self.get_fused_moe_quant_config(layer)
        assert self.moe_quant_config is not None

        if not self.static_input_scales:
            assert self.int8_backend is not None
            assert self.experts_cls is not None
            self.moe_kernel = make_int8_moe_kernel(
                int8_backend=self.int8_backend,
                moe_quant_config=self.moe_quant_config,
                moe_config=self.moe,
                experts_cls=self.experts_cls,
                routing_tables=layer._expert_routing_tables(),
            )

    def get_fused_moe_quant_config(
        self, layer: torch.nn.Module
    ) -> FusedMoEQuantConfig | None:
        # Activation- and weight-scale layouts are independent: derive each from
        # the quant key that describes it.
        activation_quant_key = self.activation_quant_key
        assert activation_quant_key is not None
        per_act_token_quant = activation_quant_key.scale.group_shape.is_per_token()
        per_out_ch_quant = self.weight_quant_key.scale.group_shape.is_per_channel()

        # Static-activation INT8 has no oracle backend (it uses the legacy
        # fused_experts path); build its config directly.
        if self.int8_backend is None:
            return int8_w8a8_moe_quant_config(
                w1_scale=layer.w13_weight_scale,
                w2_scale=layer.w2_weight_scale,
                a1_scale=layer.w13_input_scale,
                a2_scale=layer.w2_input_scale,
                w1_bias=getattr(layer, "w13_bias", None),
                w2_bias=getattr(layer, "w2_bias", None),
                per_act_token_quant=per_act_token_quant,
                per_out_ch_quant=per_out_ch_quant,
            )
        return make_int8_moe_quant_config(
            int8_backend=self.int8_backend,
            w1_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            a1_scale=layer.w13_input_scale,
            a2_scale=layer.w2_input_scale,
            w1_bias=getattr(layer, "w13_bias", None),
            w2_bias=getattr(layer, "w2_bias", None),
            per_act_token_quant=per_act_token_quant,
            per_out_ch_quant=per_out_ch_quant,
            layer=layer,
        )

    def apply(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        shared_experts: SharedExperts | None,
        shared_experts_input: torch.Tensor | None,
    ) -> torch.Tensor:
        if self.moe_kernel is not None:
            return self.moe_kernel.apply(
                hidden_states=x,
                w1=layer.w13_weight,
                w2=layer.w2_weight,
                topk_weights=topk_weights,
                topk_ids=topk_ids,
                activation=layer.activation,
                global_num_experts=layer.global_num_experts,
                apply_router_weight_on_input=layer.apply_router_weight_on_input,
                expert_map=layer.expert_map,
                shared_experts_input=shared_experts_input,
            )

        # Static-activation INT8 MoE: legacy monolithic path (the modular kernel
        # quantizes activations dynamically and cannot apply a loaded scale).
        from vllm.model_executor.layers.fused_moe import fused_experts

        return fused_experts(
            hidden_states=x,
            w1=layer.w13_weight,
            w2=layer.w2_weight,
            topk_weights=topk_weights,
            topk_ids=topk_ids,
            activation=layer.activation,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            quant_config=self.moe_quant_config,
        )

QuarkW8A8Mxfp8MoEMethod

Bases: QuarkMoEMethod

Quark OCP MX MXFP8 (W8A8) experts.

FP8 E4M3 weights with E8M0 uint8 scales per 1x32 block (Quark fp8_e4m3 per_group group_size=32 scale_format=e8m0), and activations dynamically quantized to MXFP8. The stored layout is the one the online, ModelOpt and compressed-tensors MXFP8 MoE methods produce or load ([E, rows, cols] E4M3, [E, rows, cols / 32] uint8), and the weights go through the same backend selection, kernel-format conversion and kernel (AITER_MXFP8 on gfx950).

Source code in vllm/model_executor/layers/quantization/quark/quark_moe.py
class QuarkW8A8Mxfp8MoEMethod(QuarkMoEMethod):
    """Quark OCP MX MXFP8 (W8A8) experts.

    FP8 E4M3 weights with E8M0 uint8 scales per 1x32 block (Quark `fp8_e4m3`
    `per_group` `group_size=32` `scale_format=e8m0`), and activations
    dynamically quantized to MXFP8. The stored layout is the one the online,
    ModelOpt and compressed-tensors MXFP8 MoE methods produce or load
    (`[E, rows, cols]` E4M3, `[E, rows, cols / 32]` uint8), and the weights go
    through the same backend selection, kernel-format conversion and kernel
    (`AITER_MXFP8` on gfx950).
    """

    supported_activation_quant_keys = [kMxfp8Dynamic]
    supported_weight_quant_keys = [kMxfp8Static]

    def __init__(
        self,
        moe: FusedMoEConfig,
        weight_quant_key: QuantKey,
        activation_quant_key: QuantKey | None,
    ):
        super().__init__(moe, weight_quant_key, activation_quant_key)
        if self.has_bias:
            raise NotImplementedError("Quark MXFP8 MoE does not support biases.")
        self.weight_block_size = [1, MXFP8_BLOCK_SIZE]
        self.mxfp8_backend, self.experts_cls = select_mxfp8_moe_backend(config=self.moe)
        logger.info_once(
            "Using %s backend for Quark MXFP8 (W8A8) MoE.", self.mxfp8_backend.value
        )

    def create_weights(
        self,
        layer: RoutedExperts,
        num_experts: int,
        hidden_size: int,
        intermediate_size_per_partition: int,
        params_dtype: torch.dtype,
        **extra_weight_attrs,
    ):
        if (
            hidden_size % MXFP8_BLOCK_SIZE != 0
            or intermediate_size_per_partition % MXFP8_BLOCK_SIZE != 0
        ):
            raise ValueError(
                "Quark MXFP8 MoE requires hidden_size and "
                f"intermediate_size_per_partition divisible by {MXFP8_BLOCK_SIZE}, "
                f"got {hidden_size} and {intermediate_size_per_partition}."
            )
        layer.num_experts = num_experts
        layer.orig_dtype = params_dtype
        w13_rows = self.moe.w13_num_shards * intermediate_size_per_partition
        # The scales are sharded with their weights along both dims (one scale
        # per 32 input elements), like the OCP MX block scales.
        extra_weight_attrs.update(
            {"quant_method": FusedMoeWeightScaleSupported.BLOCK.value}
        )
        shapes = {
            "w13_weight": ((num_experts, w13_rows, hidden_size), MXFP8_VALUE_DTYPE),
            "w2_weight": (
                (num_experts, hidden_size, intermediate_size_per_partition),
                MXFP8_VALUE_DTYPE,
            ),
            "w13_weight_scale": (
                (num_experts, w13_rows, hidden_size // MXFP8_BLOCK_SIZE),
                MXFP8_SCALE_DTYPE,
            ),
            "w2_weight_scale": (
                (
                    num_experts,
                    hidden_size,
                    intermediate_size_per_partition // MXFP8_BLOCK_SIZE,
                ),
                MXFP8_SCALE_DTYPE,
            ),
        }
        for name, (shape, dtype) in shapes.items():
            param = torch.nn.Parameter(
                torch.empty(shape, dtype=dtype), requires_grad=False
            )
            layer.register_parameter(name, param)
            set_weight_attrs(param, extra_weight_attrs)

        layer.w13_input_scale = None
        layer.w2_input_scale = None

    def process_weights_after_loading(self, layer: RoutedExperts) -> None:
        layer.weight_block_size = self.weight_block_size
        w13, w2, w13_scale, w2_scale = convert_to_fp8_moe_kernel_format(
            fp8_backend=self.mxfp8_backend,
            layer=layer,
            w13=layer.w13_weight,
            w2=layer.w2_weight,
            w13_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            w13_input_scale=None,
            w2_input_scale=None,
        )
        replace_parameter(layer, "w13_weight", w13)
        replace_parameter(layer, "w2_weight", w2)
        replace_parameter(layer, "w13_weight_scale", w13_scale)
        replace_parameter(layer, "w2_weight_scale", w2_scale)

        self.moe_quant_config = self.get_fused_moe_quant_config(layer)
        assert self.moe_quant_config is not None
        assert self.experts_cls is not None
        self.moe_kernel = make_fp8_moe_kernel(
            moe_quant_config=self.moe_quant_config,
            moe_config=self.moe,
            fp8_backend=self.mxfp8_backend,
            experts_cls=self.experts_cls,
            routing_tables=layer._expert_routing_tables(),
        )
        self.moe_kernel.fused_experts.keep_constants(layer)

    def get_fused_moe_quant_config(
        self, layer: RoutedExperts
    ) -> FusedMoEQuantConfig | None:
        return make_fp8_moe_quant_config(
            fp8_backend=self.mxfp8_backend,
            w1_scale=layer.w13_weight_scale,
            w2_scale=layer.w2_weight_scale,
            a1_scale=None,
            a2_scale=None,
            block_shape=self.weight_block_size,
            swiglu_limit=getattr(layer, "swiglu_limit", None),
            gemm1_alpha=getattr(layer, "swiglu_alpha", None),
            gemm1_beta=getattr(layer, "swiglu_beta", None),
            layer=layer,
        )

    @property
    def supports_eplb(self) -> bool:
        return True

    def apply(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        topk_weights: torch.Tensor,
        topk_ids: torch.Tensor,
        shared_experts: SharedExperts | None,
        shared_experts_input: torch.Tensor | None,
    ) -> torch.Tensor | UnfinalizedMoEOutput:
        assert not self.is_monolithic
        assert self.moe_kernel is not None
        return self.moe_kernel.apply(
            x,
            layer.w13_weight,
            layer.w2_weight,
            topk_weights,
            topk_ids,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            shared_experts=shared_experts,
            shared_experts_input=shared_experts_input,
        )

    def apply_monolithic(
        self,
        layer: RoutedExperts,
        x: torch.Tensor,
        router_logits: torch.Tensor,
        input_ids: torch.Tensor | None = None,
    ) -> torch.Tensor | UnfinalizedMoEOutput:
        assert self.is_monolithic
        assert self.moe_kernel is not None
        return self.moe_kernel.apply_monolithic(
            x,
            layer.w13_weight,
            layer.w2_weight,
            router_logits,
            activation=layer.activation,
            global_num_experts=layer.global_num_experts,
            expert_map=layer.expert_map,
            apply_router_weight_on_input=layer.apply_router_weight_on_input,
            num_expert_group=layer.num_expert_group,
            topk_group=layer.topk_group,
            e_score_correction_bias=layer.e_score_correction_bias,
            routed_scaling_factor=layer.routed_scaling_factor,
            routing_sink=layer.routing_sink,
        )