Skip to content

vllm.model_executor.layers.quantization.online.legacy_env

Deprecated environment variables redirected to --quantization-config.

Functions:

fp8_moe_requant_args()

Online quantization args equivalent to VLLM_FP8_MOE_REQUANT[_ACT].

Only meaningful for block-FP8 checkpoints on ROCm, which the caller checks.

Source code in vllm/model_executor/layers/quantization/online/legacy_env.py
def fp8_moe_requant_args() -> QuantizationConfigArgs | None:
    """Online quantization args equivalent to `VLLM_FP8_MOE_REQUANT[_ACT]`.

    Only meaningful for block-FP8 checkpoints on ROCm, which the caller checks.
    """
    if envs.VLLM_FP8_MOE_REQUANT != "mxfp4":
        return None
    act = envs.VLLM_FP8_MOE_REQUANT_ACT
    # a4w4 is the default activation of online MXFP4. a8w4 is MXFP8 activations
    # and a16w4 bf16 ones (`activation: null`), both on the interleaved AITER
    # kernels (`AITER_MXFP4_BF16`).
    if act == "a4w4":
        config: dict = {"moe": "mxfp4"}
        spec = QuantSpec(weight=kMxfp4Static)
    elif act == "a8w4":
        config = {"moe": {"weight": "mxfp4", "activation": "mxfp8"}}
        spec = QuantSpec(weight=kMxfp4Static, activation=kMxfp8Dynamic)
    else:
        config = {"moe": {"weight": "mxfp4", "activation": None}}
        spec = QuantSpec(weight=kMxfp4Static, activation=None)
        logger.warning_once(
            "VLLM_FP8_MOE_REQUANT_ACT=a16w4 no longer selects the "
            "aiter_triton_mxfp4_bf16 backend: `activation: null` runs bf16 "
            "activations on the backend picked by the oracle. Pass "
            "--moe-backend aiter_triton_mxfp4_bf16 to keep it."
        )
    _warn("VLLM_FP8_MOE_REQUANT and VLLM_FP8_MOE_REQUANT_ACT", config)
    return QuantizationConfigArgs(moe=spec)

muse_moe_online_quant_args(num_hidden_layers)

Online quantization args equivalent to VLLM_ROCM_MUSE_MOE_ONLINE_QUANT=mxfp4: the first and last decoder layers stay in BF16.

Source code in vllm/model_executor/layers/quantization/online/legacy_env.py
def muse_moe_online_quant_args(num_hidden_layers: int) -> QuantizationConfigArgs | None:
    """Online quantization args equivalent to
    `VLLM_ROCM_MUSE_MOE_ONLINE_QUANT=mxfp4`: the first and last decoder layers
    stay in BF16."""
    if envs.VLLM_ROCM_MUSE_MOE_ONLINE_QUANT != "mxfp4":
        return None
    ignore = ["*layers.0.*", f"*layers.{num_hidden_layers - 1}.*"]
    _warn("VLLM_ROCM_MUSE_MOE_ONLINE_QUANT", {"moe": "mxfp4", "ignore": ignore})
    return QuantizationConfigArgs(moe=QuantSpec(weight=kMxfp4Static), ignore=ignore)

reject_muse_env_with_quantization_config()

VLLM_ROCM_MUSE_MOE_ONLINE_QUANT and --quantization-config are exclusive.

Source code in vllm/model_executor/layers/quantization/online/legacy_env.py
def reject_muse_env_with_quantization_config() -> None:
    """`VLLM_ROCM_MUSE_MOE_ONLINE_QUANT` and `--quantization-config` are exclusive."""
    if envs.VLLM_ROCM_MUSE_MOE_ONLINE_QUANT == "mxfp4":
        raise ValueError(
            "VLLM_ROCM_MUSE_MOE_ONLINE_QUANT (deprecated) and --quantization-config "
            "cannot be used together. Remove VLLM_ROCM_MUSE_MOE_ONLINE_QUANT."
        )