Skip to content

vllm.models.mistral.mistral_large4.amd.specialized_decode

Modules:

  • mono –
  • runtime –

    Mistral Large 4 specialized decode on MI350X / MI355X (gfx950), enabled by

Classes:

Functions:

DecodeConfig dataclass

Methods:

  • refusal_reasons –

    Default-deny: every deployment feature the kernels do not serve.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/spec.py
@dataclass(frozen=True)
class DecodeConfig:
    tp_size: int = 4  # the fused indexer's q_b split needs 16 local heads
    blocks: int = 256  # one CTA per CU, all co-resident
    # rows a launch serves (one decode token per request): vLLM's FULL-graph
    # capture sizes, so graph replays pad decode batches to them; eager steps
    # of other sizes and batches above 4 rows run stock. The 8-row launch
    # builds and is correct but measured slower than vLLM's layers (super-linear
    # up/gate and down stages), so it is not served.
    widths: tuple[int, ...] = (1, 2, 4)
    topk: int = 2048
    # rows a request takes in a step: 1, or num_speculative_tokens + 1 for a
    # speculative verify block (the kernels' ``index_request_width``)
    q: int = 1

    def refusal_reasons(self, config: VllmConfig) -> list[str]:
        """Default-deny: every deployment feature the kernels do not serve."""
        pc = config.parallel_config
        cc = config.cache_config
        reasons: list[str] = []
        if pc.tensor_parallel_size != self.tp_size:
            reasons.append(f"requires tensor parallel size {self.tp_size}")
        if pc.pipeline_parallel_size != 1:
            reasons.append("does not support pipeline parallelism")
        if pc.data_parallel_size != 1:
            reasons.append("does not support data parallelism")
        if pc.decode_context_parallel_size != 1:
            reasons.append("does not support decode context parallelism")
        if pc.prefill_context_parallel_size != 1:
            reasons.append("does not support prefill context parallelism")
        if pc.use_ubatching:
            reasons.append("does not support DBO or ubatching")
        if pc.enable_expert_parallel:
            reasons.append("does not support expert parallelism")
        if pc.enable_eplb:
            reasons.append("does not support EPLB")
        if str(cc.cache_dtype) not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
            reasons.append("requires a bf16 or fp8 (e4m3) KV cache")
        if cc.block_size is not None and cc.block_size % 16:
            reasons.append("requires a KV cache block size that is a multiple of 16")
        if cc.kv_offloading_size is not None:
            reasons.append("does not support KV cache offloading")
        if config.lora_config is not None:
            reasons.append("does not support LoRA")
        transfer = config.kv_transfer_config
        if transfer is not None and transfer.kv_connector is not None:
            reasons.append("does not support a KV connector")
        spec = config.speculative_config
        if spec is not None:
            if getattr(spec, "method", None) not in SPEC_METHODS:
                reasons.append(
                    "supports only " + "/".join(SPEC_METHODS) + " speculative decoding"
                )
            dynamic = getattr(spec, "uses_dynamic_speculative_decoding", None)
            if dynamic is not None and dynamic():
                reasons.append("does not support dynamic speculative decoding")
            if (getattr(spec, "num_speculative_tokens", 0) or 0) + 1 > MAX_ROWS:
                reasons.append(f"supports at most {MAX_ROWS - 1} speculative tokens")
        if config.model_config.enable_sleep_mode:
            reasons.append("does not support sleep mode")
        if config.weight_transfer_config is not None:
            reasons.append("does not support weight transfer or updates")
        offload = config.offload_config
        if offload.uva.cpu_offload_gb > 0 or offload.prefetch.offload_group_size > 0:
            reasons.append("does not support model weight offloading")
        if config.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
            reasons.append("does not support stock torch.compile")
        aux = config.aux_output_config
        if getattr(aux, "enable_return_routed_experts", False):
            reasons.append("does not support routed-expert capture")
        return reasons

refusal_reasons(config)

Default-deny: every deployment feature the kernels do not serve.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/spec.py
def refusal_reasons(self, config: VllmConfig) -> list[str]:
    """Default-deny: every deployment feature the kernels do not serve."""
    pc = config.parallel_config
    cc = config.cache_config
    reasons: list[str] = []
    if pc.tensor_parallel_size != self.tp_size:
        reasons.append(f"requires tensor parallel size {self.tp_size}")
    if pc.pipeline_parallel_size != 1:
        reasons.append("does not support pipeline parallelism")
    if pc.data_parallel_size != 1:
        reasons.append("does not support data parallelism")
    if pc.decode_context_parallel_size != 1:
        reasons.append("does not support decode context parallelism")
    if pc.prefill_context_parallel_size != 1:
        reasons.append("does not support prefill context parallelism")
    if pc.use_ubatching:
        reasons.append("does not support DBO or ubatching")
    if pc.enable_expert_parallel:
        reasons.append("does not support expert parallelism")
    if pc.enable_eplb:
        reasons.append("does not support EPLB")
    if str(cc.cache_dtype) not in ("auto", "bfloat16", "fp8", "fp8_e4m3"):
        reasons.append("requires a bf16 or fp8 (e4m3) KV cache")
    if cc.block_size is not None and cc.block_size % 16:
        reasons.append("requires a KV cache block size that is a multiple of 16")
    if cc.kv_offloading_size is not None:
        reasons.append("does not support KV cache offloading")
    if config.lora_config is not None:
        reasons.append("does not support LoRA")
    transfer = config.kv_transfer_config
    if transfer is not None and transfer.kv_connector is not None:
        reasons.append("does not support a KV connector")
    spec = config.speculative_config
    if spec is not None:
        if getattr(spec, "method", None) not in SPEC_METHODS:
            reasons.append(
                "supports only " + "/".join(SPEC_METHODS) + " speculative decoding"
            )
        dynamic = getattr(spec, "uses_dynamic_speculative_decoding", None)
        if dynamic is not None and dynamic():
            reasons.append("does not support dynamic speculative decoding")
        if (getattr(spec, "num_speculative_tokens", 0) or 0) + 1 > MAX_ROWS:
            reasons.append(f"supports at most {MAX_ROWS - 1} speculative tokens")
    if config.model_config.enable_sleep_mode:
        reasons.append("does not support sleep mode")
    if config.weight_transfer_config is not None:
        reasons.append("does not support weight transfer or updates")
    offload = config.offload_config
    if offload.uva.cpu_offload_gb > 0 or offload.prefetch.offload_group_size > 0:
        reasons.append("does not support model weight offloading")
    if config.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
        reasons.append("does not support stock torch.compile")
    aux = config.aux_output_config
    if getattr(aux, "enable_return_routed_experts", False):
        reasons.append("does not support routed-expert capture")
    return reasons

specialized_decode_config(config)

The deployment's specialized decode, None for vLLM's decode. Raises with every reason when it is requested for an unsupported deployment.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/spec.py
def specialized_decode_config(config: VllmConfig) -> DecodeConfig | None:
    """The deployment's specialized decode, None for vLLM's decode. Raises with
    every reason when it is requested for an unsupported deployment."""
    backend = decode_backend(config)
    if backend == AUTO:
        return None
    if backend != SPECIALIZED:
        raise ValueError(
            f"decode_backend must be '{AUTO}' or '{SPECIALIZED}', got {backend!r}"
        )
    reasons = DECODE_CONFIG.refusal_reasons(config)
    if reasons:
        raise ValueError(
            "Mistral Large 4 specialized decode unsupported: " + "; ".join(reasons)
        )
    spec = config.speculative_config
    if spec is None:
        return DECODE_CONFIG
    # one request's verify block (q rows) per launch: two requests (8 rows at
    # q=4) build and are correct but measured slower than vLLM's layers, as the
    # 8-row launch without speculation (see ``widths``)
    q = spec.num_speculative_tokens + 1
    return replace(DECODE_CONFIG, q=q, widths=(q,))