Skip to content

vllm.models.deepseek_v41.nvidia.specialized_decode

DeepSeek-V4.1 specialized decode on NVIDIA GPUs: a decode executor over the stock backbone, and one over the DSpark draft model's, enabled by additional_config={"decode_backend": "latency_specialized"}.

The executors run FULL-graph decode steps; every other step runs the stock forward. They issue cross-rank collectives, so the deployment check below is collective over the TP group and every per-step decision reads host state that all TP ranks share.

Modules:

Functions:

  • decode_backend –

    additional_config["decode_backend"]: auto (the default, the stock

  • records_decode_metadata –

    Whether SM80 or SM90 records the attention metadata builds of FULL

  • register_draft –

    Have the target model's state build draft a decode executor.

  • specialized_decode_config –

    The specialized decode settings, or None for the stock decode.

  • take_drafts –

    The live drafts registered since the last call, in registration

  • unsupported –

    Every requirement of the specialized decode this deployment misses.

  • uses_checked_flashinfer –

    Whether the installed FlashInfer is at least the version whose internals

decode_backend(vllm_config)

additional_config["decode_backend"]: auto (the default, the stock decode) or latency_specialized.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
def decode_backend(vllm_config: VllmConfig) -> str:
    """`additional_config["decode_backend"]`: `auto` (the default, the stock
    decode) or `latency_specialized`."""
    extra = vllm_config.additional_config
    backend = extra.get("decode_backend", "auto") if isinstance(extra, dict) else "auto"
    if backend not in ("auto", "latency_specialized"):
        raise ValueError("Supported decode_backend values are auto/latency_specialized")
    return backend

records_decode_metadata(vllm_config)

Whether SM80 or SM90 records the attention metadata builds of FULL decode steps (runtime/metadata_graphs.py): with FULL decode CUDA graphs, without micro-batching or context parallelism.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
def records_decode_metadata(vllm_config: VllmConfig) -> bool:
    """Whether SM80 or SM90 records the attention metadata builds of FULL
    decode steps (`runtime/metadata_graphs.py`): with FULL decode CUDA
    graphs, without micro-batching or context parallelism."""
    from .ampere import is_ampere
    from .hopper import is_hopper

    parallel = vllm_config.parallel_config
    cudagraph_mode = vllm_config.compilation_config.cudagraph_mode
    return (
        (is_ampere() or is_hopper())
        and cudagraph_mode is not None
        and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
        and not parallel.use_ubatching
        and parallel.decode_context_parallel_size == 1
        and parallel.prefill_context_parallel_size == 1
    )

register_draft(draft)

Have the target model's state build draft a decode executor.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
def register_draft(draft: "DSparkDeepseekV4ForCausalLM") -> None:
    """Have the target model's state build `draft` a decode executor."""
    _drafts.append(weakref.ref(draft))

specialized_decode_config(vllm_config)

The specialized decode settings, or None for the stock decode.

Collective over the TP group when requested: every rank raises if any rank cannot run the specialized decode.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
def specialized_decode_config(vllm_config: VllmConfig) -> DecodeConfig | None:
    """The specialized decode settings, or None for the stock decode.

    Collective over the TP group when requested: every rank raises if any rank
    cannot run the specialized decode.
    """
    if decode_backend(vllm_config) == "auto":
        return None
    missing = unsupported(vllm_config)
    supported = torch.tensor([0 if missing else 1], dtype=torch.int32)
    # The NVLink multicast check is rank-local (the custom all-reduce catches a
    # failed setup on each rank), and ranks that disagree would hang.
    dist.all_reduce(supported, op=dist.ReduceOp.MIN, group=get_tp_group().cpu_group)
    if not supported.item():
        reason = "; ".join(missing) or "another tensor parallel rank to support it"
        raise ValueError(f"decode_backend=latency_specialized needs {reason}.")
    logger.info_once(
        "DeepSeek-V4.1 specialized decode on, for FULL-graph decode steps of up "
        "to %d rows.",
        FUSED_MHC_MAX_ROWS,
    )
    kernel = vllm_config.kernel_config
    return DecodeConfig(
        autotune=kernel.enable_flashinfer_autotune is not False,
        jit_warmup=bool(kernel.enable_jit_warmup),
    )

take_drafts()

The live drafts registered since the last call, in registration order.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
def take_drafts() -> list["DSparkDeepseekV4ForCausalLM"]:
    """The live drafts registered since the last call, in registration
    order."""
    drafts = [draft for ref in _drafts if (draft := ref()) is not None]
    _drafts.clear()
    return drafts

unsupported(vllm_config)

Every requirement of the specialized decode this deployment misses.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
def unsupported(vllm_config: VllmConfig) -> list[str]:
    """Every requirement of the specialized decode this deployment misses."""
    parallel = vllm_config.parallel_config
    attention = vllm_config.attention_config
    spec = vllm_config.speculative_config
    compilation = vllm_config.compilation_config
    cudagraph_mode = compilation.cudagraph_mode
    requirements = {
        "a GB300 (SM103) GPU": current_platform.is_device_capability(103),
        "tensor parallel size 4 without pipeline, data, expert or context "
        "parallelism or micro-batching": (
            parallel.tensor_parallel_size == 4
            and parallel.pipeline_parallel_size == 1
            and parallel.data_parallel_size == 1
            and not parallel.enable_expert_parallel
            and parallel.decode_context_parallel_size == 1
            and parallel.prefill_context_parallel_size == 1
            and not parallel.use_ubatching
        ),
        "DSpark speculative decoding without adaptive verification": (
            spec is not None
            and spec.method == "dspark"
            and not spec.enable_adaptive_verification
        ),
        "the FLASHINFER_MLA_SPARSE_DSV41 attention backend, an fp8 KV cache "
        "and the MXFP4 indexer with sparse logits": (
            attention.backend == AttentionBackendEnum.FLASHINFER_MLA_SPARSE_DSV41
            and vllm_config.cache_config.cache_dtype in ("fp8", "fp8_e4m3")
            and attention.indexer_kv_dtype == "mxfp4"
            and attention.indexer_sparse_logits
        ),
        "FULL CUDA graphs for decode batches only (FULL_AND_PIECEWISE or "
        "FULL_DECODE_ONLY)": (
            cudagraph_mode is not None
            and cudagraph_mode.decode_mode() == CUDAGraphMode.FULL
            and cudagraph_mode.mixed_mode() != CUDAGraphMode.FULL
        ),
        # The decode loop runs in forward; a compiled forward would trace it.
        "no torch.compile of the model": (
            compilation.mode != CompilationMode.STOCK_TORCH_COMPILE
        ),
        # The routed-expert GEMMs and the publishing wo_b bind weight addresses.
        "no sleep mode": not vllm_config.model_config.enable_sleep_mode,
        # The replicated embedding table is copied from the shards once, at load.
        "no runtime weight updates": vllm_config.weight_transfer_config is None,
        "no LoRA, KV transfer or batch invariance": (
            vllm_config.lora_config is None
            and vllm_config.kv_transfer_config is None
            and not envs.VLLM_BATCH_INVARIANT
        ),
        "DeepGEMM and NVLink multicast for the fused all-reduce + mHC": (
            supports_mhc_overlap(vllm_config) and supports_mhc_all_reduce(vllm_config)
        ),
    }
    return [need for need, ok in requirements.items() if not ok]

uses_checked_flashinfer(feature) cached

Whether the installed FlashInfer is at least the version whose internals feature was checked against; logs once per feature when it is not.

Source code in vllm/models/deepseek_v41/nvidia/specialized_decode/__init__.py
@cache
def uses_checked_flashinfer(feature: str) -> bool:
    """Whether the installed FlashInfer is at least the version whose internals
    `feature` was checked against; logs once per feature when it is not."""
    try:
        version = importlib.metadata.version("flashinfer-python")
    except importlib.metadata.PackageNotFoundError:
        version = None
    if version is not None and Version(version) >= Version(MIN_FLASHINFER_VERSION):
        return True
    logger.warning(
        "DeepSeek-V4.1 specialized decode: %s needs flashinfer-python >= %s "
        "(found %s); using the stock path instead.",
        feature,
        MIN_FLASHINFER_VERSION,
        version,
    )
    return False