Skip to content

vllm.models.mistral.mistral_large4.nvidia.model

Mistral Large 4: MLA + DSA sparse attention, MoE, NoPE layers, KV sharing.

The text model is hosted by PixtralForConditionalGeneration for the multimodal checkpoint.

Classes:

Functions:

  • defers_allreduce –

    Whether sublayers leave their outputs TP-partial and fuse the all-reduce

MistralLarge4DecoderLayer

Bases: Module

Methods:

  • forward –

    Run the layer on the outputs of input_norm.

  • input_norm –

    Fold the previous layer's output into the residual stream and

  • residual_norm –

    Add a sublayer output to the residual stream and normalize; with

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
class MistralLarge4DecoderLayer(nn.Module):
    def __init__(
        self,
        vllm_config: VllmConfig,
        config: PreTrainedConfig,
        spec: MistralLayerSpec,
        prefix: str,
        topk_indices_buffer: torch.Tensor | None,
        index_group_builder: SparseMLAIndexGroupBuilder | None,
        zero_kv_inputs: tuple[torch.Tensor, torch.Tensor],
        quant_config: QuantizationConfig | None,
        non_causal: bool = False,
        sliding_window: int | None = None,
    ) -> None:
        super().__init__()
        self.defer_allreduce = defers_allreduce(vllm_config)
        self.input_layernorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
        self.self_attn = MistralMLAAttention(
            vllm_config,
            config,
            spec,
            prefix=f"{prefix}.self_attn",
            topk_indices_buffer=topk_indices_buffer,
            index_group_builder=index_group_builder,
            zero_kv_inputs=zero_kv_inputs,
            quant_config=quant_config,
            non_causal=non_causal,
            sliding_window=sliding_window,
            reduce_results=not self.defer_allreduce,
        )
        self.post_attention_layernorm = RMSNorm(
            config.hidden_size, eps=config.rms_norm_eps
        )
        self.mlp: nn.Module
        if spec.is_moe:
            self.mlp = MistralMoE(
                vllm_config,
                config,
                prefix=f"{prefix}.mlp",
                reduce_results=not self.defer_allreduce,
            )
        else:
            self.mlp = MistralMLP(
                config.hidden_size,
                config.intermediate_size,
                quant_config,
                reduce_results=not self.defer_allreduce,
                prefix=f"{prefix}.mlp",
            )

    def residual_norm(
        self, norm: RMSNorm, hidden_states: torch.Tensor, residual: torch.Tensor
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Add a sublayer output to the residual stream and normalize; with
        `defer_allreduce` the output is a TP-partial sum reduced here."""
        if self.defer_allreduce:
            return fused_allreduce_rms_norm(hidden_states, residual, norm)
        return norm(hidden_states, residual)

    def input_norm(
        self, hidden_states: torch.Tensor, residual: torch.Tensor | None
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Fold the previous layer's output into the residual stream and
        normalize; the returned residual is the stream entering this layer."""
        if residual is None:
            return self.input_layernorm(hidden_states), hidden_states
        return self.residual_norm(self.input_layernorm, hidden_states, residual)

    def forward(
        self,
        positions: torch.Tensor,
        hidden_states: torch.Tensor,
        residual: torch.Tensor,
        log_positions: torch.Tensor,
        rotary_emb: MistralRotaryEmbedding,
    ) -> tuple[torch.Tensor, torch.Tensor]:
        """Run the layer on the outputs of `input_norm`."""
        hidden_states = self.self_attn(
            positions, hidden_states, log_positions, rotary_emb
        )
        hidden_states, residual = self.residual_norm(
            self.post_attention_layernorm, hidden_states, residual
        )
        return self.mlp(hidden_states), residual

forward(positions, hidden_states, residual, log_positions, rotary_emb)

Run the layer on the outputs of input_norm.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def forward(
    self,
    positions: torch.Tensor,
    hidden_states: torch.Tensor,
    residual: torch.Tensor,
    log_positions: torch.Tensor,
    rotary_emb: MistralRotaryEmbedding,
) -> tuple[torch.Tensor, torch.Tensor]:
    """Run the layer on the outputs of `input_norm`."""
    hidden_states = self.self_attn(
        positions, hidden_states, log_positions, rotary_emb
    )
    hidden_states, residual = self.residual_norm(
        self.post_attention_layernorm, hidden_states, residual
    )
    return self.mlp(hidden_states), residual

input_norm(hidden_states, residual)

Fold the previous layer's output into the residual stream and normalize; the returned residual is the stream entering this layer.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def input_norm(
    self, hidden_states: torch.Tensor, residual: torch.Tensor | None
) -> tuple[torch.Tensor, torch.Tensor]:
    """Fold the previous layer's output into the residual stream and
    normalize; the returned residual is the stream entering this layer."""
    if residual is None:
        return self.input_layernorm(hidden_states), hidden_states
    return self.residual_norm(self.input_layernorm, hidden_states, residual)

residual_norm(norm, hidden_states, residual)

Add a sublayer output to the residual stream and normalize; with defer_allreduce the output is a TP-partial sum reduced here.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def residual_norm(
    self, norm: RMSNorm, hidden_states: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
    """Add a sublayer output to the residual stream and normalize; with
    `defer_allreduce` the output is a TP-partial sum reduced here."""
    if self.defer_allreduce:
        return fused_allreduce_rms_norm(hidden_states, residual, norm)
    return norm(hidden_states, residual)

_check_pp_stage(specs, start, end)

A pipeline stage must own the KV cache and the top-k indices its layers read: KV-sharing sources and the indexer a shared layer reuses stay local.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def _check_pp_stage(specs: list[MistralLayerSpec], start: int, end: int) -> None:
    """A pipeline stage must own the KV cache and the top-k indices its layers
    read: KV-sharing sources and the indexer a shared layer reuses stay local."""
    for spec in specs[start:end]:
        assert start <= spec.kv_source_layer < end, (
            f"Layer {spec.layer_idx} shares the KV cache of layer "
            f"{spec.kv_source_layer}, outside its pipeline stage [{start}, {end}); "
            "set VLLM_PP_LAYER_PARTITION so they share a stage"
        )
    first = specs[start] if start < end else None
    assert first is None or not first.use_sparse or first.has_indexer, (
        f"Pipeline stage [{start}, {end}) starts at layer {start}, which reuses "
        "the top-k indices of a previous layer; set VLLM_PP_LAYER_PARTITION so "
        "every stage starts with an indexer layer"
    )

_load_weights(model, mistral_model, weights)

Load a Mistral consolidated checkpoint into model.

model holds mistral_model as model.model and the LM head as model.lm_head.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def _load_weights(
    model: nn.Module,
    mistral_model: MistralLarge4Model,
    weights: Iterable[tuple[str, torch.Tensor]],
) -> set[str]:
    """Load a Mistral consolidated checkpoint into `model`.

    `model` holds `mistral_model` as `model.model` and the LM head as
    `model.lm_head`.
    """
    config = mistral_model.config
    params = dict(model.named_parameters())
    expert_mapping = fused_moe_make_expert_params_mapping(
        model,
        ckpt_gate_proj_name="w1",
        ckpt_down_proj_name="w2",
        ckpt_up_proj_name="w3",
        num_experts=getattr(config, "n_routed_experts", None) or 0,
    )
    loaded: set[str] = set()
    lerope_alpha: torch.Tensor | None = None

    for ckpt_name, weight in weights:
        if ckpt_name == "lerope.alpha":
            lerope_alpha = weight
            continue
        if ckpt_name.startswith("mtp_"):
            # Speculative-decoding heads are loaded by their draft model.
            continue
        # Layers beyond the configured depth (e.g. a truncated debug model).
        layer = re.match(r"layers\.(\d+)\.", ckpt_name)
        if layer is not None and int(layer.group(1)) >= config.num_hidden_layers:
            continue
        name = _to_vllm_weight_name(ckpt_name)
        if is_pp_missing_parameter(name, model):
            continue

        if ".mlp.experts." in name:
            is_expert_weight = False
            for param_name, weight_name, expert_id, shard_id in expert_mapping:
                if weight_name not in name:
                    continue
                is_expert_weight = True
                mapped = name.replace(weight_name, param_name)
                param = params[mapped]
                loader = cast(Callable[..., bool], param.weight_loader)
                if loader(
                    param,
                    weight,
                    mapped,
                    shard_id=shard_id,
                    expert_id=expert_id,
                    return_success=True,
                ):
                    loaded.add(mapped)
                    break
            else:
                # A known expert weight that is not local to this EP rank is skipped.
                if not is_expert_weight:
                    raise ValueError(f"Unexpected expert weight: {ckpt_name}")
            continue

        for param_part, ckpt_part, stacked_shard_id in _STACKED_PARAMS:
            if ckpt_part not in name:
                continue
            name = name.replace(ckpt_part, param_part)
            param = params[name]
            if stacked_shard_id < 0:
                param.weight_loader(param, weight)
            else:
                param.weight_loader(param, weight, stacked_shard_id)
            break
        else:
            param = params[name]
            getattr(param, "weight_loader", default_weight_loader)(param, weight)
        loaded.add(name)

    # The default loader skips this check for quantized models.
    if missing := sorted(set(params) - loaded):
        raise ValueError(f"Parameters not loaded from the checkpoint: {missing}")

    if getattr(config, "lerope", False):
        assert lerope_alpha is not None, "Checkpoint is missing lerope.alpha"
        mistral_model.rotary_emb.set_lerope_alpha(lerope_alpha)
    return loaded

_map_quant_config_layers(quant_config, prefix)

Point the layer names of a Mistral-format quantization config at this model's modules.

Such a config lists checkpoint modules (e.g. ModelOpt quantized_layers keys such as layers.1.attention.wq or layers.1.experts); a fused module resolves through the checkpoint modules stacked into it.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def _map_quant_config_layers(quant_config: QuantizationConfig, prefix: str) -> None:
    """Point the layer names of a Mistral-format quantization config at this
    model's modules.

    Such a config lists checkpoint modules (e.g. ModelOpt `quantized_layers`
    keys such as `layers.1.attention.wq` or `layers.1.experts`); a fused module
    resolves through the checkpoint modules stacked into it.
    """
    quant_config.apply_vllm_mapper(
        WeightsMapper(
            orig_to_new_regex={
                re.compile(pattern): repl for pattern, repl in _CHECKPOINT_NAME_MAP
            },
            orig_to_new_prefix={
                "model.": f"{maybe_prefix(prefix, 'model')}.",
                "lm_head": maybe_prefix(prefix, "lm_head"),
            },
        )
    )
    stacked: dict[str, list[str]] = {}
    for param_part, ckpt_part, _ in _STACKED_PARAMS:
        stacked.setdefault(param_part.strip("."), []).append(ckpt_part.strip("."))
    quant_config.packed_modules_mapping = {
        **quant_config.packed_modules_mapping,
        **stacked,
    }

defers_allreduce(vllm_config)

Whether sublayers leave their outputs TP-partial and fuse the all-reduce into the next residual RMSNorm.

Source code in vllm/models/mistral/mistral_large4/nvidia/model.py
def defers_allreduce(vllm_config: VllmConfig) -> bool:
    """Whether sublayers leave their outputs TP-partial and fuse the all-reduce
    into the next residual RMSNorm."""
    parallel_config = vllm_config.parallel_config
    return bool(
        vllm_config.compilation_config.pass_config.fuse_allreduce_rms
        and vllm_config.model_config.dtype == torch.bfloat16
        and not parallel_config.enable_expert_parallel
        and not parallel_config.use_sequence_parallel_moe
    )