Skip to content

vllm.models.mistral.mistral_large4.amd.specialized_decode.runtime.variants

Per-layer-type instances of the kernels package.

The layer geometry (NoPE query width, KV sharing) is a module-level constant of the kernel sources (kernels/variant.py), so each layer type loads the one source tree under its own package name with its own variant module.

Functions:

  • load_variant –

    The kernel package compiled for one ML4 layer type.

load_variant(nope, kv_share)

The kernel package compiled for one ML4 layer type.

Source code in vllm/models/mistral/mistral_large4/amd/specialized_decode/runtime/variants.py
def load_variant(nope: bool, kv_share: bool) -> KernelVariant:
    """The kernel package compiled for one ML4 layer type."""
    assert nope or not kv_share, "KV-sharing layers are NoPE layers"
    name = f"{__package__}._kernels_{'nope' if nope else 'rope'}{'_kvs' if kv_share else ''}"  # noqa: E501
    if name not in sys.modules:
        variant = types.ModuleType(f"{name}.variant")
        variant.NOPE_LAYER = nope  # type: ignore[attr-defined]
        variant.KV_SHARE = kv_share  # type: ignore[attr-defined]
        sys.modules[variant.__name__] = variant
        spec = importlib.util.spec_from_file_location(
            name, _KERNELS / "__init__.py", submodule_search_locations=[str(_KERNELS)]
        )
        assert spec is not None and spec.loader is not None
        package = importlib.util.module_from_spec(spec)
        sys.modules[name] = package
        spec.loader.exec_module(package)
    return KernelVariant(
        importlib.import_module(f"{name}.op"),
        importlib.import_module(f"{name}.layout"),
        importlib.import_module(f"{name}.geometry_config"),
    )