Skip to content

vllm.models.glm5next.common.multimodal

GLM-5.3-Flash vision tower and encoder profiling inputs.

Classes:

Glm5NextProcessingInfo

Bases: Glm4vProcessingInfo

Token-budget geometry for the multimodal checkpoint.

This checkpoint's processor_config.json ships the token-budget style (min_image_tokens / max_image_tokens) with no size key, so the inherited Glm4v size.longest_edge path does not apply.

Source code in vllm/models/glm5next/common/multimodal.py
class Glm5NextProcessingInfo(Glm4vProcessingInfo):
    """Token-budget geometry for the multimodal checkpoint.

    This checkpoint's `processor_config.json` ships the token-budget style
    (`min_image_tokens` / `max_image_tokens`) with no `size` key, so the
    inherited Glm4v `size.longest_edge` path does not apply.
    """

    @staticmethod
    def _alignment_factor(proc) -> int:
        """Spatial factor canvases align to, as `smart_resize` takes it."""
        return proc.patch_size * proc.merge_size * proc.patch_expand_factor

    @classmethod
    def _pixels_per_token(cls, proc) -> int:
        """Pixels one vision token covers, matching `smart_resize`'s own
        `temporal_factor * factor ** 2`."""
        return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2

    def _processor_pixel_budget(self, proc) -> tuple[int, int]:
        """(min_pixels, max_pixels) from the processor's token bounds."""
        if proc.min_image_tokens is None or proc.max_image_tokens is None:
            raise ValueError(
                "min_image_tokens and max_image_tokens must be provided by "
                "processor_config.json (or per-call kwargs)."
            )
        pixels_per_token = self._pixels_per_token(proc)
        return (
            proc.min_image_tokens * pixels_per_token,
            proc.max_image_tokens * pixels_per_token,
        )

    def _get_image_max_pixels(self) -> int:
        mm_kwargs = self.ctx.get_merged_mm_kwargs({})
        if (override := mm_kwargs.get("max_pixels")) is not None:
            return int(override)
        return self._processor_pixel_budget(self.get_hf_processor().image_processor)[1]

    def _get_video_max_pixels(self) -> int:
        mm_kwargs = self.ctx.get_merged_mm_kwargs({})
        if (override := mm_kwargs.get("max_pixels")) is not None:
            return int(override)
        return self._processor_pixel_budget(self.get_hf_processor().video_processor)[1]

    def _get_max_video_frames(self, max_tokens: int) -> int:
        image_processor = self.get_hf_processor().image_processor
        temporal_factor = image_processor.temporal_patch_size
        factor = self._alignment_factor(image_processor)
        max_pixels = self._get_video_max_pixels()
        # Each temporal group needs at least one aligned spatial patch.
        max_frames = max_pixels // factor**2
        max_frames = min(
            max_frames,
            max_tokens * temporal_factor // image_processor.patch_expand_factor**2,
        )
        width, height = self.get_image_size_with_most_features()
        best_frames, best_tokens = temporal_factor, 0
        for num_frames in range(temporal_factor, max_frames + 1, temporal_factor):
            _, num_tokens = self._get_vision_info(
                image_width=width,
                image_height=height,
                num_frames=num_frames,
                max_image_pixels=max_pixels,
            )
            if best_tokens < num_tokens <= max_tokens:
                best_frames, best_tokens = num_frames, num_tokens
                if best_tokens == max_tokens:
                    break
        return best_frames

    def get_image_size_with_most_features(self) -> ImageSize:
        # The inherited square probe strands budget whenever the token
        # ceiling is not a perfect square: with max_image_tokens=8000 the
        # square refits to 2492x2492 (89x89 = 7921 tokens) while a
        # 2240x2800 canvas reaches 80x100 = 8000, so the encoder cache came
        # up short of the processor's own maximum and refused valid images
        # (#59539). Factor the ceiling exactly instead of probing a square.
        vision_config = self.get_hf_config().vision_config
        factor = (
            vision_config.patch_size
            * vision_config.spatial_merge_size
            * self.get_hf_processor().image_processor.patch_expand_factor
        )
        pixels_per_token = vision_config.temporal_patch_size * factor * factor
        max_tokens = max(1, self._get_image_max_pixels() // pixels_per_token)
        short_side = math.isqrt(max_tokens)
        while max_tokens % short_side:
            short_side -= 1
        return ImageSize(
            width=(max_tokens // short_side) * factor,
            height=short_side * factor,
        )

    def _get_vision_info(
        self,
        *,
        image_width: int,
        image_height: int,
        num_frames: int = 16,
        do_resize: bool = True,
        max_image_pixels: int = 28 * 28 * 2 * 30000,
    ) -> tuple[ImageSize, int]:
        """GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.

        The inherited Glm4v path resolves the pixel budget from
        `size.longest_edge` and resizes with GLM-4V's `smart_resize`. This
        checkpoint ships token bounds instead, and the alignment factor carries
        `patch_expand_factor` — resolve both from the processor so profiling
        matches runtime geometry.
        """
        from transformers.models.glm5_next.image_processing_glm5_next import (
            smart_resize,
        )

        vision_config = self.get_hf_config().vision_config
        patch_size = vision_config.patch_size
        merge_size = vision_config.spatial_merge_size
        temporal_patch_size = vision_config.temporal_patch_size

        image_processor = self.get_hf_processor().image_processor
        factor = self._alignment_factor(image_processor)
        temporal_factor = image_processor.temporal_patch_size
        # `smart_resize` denominates its bounds in vision tokens.
        max_image_tokens = max(
            max_image_pixels // self._pixels_per_token(image_processor), 1
        )

        if do_resize:
            t = num_frames if num_frames > temporal_factor else temporal_factor
            resized_height, resized_width = smart_resize(
                num_frames=t,
                height=image_height,
                width=image_width,
                temporal_factor=temporal_factor,
                factor=factor,
                min_pixels=1,
                max_pixels=max_image_tokens,
            )
            preprocessed_size = ImageSize(width=resized_width, height=resized_height)
        else:
            preprocessed_size = ImageSize(width=image_width, height=image_height)

        padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
        grid_t = max(padded_num_frames // temporal_patch_size, 1)
        grid_h = preprocessed_size.height // patch_size
        grid_w = preprocessed_size.width // patch_size

        num_patches = grid_t * grid_h * grid_w
        num_vision_tokens = num_patches // (merge_size**2)

        return preprocessed_size, num_vision_tokens

_alignment_factor(proc) staticmethod

Spatial factor canvases align to, as smart_resize takes it.

Source code in vllm/models/glm5next/common/multimodal.py
@staticmethod
def _alignment_factor(proc) -> int:
    """Spatial factor canvases align to, as `smart_resize` takes it."""
    return proc.patch_size * proc.merge_size * proc.patch_expand_factor

_get_vision_info(*, image_width, image_height, num_frames=16, do_resize=True, max_image_pixels=28 * 28 * 2 * 30000)

GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.

The inherited Glm4v path resolves the pixel budget from size.longest_edge and resizes with GLM-4V's smart_resize. This checkpoint ships token bounds instead, and the alignment factor carries patch_expand_factor — resolve both from the processor so profiling matches runtime geometry.

Source code in vllm/models/glm5next/common/multimodal.py
def _get_vision_info(
    self,
    *,
    image_width: int,
    image_height: int,
    num_frames: int = 16,
    do_resize: bool = True,
    max_image_pixels: int = 28 * 28 * 2 * 30000,
) -> tuple[ImageSize, int]:
    """GLM-5.3-Flash canvas geometry for token budgeting and dummy inputs.

    The inherited Glm4v path resolves the pixel budget from
    `size.longest_edge` and resizes with GLM-4V's `smart_resize`. This
    checkpoint ships token bounds instead, and the alignment factor carries
    `patch_expand_factor` — resolve both from the processor so profiling
    matches runtime geometry.
    """
    from transformers.models.glm5_next.image_processing_glm5_next import (
        smart_resize,
    )

    vision_config = self.get_hf_config().vision_config
    patch_size = vision_config.patch_size
    merge_size = vision_config.spatial_merge_size
    temporal_patch_size = vision_config.temporal_patch_size

    image_processor = self.get_hf_processor().image_processor
    factor = self._alignment_factor(image_processor)
    temporal_factor = image_processor.temporal_patch_size
    # `smart_resize` denominates its bounds in vision tokens.
    max_image_tokens = max(
        max_image_pixels // self._pixels_per_token(image_processor), 1
    )

    if do_resize:
        t = num_frames if num_frames > temporal_factor else temporal_factor
        resized_height, resized_width = smart_resize(
            num_frames=t,
            height=image_height,
            width=image_width,
            temporal_factor=temporal_factor,
            factor=factor,
            min_pixels=1,
            max_pixels=max_image_tokens,
        )
        preprocessed_size = ImageSize(width=resized_width, height=resized_height)
    else:
        preprocessed_size = ImageSize(width=image_width, height=image_height)

    padded_num_frames = num_frames + (-num_frames % temporal_patch_size)
    grid_t = max(padded_num_frames // temporal_patch_size, 1)
    grid_h = preprocessed_size.height // patch_size
    grid_w = preprocessed_size.width // patch_size

    num_patches = grid_t * grid_h * grid_w
    num_vision_tokens = num_patches // (merge_size**2)

    return preprocessed_size, num_vision_tokens

_pixels_per_token(proc) classmethod

Pixels one vision token covers, matching smart_resize's own temporal_factor * factor ** 2.

Source code in vllm/models/glm5next/common/multimodal.py
@classmethod
def _pixels_per_token(cls, proc) -> int:
    """Pixels one vision token covers, matching `smart_resize`'s own
    `temporal_factor * factor ** 2`."""
    return proc.temporal_patch_size * cls._alignment_factor(proc) ** 2

_processor_pixel_budget(proc)

(min_pixels, max_pixels) from the processor's token bounds.

Source code in vllm/models/glm5next/common/multimodal.py
def _processor_pixel_budget(self, proc) -> tuple[int, int]:
    """(min_pixels, max_pixels) from the processor's token bounds."""
    if proc.min_image_tokens is None or proc.max_image_tokens is None:
        raise ValueError(
            "min_image_tokens and max_image_tokens must be provided by "
            "processor_config.json (or per-call kwargs)."
        )
    pixels_per_token = self._pixels_per_token(proc)
    return (
        proc.min_image_tokens * pixels_per_token,
        proc.max_image_tokens * pixels_per_token,
    )

Glm5NextProfilingProcessor

Bases: BaseMultiModalProcessor[Glm5NextProcessingInfo]

Generate synthetic encoder inputs; Rust owns media preprocessing.

Source code in vllm/models/glm5next/common/multimodal.py
class Glm5NextProfilingProcessor(BaseMultiModalProcessor[Glm5NextProcessingInfo]):
    """Generate synthetic encoder inputs; Rust owns media preprocessing."""

    def __init__(
        self,
        info: Glm5NextProcessingInfo,
        dummy_inputs: BaseDummyInputsBuilder[Glm5NextProcessingInfo],
    ) -> None:
        if not envs.VLLM_USE_RUST_FRONTEND:
            raise ValueError(_RUST_FRONTEND_REQUIRED)
        super().__init__(info, dummy_inputs)

    def apply(
        self, inputs: ProcessorInputs, timing_ctx: TimingContext
    ) -> MultiModalInput:
        raise ValueError(_RUST_FRONTEND_REQUIRED)

    def _get_mm_fields_config(self, hf_inputs, hf_processor_mm_kwargs):
        return _create_qwen2vl_field_factory(
            self.info.get_hf_config().vision_config.spatial_merge_size
        )(hf_inputs)

    def _get_prompt_updates(self, mm_items, hf_processor_mm_kwargs, out_mm_kwargs):
        raise ValueError(_RUST_FRONTEND_REQUIRED)

    def get_dummy_mm_inputs(
        self,
        mm_counts: Mapping[str, int],
        *,
        cache: BaseMultiModalProcessorCache | None = None,
        scheduler_config: SchedulerConfig | None = None,
    ) -> MultiModalInput:
        info = self.info
        vision_config = info.get_hf_config().vision_config
        seq_len = info.ctx.model_config.max_model_len
        if scheduler_config is not None and scheduler_config.enable_chunked_prefill:
            seq_len = min(seq_len, scheduler_config.max_num_batched_tokens)

        width, height = info.get_image_size_with_most_features()
        patch_size = vision_config.patch_size
        temporal_patch_size = vision_config.temporal_patch_size
        patch_dim = vision_config.in_channels * temporal_patch_size * patch_size**2
        data = {}
        placeholders = {}
        hashes = {}
        prompt_ids: list[int] = []
        for modality, count in mm_counts.items():
            if count == 0:
                continue
            if modality == "image":
                num_frames = 1
                max_pixels = info._get_image_max_pixels()
                pixel_key = "pixel_values"
            elif modality == "video":
                num_frames = info.get_num_frames_with_most_features(seq_len, count)
                max_pixels = info._get_video_max_pixels()
                pixel_key = "pixel_values_videos"
            else:
                raise ValueError(f"Unsupported modality: {modality}")

            size, num_tokens = info._get_vision_info(
                image_width=width,
                image_height=height,
                num_frames=num_frames,
                max_image_pixels=max_pixels,
            )
            grid = (
                max(1, math.ceil(num_frames / temporal_patch_size)),
                size.height // patch_size,
                size.width // patch_size,
            )
            data[f"{modality}_grid_thw"] = torch.tensor(
                [grid] * count, dtype=torch.long, device="cpu"
            )
            data[pixel_key] = torch.zeros(
                (math.prod(grid) * count, patch_dim), dtype=torch.float32, device="cpu"
            )
            placeholders[modality] = [
                PlaceholderRange(
                    offset=len(prompt_ids) + i * num_tokens, length=num_tokens
                )
                for i in range(count)
            ]
            hashes[modality] = [
                f"glm5next-profiling-{modality}-{i}" for i in range(count)
            ]
            prompt_ids.extend(
                [info.get_hf_config().image_token_id] * num_tokens * count
            )

        prompt_ids.extend([0] * max(0, seq_len - len(prompt_ids)))
        return mm_input(
            prompt_token_ids=prompt_ids,
            mm_kwargs=MultiModalKwargsItems.from_hf_inputs(
                data, self._get_mm_fields_config(data, {})
            ),
            mm_hashes=hashes,
            mm_placeholders=placeholders,
        )

Glm5NextVisionTransformer

Bases: Module

Methods:

Source code in vllm/models/glm5next/common/multimodal.py
class Glm5NextVisionTransformer(nn.Module):
    # Stacked-weight remap for the GLM-OCR/GLM-4V vision checkpoint layout.
    hf_to_vllm_mapper = WeightsMapper(
        orig_to_new_stacked={
            ".attn.q.": (".attn.qkv.", "q"),
            ".attn.k.": (".attn.qkv.", "k"),
            ".attn.v.": (".attn.qkv.", "v"),
            ".gate_proj": (".gate_up_proj", 0),
            ".up_proj": (".gate_up_proj", 1),
        }
    )

    def __init__(
        self,
        text_config,  # noqa: ANN001
        vision_config,
        norm_eps: float = 1e-6,
        quant_config: QuantizationConfig | None = None,
        input_norm: nn.Module | None = None,
        prefix: str = "",
    ) -> None:
        super().__init__()
        use_data_parallel = is_vit_use_data_parallel()
        self.tp_size = (
            1 if use_data_parallel else get_tensor_model_parallel_world_size()
        )

        patch_size = vision_config.patch_size
        temporal_patch_size = vision_config.temporal_patch_size
        in_channels = vision_config.in_channels
        depth = vision_config.depth
        self.hidden_size = vision_config.hidden_size
        self.num_heads = vision_config.num_heads

        self.patch_size = vision_config.patch_size
        self.spatial_merge_size = vision_config.spatial_merge_size
        self.out_hidden_size = vision_config.out_hidden_size

        swiglu_limit = vision_config.swiglu_limit
        if swiglu_limit is None:
            swiglu_limit = text_config.swiglu_limit
        assert swiglu_limit is not None, (
            "GLM-5.3-Flash vision requires swiglu_limit (vision_config or text_config)"
        )

        # Single construction pass — no abs-pos embeddings / post-conv norm (OCR delta).
        self.patch_embed = Glm5NextVisionPatchEmbed(
            patch_size=patch_size,
            temporal_patch_size=temporal_patch_size,
            in_channels=in_channels,
            hidden_size=self.hidden_size,
        )
        self.input_norm = input_norm if input_norm is not None else IdentityInputNorm()

        norm_layer = partial(RMSNorm, eps=norm_eps)
        head_dim = self.hidden_size // self.num_heads
        self.rotary_pos_emb = get_rope(
            head_size=head_dim,
            max_position=text_config.max_position_embeddings,
            is_neox_style=True,
            rope_parameters={"partial_rotary_factor": 0.5},
        )
        self.blocks = nn.ModuleList(
            [
                Glm5NextVisionBlock(
                    dim=self.hidden_size,
                    num_heads=self.num_heads,
                    mlp_hidden_dim=vision_config.intermediate_size,
                    swiglu_limit=swiglu_limit,
                    norm_layer=norm_layer,
                    quant_config=quant_config,
                    prefix=f"{prefix}.blocks.{layer_idx}",
                )
                for layer_idx in range(depth)
            ]
        )
        # GLM-5.3-Flash merger bottleneck width.
        self.merger = Glm5NextPatchMerger(
            d_model=vision_config.out_hidden_size,
            context_dim=vision_config.projection_intermediate_size,
            swiglu_limit=swiglu_limit,
            quant_config=quant_config,
            bias=False,
            prefix=f"{prefix}.merger",
        )

        self.downsample = Conv2dLayer(
            in_channels=vision_config.hidden_size,
            out_channels=vision_config.out_hidden_size,
            kernel_size=vision_config.spatial_merge_size,
            stride=vision_config.spatial_merge_size,
        )
        self.post_layernorm = RMSNorm(
            vision_config.hidden_size, eps=vision_config.rms_norm_eps
        )

        self.attn_backend = get_vit_attn_backend(
            head_size=head_dim,
            dtype=torch.get_default_dtype(),
        )

    @property
    def dtype(self) -> torch.dtype:
        return self.patch_embed.proj.weight.dtype

    @property
    def device(self) -> torch.device:
        return self.patch_embed.proj.weight.device

    def rot_pos_emb(
        self, grid_thw: list[list[int]]
    ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
        pos_ids = []
        for t, h, w in grid_thw:
            hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
            wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
            hpos_ids = (
                hpos_ids.reshape(
                    h // self.spatial_merge_size,
                    self.spatial_merge_size,
                    w // self.spatial_merge_size,
                    self.spatial_merge_size,
                )
                .permute(0, 2, 1, 3)
                .flatten()
            )
            wpos_ids = (
                wpos_ids.reshape(
                    h // self.spatial_merge_size,
                    self.spatial_merge_size,
                    w // self.spatial_merge_size,
                    self.spatial_merge_size,
                )
                .permute(0, 2, 1, 3)
                .flatten()
            )
            pos_ids.append(torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
        pos_ids = torch.cat(pos_ids, dim=0)
        max_grid_size = max(max(h, w) for _, h, w in grid_thw)

        cos, sin = self.rotary_pos_emb.get_cos_sin(max_grid_size)

        pos_ids = async_tensor_h2d(pos_ids, device=cos.device)
        cos_combined = cos[pos_ids].flatten(1)
        sin_combined = sin[pos_ids].flatten(1)
        return cos_combined, sin_combined, pos_ids

    def prepare_encoder_metadata(
        self,
        grid_thw_list: list[list[int]],
        *,
        max_batch_size: int | None = None,
        max_frames_per_batch: int | None = None,
        max_seqlen_override: int | None = None,
        device: torch.device | None = None,
    ) -> dict[str, torch.Tensor | None]:
        """Compute encoder metadata for eager and CUDA graph execution."""
        if device is None:
            device = self.device

        metadata: dict[str, torch.Tensor | None] = {}

        rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
        metadata["rotary_pos_emb_cos"] = rotary_cos
        metadata["rotary_pos_emb_sin"] = rotary_sin

        grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
        patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
        cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
            dtype=np.int32
        )
        cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])

        pad_to = (
            max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
        )
        if pad_to is not None:
            num_seqs = len(cu_seqlens) - 1
            if num_seqs < pad_to:
                cu_seqlens = np.concatenate(
                    [
                        cu_seqlens,
                        np.full(
                            pad_to - num_seqs,
                            cu_seqlens[-1],
                            dtype=np.int32,
                        ),
                    ]
                )

        metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
            self.attn_backend, cu_seqlens, device
        )

        if max_seqlen_override is not None:
            max_seqlen_val = max_seqlen_override
        else:
            max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
                self.attn_backend, cu_seqlens
            )
        metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)

        metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
            self.attn_backend,
            cu_seqlens,
            self.hidden_size,
            self.tp_size,
            device,
        )

        return metadata

    def forward(
        self,
        x: torch.Tensor,
        grid_thw: torch.Tensor | list[list[int]],
        *,
        encoder_metadata: dict[str, torch.Tensor] | None = None,
    ) -> torch.Tensor:
        # patchify
        x = self.input_norm(x.to(device=self.device), self.dtype)
        x = self.patch_embed(x)

        if encoder_metadata is None:
            if not isinstance(grid_thw, list):
                grid_thw = grid_thw.tolist()
            encoder_metadata = self.prepare_encoder_metadata(grid_thw)
        rotary_pos_emb_cos_sin = torch.cat(
            [
                encoder_metadata["rotary_pos_emb_cos"],
                encoder_metadata["rotary_pos_emb_sin"],
            ],
            dim=-1,
        )
        rotary_positions = torch.arange(x.shape[0], device=x.device)

        # transformers
        x = x.unsqueeze(1)
        residual = None
        for blk in self.blocks:
            x, residual = blk(
                x,
                residual,
                cu_seqlens=encoder_metadata["cu_seqlens"],
                rotary_pos_emb_cos_sin=rotary_pos_emb_cos_sin,
                rotary_positions=rotary_positions,
                max_seqlen=encoder_metadata["max_seqlen"],
                sequence_lengths=encoder_metadata["sequence_lengths"],
            )

        # adapter
        x, _ = self.post_layernorm(x, residual)
        x = x.view(-1, self.spatial_merge_size, self.spatial_merge_size, x.shape[-1])
        x = x.permute(0, 3, 1, 2)
        x = self.downsample(x).view(-1, self.out_hidden_size)
        x = self.merger(x)
        return x

    def load_weights(self, weights) -> set[str]:
        loader = AutoWeightsLoader(self)
        return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)

prepare_encoder_metadata(grid_thw_list, *, max_batch_size=None, max_frames_per_batch=None, max_seqlen_override=None, device=None)

Compute encoder metadata for eager and CUDA graph execution.

Source code in vllm/models/glm5next/common/multimodal.py
def prepare_encoder_metadata(
    self,
    grid_thw_list: list[list[int]],
    *,
    max_batch_size: int | None = None,
    max_frames_per_batch: int | None = None,
    max_seqlen_override: int | None = None,
    device: torch.device | None = None,
) -> dict[str, torch.Tensor | None]:
    """Compute encoder metadata for eager and CUDA graph execution."""
    if device is None:
        device = self.device

    metadata: dict[str, torch.Tensor | None] = {}

    rotary_cos, rotary_sin, _ = self.rot_pos_emb(grid_thw_list)
    metadata["rotary_pos_emb_cos"] = rotary_cos
    metadata["rotary_pos_emb_sin"] = rotary_sin

    grid_thw_np = np.array(grid_thw_list, dtype=np.int32)
    patches_per_frame = grid_thw_np[:, 1] * grid_thw_np[:, 2]
    cu_seqlens = np.repeat(patches_per_frame, grid_thw_np[:, 0]).cumsum(
        dtype=np.int32
    )
    cu_seqlens = np.concatenate([np.zeros(1, dtype=np.int32), cu_seqlens])

    pad_to = (
        max_frames_per_batch if max_frames_per_batch is not None else max_batch_size
    )
    if pad_to is not None:
        num_seqs = len(cu_seqlens) - 1
        if num_seqs < pad_to:
            cu_seqlens = np.concatenate(
                [
                    cu_seqlens,
                    np.full(
                        pad_to - num_seqs,
                        cu_seqlens[-1],
                        dtype=np.int32,
                    ),
                ]
            )

    metadata["sequence_lengths"] = MMEncoderAttention.maybe_compute_seq_lens(
        self.attn_backend, cu_seqlens, device
    )

    if max_seqlen_override is not None:
        max_seqlen_val = max_seqlen_override
    else:
        max_seqlen_val = MMEncoderAttention.compute_max_seqlen(
            self.attn_backend, cu_seqlens
        )
    metadata["max_seqlen"] = torch.tensor(max_seqlen_val, dtype=torch.int32)

    metadata["cu_seqlens"] = MMEncoderAttention.maybe_recompute_cu_seqlens(
        self.attn_backend,
        cu_seqlens,
        self.hidden_size,
        self.tp_size,
        device,
    )

    return metadata