From 5e9f48e567f6a45398a4f158922893f7f511bae3 Mon Sep 17 00:00:00 2001 From: faresobeid <111092724+faresobeid@users.noreply.github.com> Date: Fri, 30 Oct 2026 15:15:53 +0000 Subject: [PATCH] feat(v1): carry media placeholder positions on message nodes MessageNode.mm_placeholders records, per node, where the renderer placed each media item in the (vLLM-expanded) token stream. Branch.mm_placeholders and Branch.mm_token_type_ids derive from them, so a placeholder token id the policy sampled, or one in tool/user text, stays text. Prompt attribution is projected with the renderer's logical placeholder ranges instead of token ids, and bridges pass the prefix's placeholder positions to the renderer. The renderer token map (Trace.mm_token_type_id_map) is removed. --- tests/unit/test_graph.py | 64 +++++++++++++++-- verifiers/v1/clients/train.py | 63 +++++++++++----- verifiers/v1/graph.py | 131 ++++++++++++++++++++++++++++------ verifiers/v1/serve/delta.py | 1 - verifiers/v1/trace.py | 47 ++++++------ verifiers/v1/types.py | 15 ++-- 6 files changed, 253 insertions(+), 68 deletions(-) diff --git a/tests/unit/test_graph.py b/tests/unit/test_graph.py index 4df44699ed..d6454d04e0 100644 --- a/tests/unit/test_graph.py +++ b/tests/unit/test_graph.py @@ -657,8 +657,8 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens(): completion_ids=[4], message_spans=[(0, 2)], is_content=[False, True, False], - mm_placeholders=[(1, 2)], - mm_token_type_id_map={9: 1}, + mm_placeholders={"image": [(1, 2)]}, + renderer_mm_placeholders={"image": [(1, 1)]}, ), ) ) @@ -666,9 +666,11 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens(): assert list(trace.branches[0].token_ids) == [1, 9, 9, 3, 4] assert list(trace.branches[0].mm_token_type_ids) == [0, 1, 1, 0, 0] assert list(trace.nodes[0].token_ids) == [1, 9, 9] + assert trace.nodes[0].mm_placeholders == {"image": [(1, 2)]} assert list(trace.nodes[0].is_content) == [False, True, True] turn = graph.prepare_turn(trace, [user, assistant, vf.UserMessage(content="next")]) assert turn.previous_renderer_token_ids() == ([1, 9, 3], [4]) + assert turn.previous_mm_placeholders() == {"image": [(1, 1)]} with pytest.raises(ValueError, match="exactly extend"): turn.commit( @@ -703,8 +705,8 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens(): completion_ids=[8], message_spans=[None, None, (4, 6)], is_content=[False, True, False, True, False, True, False], - mm_placeholders=[(1, 2)], - mm_token_type_id_map={9: 1}, + mm_placeholders={"image": [(1, 2)]}, + renderer_mm_placeholders={"image": [(1, 1)]}, ), ) ) @@ -716,6 +718,60 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens(): assert list(tool_node.is_content) == [False, True] +def test_placeholder_token_ids_outside_placed_media_are_text(): + """A sampled placeholder id (9) and one in tool text stay text; only the image the + renderer placed is a placeholder, even where the text one comes first.""" + trace = vf.Trace( + agent=vf.AgentInfo(config=vf.AgentConfig()), + task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")), + ) + user = vf.UserMessage(content="image") + assistant = vf.AssistantMessage(content="sampled pad") + graph.prepare_turn(trace, [user]).commit( + vf.Response( + id="a", + created=0, + model="t", + message=assistant, + finish_reason="stop", + tokens=TurnTokens( + prompt_ids=[1, 9, 9, 3], + renderer_prompt_ids=[1, 9, 3], + completion_ids=[9, 4], + message_spans=[(0, 2)], + mm_placeholders={"image": [(1, 2)]}, + renderer_mm_placeholders={"image": [(1, 1)]}, + ), + ) + ) + assert list(trace.branches[0].mm_token_type_ids) == [0, 1, 1, 0, 0, 0] + + # The tool result has a literal placeholder (9) before its image (logical 7). + tool = vf.ToolMessage(content="pad text, then image", tool_call_id="call_0") + graph.prepare_turn(trace, [user, assistant, tool]).commit( + vf.Response( + id="b", + created=0, + model="t", + message=vf.AssistantMessage(content="done"), + finish_reason="stop", + tokens=TurnTokens( + prompt_ids=[1, 9, 9, 3, 9, 4, 5, 9, 6, 7, 7, 7, 2], + renderer_prompt_ids=[1, 9, 3, 9, 4, 5, 9, 6, 7, 2], + bridged=True, + completion_ids=[8], + message_spans=[None, None, (5, 9)], + mm_placeholders={"image": [(1, 2), (9, 3)]}, + renderer_mm_placeholders={"image": [(1, 1), (8, 1)]}, + ), + ) + ) + [branch] = trace.branches + assert branch.mm_placeholders == {"image": [(1, 2), (9, 3)]} + assert list(branch.mm_token_type_ids) == [0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0] + assert trace.nodes[-2].mm_placeholders == {"image": [(3, 3)]} + + def test_prompt_supplied_assistant_messages_are_not_sampled_turns(): task = vf.TaskData(idx=0, prompt="few-shot") trace = vf.Trace( diff --git a/verifiers/v1/clients/train.py b/verifiers/v1/clients/train.py index 67ddb99b29..884fb6aec2 100644 --- a/verifiers/v1/clients/train.py +++ b/verifiers/v1/clients/train.py @@ -12,7 +12,7 @@ import numpy as np from openai import OpenAIError from renderers import OverlongPromptError, RenderedTokens, Renderer, RendererConfig -from renderers.base import ToolCallParseStatus, is_multimodal +from renderers.base import MultiModalData, PlaceholderRange, ToolCallParseStatus from verifiers.v1.clients.base import build_async_openai from verifiers.v1.clients.client import SESSION_ID_HEADER, Client @@ -25,6 +25,7 @@ from verifiers.v1.types import ( AssistantMessage, FinishReason, + MMPlaceholders, Response, SamplingMask, Tool, @@ -106,11 +107,28 @@ def serialize_completion(response: Response, model: str) -> dict: } +def _placeholders_from( + ranges: Mapping[str, Any] | None, +) -> MMPlaceholders | None: + """`{modality: [(offset, length)]}` from renderer `PlaceholderRange`s or vLLM's + `{"offset", "length"}` dicts.""" + if ranges is None: + return None + return { + modality: [ + (r["offset"], r["length"]) + if isinstance(r, Mapping) + else (r.offset, r.length) + for r in modality_ranges + ] + for modality, modality_ranges in ranges.items() + } + + def response_from_generate( result: dict, model: str, bridged_turn: PendingTurn | None = None, - mm_token_type_id_map: dict[int, int] | None = None, ) -> Response: """Parse a `renderers.client.generate` result dict into a typed `Response`, mirroring the chat client's `response_from_wire` (plus the token encoding).""" @@ -146,15 +164,14 @@ def response_from_generate( message_spans = bridged_turn.prompt_message_spans(attribution) else: message_spans = attribution.message_token_spans() - raw_mm_placeholders = result.get("mm_placeholders") + # Where the renderer placed media in its logical prompt; vLLM's ranges cover the + # expanded prompt it returned (None when nothing was expanded, so the two coincide). + mm_data = attribution.multi_modal_data if attribution is not None else None + renderer_mm_placeholders = _placeholders_from( + mm_data.mm_placeholders if mm_data is not None else None + ) mm_placeholders = ( - sorted( - (placeholder["offset"], placeholder["length"]) - for ranges in raw_mm_placeholders.values() - for placeholder in ranges - ) - if raw_mm_placeholders is not None - else None + _placeholders_from(result.get("mm_placeholders")) or renderer_mm_placeholders ) return Response( id=result.get("request_id", ""), @@ -188,7 +205,7 @@ def response_from_generate( if attribution is not None and attribution.is_content is not None else None, mm_placeholders=mm_placeholders, - mm_token_type_id_map=mm_token_type_id_map, + renderer_mm_placeholders=renderer_mm_placeholders, routed_experts=result.get("routed_experts"), sampling_mask=SamplingMask.from_sampling_mask(mask) if (mask := result.get("sampling_mask")) @@ -410,9 +427,6 @@ async def get_response( async with pool.acquire() as slot: renderer = slot.renderer - mm_token_type_id_map = ( - renderer.mm_token_type_id_map if is_multimodal(renderer) else None - ) has_images = _has_multimodal_content(prompt) process_multimodal = not has_images if has_images and not getattr( @@ -428,12 +442,29 @@ async def get_response( if previous_ids is not None: previous_prompt_ids, previous_completion_ids = previous_ids + # Token ids can't tell placed media from a sampled or literal placeholder + # token, so the bridge gets the prefix's placeholder positions. + previous_placeholders = turn.previous_mm_placeholders() + bridge_kwargs = ( + { + "previous_multi_modal_data": MultiModalData( + mm_placeholders={ + modality: [PlaceholderRange(*r) for r in ranges] + for modality, ranges in previous_placeholders.items() + } + ) + } + if previous_placeholders + else {} + ) + def bridge(): return renderer.bridge_to_next_turn( previous_prompt_ids, previous_completion_ids, wire_tail, tools=wire_tools, + **bridge_kwargs, **render_kwargs, ) @@ -483,9 +514,7 @@ def bridge(): raise ProviderError(str(e), status_code=400) from e except OpenAIError as e: raise model_error(e) from e - response = response_from_generate( - result, model, bridged_turn, mm_token_type_id_map - ) + response = response_from_generate(result, model, bridged_turn) # No provider response to relay (we generated), so serialize one for the program; the # interception server hands `Response.raw` back regardless of client. response.raw = serialize_completion(response, model) diff --git a/verifiers/v1/graph.py b/verifiers/v1/graph.py index 15db0be3aa..1fe92c6ba3 100644 --- a/verifiers/v1/graph.py +++ b/verifiers/v1/graph.py @@ -48,6 +48,7 @@ FloatStream, IntStream, Message, + MMPlaceholders, Response, SamplingMask, TextContentPart, @@ -165,6 +166,12 @@ class MessageNode(BaseModel): renderer_token_ids: IntStream | None = Field(default=None, exclude=True) """Logical renderer tokens retained only while extending a live rollout. None means they are identical to `token_ids`; an empty array is a real empty slice.""" + mm_placeholders: MMPlaceholders = Field( + default_factory=dict, exclude_if=lambda placeholders: not placeholders + ) + """Media placeholder ranges per modality, `(offset, length)` into this node's `token_ids`: + where the renderer placed each media item, as the engine expanded it. A token with a + placeholder id outside these ranges (sampled, or in text) is plain text.""" mask: BoolStream = Field(default_factory=empty_stream(np.bool_)) """Per-token, parallel to `token_ids`: True for trainable, model-sampled tokens (only an @@ -228,6 +235,23 @@ def logical_ids(self) -> np.ndarray: else self.renderer_token_ids ) + @property + def logical_mm_placeholders(self) -> MMPlaceholders: + """`mm_placeholders` over `logical_ids`: each expanded range is one logical token.""" + if self.renderer_token_ids is None: + return self.mm_placeholders + flat = sorted( + (offset, length, modality) + for modality, ranges in self.mm_placeholders.items() + for offset, length in ranges + ) + logical: MMPlaceholders = {} + shift = 0 + for offset, length, modality in flat: + logical.setdefault(modality, []).append((offset - shift, 1)) + shift += length - 1 + return logical + def record_streams( self, float_decimals: int | None, exclude_none: bool ) -> dict[str, Any]: @@ -627,6 +651,19 @@ def previous_renderer_token_ids(self) -> tuple[list[int], list[int]] | None: return None return ids[:-num_sampled].tolist(), ids[-num_sampled:].tolist() + def previous_mm_placeholders(self) -> MMPlaceholders: + """Media placeholder ranges over the bridge anchor's logical renderer stream.""" + placeholders: MMPlaceholders = {} + offset = 0 + for nid in self.prefix_node_ids: + node = self.trace.nodes[nid] + for modality, ranges in node.logical_mm_placeholders.items(): + placeholders.setdefault(modality, []).extend( + (offset + start, length) for start, length in ranges + ) + offset += len(node.logical_ids) + return placeholders + def prompt_message_spans( self, tail_attribution: RenderedTokens ) -> list[tuple[int, int] | None]: @@ -819,6 +856,48 @@ def _attribute_routed_experts( off = end +def _attribute_mm_placeholders( + trace: Trace, + prefix_node_ids: list[int], + new_node_ids: list[int], + placeholders: MMPlaceholders | None, +) -> None: + """Hand each node created this turn the placeholder ranges inside its tokens. The new + nodes tile the prompt after the reused prefix, which must already hold exactly the + ranges that fall inside it.""" + flat = _flat_placeholders(placeholders) + stored: list[tuple[int, int, str]] = [] + off = 0 + for nid in prefix_node_ids: + node = trace.nodes[nid] + stored.extend( + (off + offset, length, modality) + for modality, ranges in node.mm_placeholders.items() + for offset, length in ranges + ) + off += len(node.token_ids) + if [p for p in flat if p[0] < off] != sorted(stored): + raise ValueError( + f"multimodal placeholders {flat} do not match the stored rollout prefix " + f"{sorted(stored)}" + ) + remaining = [p for p in flat if p[0] >= off] + for nid in new_node_ids: + node = trace.nodes[nid] + end = off + len(node.token_ids) + while remaining and remaining[0][0] < end: + offset, length, modality = remaining.pop(0) + if offset + length > end: + raise ValueError( + f"multimodal placeholder ({offset}, {length}) crosses a message " + f"boundary at token {end}" + ) + node.mm_placeholders.setdefault(modality, []).append((offset - off, length)) + off = end + if remaining: + raise ValueError(f"multimodal placeholders {remaining} lie past the prompt") + + def _attribute_sampling_mask( trace: Trace, assistant_id: int, payload: SamplingMask | None ) -> None: @@ -833,39 +912,51 @@ def _attribute_sampling_mask( node.sampling_mask = payload +def _flat_placeholders( + placeholders: MMPlaceholders | None, +) -> list[tuple[int, int, str]]: + """`(offset, length, modality)` across modalities, in prompt order.""" + return sorted( + (offset, length, modality) + for modality, ranges in (placeholders or {}).items() + for offset, length in ranges + ) + + def _project_prompt_attribution( renderer_prompt_ids: np.ndarray, prompt_ids: np.ndarray, - mm_token_type_id_map: dict[int, int], - mm_placeholders: list[tuple[int, int]] | None, + renderer_mm_placeholders: MMPlaceholders | None, + mm_placeholders: MMPlaceholders | None, message_spans: list[tuple[int, int] | None] | None, is_content: np.ndarray | None, ) -> tuple[list[tuple[int, int] | None] | None, np.ndarray | None]: - """Project logical attribution using vLLM's multimodal placeholder ranges.""" + """Project logical attribution onto the expanded prompt: each logical placeholder the + renderer placed covers its whole vLLM placeholder range; every other token covers one.""" if renderer_prompt_ids is prompt_ids or same_tokens( renderer_prompt_ids, prompt_ids ): return message_spans, is_content - if not mm_token_type_id_map or mm_placeholders is None: + logical = _flat_placeholders(renderer_mm_placeholders) + expanded = _flat_placeholders(mm_placeholders) + if not expanded or [m for *_, m in logical] != [m for *_, m in expanded]: raise ValueError( - "cannot align renderer and vLLM prompt tokens without multimodal placeholders" + f"renderer placed {len(logical)} media placeholders but vLLM expanded " + f"{len(expanded)}: {logical} vs {expanded}" ) - - # Each logical token covers one prompt token, or one placeholder's whole range. - is_mm = np.isin(renderer_prompt_ids, list(mm_token_type_id_map)) - placeholders = np.asarray(mm_placeholders, dtype=np.int64).reshape(-1, 2) - if np.count_nonzero(is_mm) > len(placeholders): + if any(length != 1 for _, length, _ in logical) or any( + length < 1 for _, length, _ in expanded + ): raise ValueError( "vLLM multimodal placeholders do not align with renderer prompt" ) - if np.count_nonzero(is_mm) < len(placeholders): - raise ValueError("renderer prompt does not align with vLLM prompt tokens") + + is_mm = np.zeros(len(renderer_prompt_ids), dtype=np.bool_) + is_mm[[offset for offset, _, _ in logical]] = True lengths = np.ones(len(renderer_prompt_ids), dtype=np.int64) - lengths[is_mm] = placeholders[:, 1] + lengths[is_mm] = [length for _, length, _ in expanded] offsets = np.concatenate([[0], np.cumsum(lengths)]) - if (placeholders[:, 1] < 1).any() or not np.array_equal( - placeholders[:, 0], offsets[:-1][is_mm] - ): + if not np.array_equal([offset for offset, _, _ in expanded], offsets[:-1][is_mm]): raise ValueError( "vLLM multimodal placeholders do not align with renderer prompt" ) @@ -917,9 +1008,6 @@ def _commit_turn(turn: PendingTurn, response: Response) -> int: trace = turn.trace prompt = turn.prompt tokens = response.tokens - # Constant per renderer, so re-stamping every turn is idempotent. - if tokens is not None and tokens.mm_token_type_id_map: - trace.mm_token_type_id_map = tokens.mm_token_type_id_map prompt_ids = np.asarray(tokens.prompt_ids if tokens else (), dtype=np.int32) # Logical renderer ids differ from `prompt_ids` only after multimodal expansion; otherwise # the nodes keep none and read `token_ids` (`MessageNode.logical_ids`). @@ -969,7 +1057,7 @@ def _commit_turn(turn: PendingTurn, response: Response) -> int: spans, is_content = _project_prompt_attribution( renderer_prompt_ids, prompt_ids, - trace.mm_token_type_id_map, + tokens.renderer_mm_placeholders if tokens else None, tokens.mm_placeholders if tokens else None, renderer_spans, renderer_is_content, @@ -1118,6 +1206,9 @@ def _commit_turn(turn: PendingTurn, response: Response) -> int: _index_node(trace, assistant_id) new_node_ids.append(assistant_id) + if tokens is not None: + _attribute_mm_placeholders(trace, prefix, new_node_ids, tokens.mm_placeholders) + # Attribute this turn's expert-routing array onto the nodes created this turn (new input # nodes in creation order, then the assistant node), each getting the routing for its tokens. # The prefix goes in too, so the position the previous turn could only pad can be corrected. diff --git a/verifiers/v1/serve/delta.py b/verifiers/v1/serve/delta.py index b5d764ea00..7c2120ab89 100644 --- a/verifiers/v1/serve/delta.py +++ b/verifiers/v1/serve/delta.py @@ -55,7 +55,6 @@ SCALAR_FIELDS = ( "agent", "tools", - "mm_token_type_id_map", "rewards", "metrics", "info", diff --git a/verifiers/v1/trace.py b/verifiers/v1/trace.py index c0cbd6bbaa..1de04c1244 100644 --- a/verifiers/v1/trace.py +++ b/verifiers/v1/trace.py @@ -10,7 +10,7 @@ import numpy as np import orjson -from pydantic import BaseModel, Field, PrivateAttr, computed_field, field_serializer +from pydantic import BaseModel, Field, PrivateAttr, computed_field from typing_extensions import TypeVar if TYPE_CHECKING: @@ -30,6 +30,7 @@ FinishReason, Message, Messages, + MMPlaceholders, SamplingMask, Tool, ToolMessage, @@ -41,6 +42,9 @@ TRACE_VERSION = 1 """Current version of the trace schema.""" +MM_TOKEN_TYPES = {"image": 1, "video": 2} +"""`Branch.mm_token_type_ids` marker per placeholder modality; 0 is text.""" + EXCLUDE_FIELDS: dict = { "nodes": { @@ -244,8 +248,6 @@ class Branch(BaseModel): calls: list[ModelCall] = Field(default_factory=list) trainable: bool = True """Whether this physical path contributes a training sample.""" - mm_token_type_id_map: dict[int, int] = Field(default_factory=dict) - """The trace's `mm_token_type_id_map`, carried so `mm_token_type_ids` is self-contained.""" @property def messages(self) -> Messages: @@ -341,16 +343,32 @@ def loss_weights(self, name: str) -> np.ndarray | None: parts.append(node_weights) return _concat(parts, np.float32) + @property + def mm_placeholders(self) -> MMPlaceholders: + """Media placeholder ranges per modality, `(offset, length)` into `token_ids`.""" + placeholders: MMPlaceholders = {} + offset = 0 + for node in self.nodes: + for modality, ranges in node.mm_placeholders.items(): + placeholders.setdefault(modality, []).extend( + (offset + start, length) for start, length in ranges + ) + offset += len(node.token_ids) + return placeholders + @property def mm_token_type_ids(self) -> np.ndarray | None: """Per-token int32 modality markers aligned to `token_ids` (0 = text, 1 = image - placeholder, 2 = video placeholder); None when none are present.""" - if not self.mm_token_type_id_map: + placeholder, 2 = video placeholder) from `mm_placeholders`; None when there are none.""" + placeholders = self.mm_placeholders + if not placeholders: return None - token_ids = self.token_ids - token_types = np.zeros(len(token_ids), dtype=np.int32) - for token, token_type in self.mm_token_type_id_map.items(): - token_types[token_ids == token] = token_type + token_types = np.zeros( + sum(len(n.token_ids) for n in self.nodes), dtype=np.int32 + ) + for modality, ranges in placeholders.items(): + for offset, length in ranges: + token_types[offset : offset + length] = MM_TOKEN_TYPES.get(modality, 0) return token_types if token_types.any() else None @property @@ -450,10 +468,6 @@ class Trace(BaseModel, Generic[DataT, StateT, AgentConfigT]): """The message graph, including physical and semantic parent links.""" calls: list[ModelCall] = Field(default_factory=list) """Every model call; automatically recorded at intercept time + linked into `nodes`.""" - mm_token_type_id_map: dict[int, int] = Field(default_factory=dict) - """Special-token id -> modality marker (1 = image placeholder, 2 = video placeholder) - from the renderer that tokenized this trace, stamped at turn commit. Applied to a - branch's `token_ids` by `Branch.mm_token_type_ids`; empty for text-only renderers.""" request_rewrites: list[InterceptRecord] = Field(default_factory=list) """Request changes made by `@intercept`, in execution order.""" response_rewrites: list[InterceptRecord] = Field(default_factory=list) @@ -524,12 +538,6 @@ def clear_preview(self, key: object) -> None: """The request's previewed messages are committed (or abandoned).""" self._pending.pop(id(key), None) - @field_serializer("mm_token_type_id_map") - def serialize_mm_token_type_id_map(self, mapping: dict[int, int]) -> dict[str, int]: - """The wire unpacks with msgpack's `strict_map_key`, which forbids int map keys — - dump str keys; validation coerces them back to int.""" - return {str(k): v for k, v in mapping.items()} - @property def reward(self) -> float: return sum(r.value for r in self.rewards.values() if r is not None) @@ -612,7 +620,6 @@ def branches(self) -> list[Branch]: not is_compaction_attempt or leaf in accepted_compaction_attempts ), - mm_token_type_id_map=self.mm_token_type_id_map, ) ) return branches diff --git a/verifiers/v1/types.py b/verifiers/v1/types.py index 9d79d837ab..2455b746fb 100644 --- a/verifiers/v1/types.py +++ b/verifiers/v1/types.py @@ -251,6 +251,10 @@ class RoutedExperts(TypedDict): start: int +MMPlaceholders = dict[str, list[tuple[int, int]]] +"""Media placeholder ranges per modality: `(offset, length)` token positions, in order.""" + + class WeightVersionSpan(BaseModel): """Completion tokens `[start, end)` were sampled by the engine weights labelled `version`.""" @@ -356,12 +360,11 @@ class TurnTokens(BaseModel): default=None, exclude=True ) is_content: BoolStream | None = Field(default=None, exclude=True) - # Authoritative effective-prompt ranges returned by vLLM, flattened across modalities. - mm_placeholders: list[tuple[int, int]] | None = Field(default=None, exclude=True) - # Transient carrier (excluded): the renderer's special-token id -> modality marker map, - # stamped onto `Trace.mm_token_type_id_map` by the turn's `commit`. None unless the - # rendering renderer is multimodal. - mm_token_type_id_map: dict[int, int] | None = Field(default=None, exclude=True) + # Transient carriers (excluded): media placeholder ranges per modality, consumed by the + # turn's `commit`. `mm_placeholders` covers `prompt_ids` (vLLM's ranges after expansion); + # `renderer_mm_placeholders` covers `renderer_prompt_ids` (where the renderer placed media). + mm_placeholders: MMPlaceholders | None = Field(default=None, exclude=True) + renderer_mm_placeholders: MMPlaceholders | None = Field(default=None, exclude=True) # Transient carrier (excluded): the MoE expert-routing data from `generate` (expert ids # per token), attributed per node by the turn's `commit` into `MessageNode.routed_experts`, # then dropped. None unless the engine ran with `enable_return_routed_experts`.