feat(v1): carry media placeholder positions on message nodes

tests/unit/test_graph.py

@@ -657,18 +657,20 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
                 completion_ids=[4],
                 message_spans=[(0, 2)],
                 is_content=[False, True, False],
-                mm_placeholders=[(1, 2)],
-                mm_token_type_id_map={9: 1},
+                mm_placeholders={"image": [(1, 2)]},
+                renderer_mm_placeholders={"image": [(1, 1)]},
             ),
         )
     )
 
     assert list(trace.branches[0].token_ids) == [1, 9, 9, 3, 4]
     assert list(trace.branches[0].mm_token_type_ids) == [0, 1, 1, 0, 0]
     assert list(trace.nodes[0].token_ids) == [1, 9, 9]
+    assert trace.nodes[0].mm_placeholders == {"image": [(1, 2)]}
     assert list(trace.nodes[0].is_content) == [False, True, True]
     turn = graph.prepare_turn(trace, [user, assistant, vf.UserMessage(content="next")])
     assert turn.previous_renderer_token_ids() == ([1, 9, 3], [4])
+    assert turn.previous_mm_placeholders() == {"image": [(1, 1)]}
 
     with pytest.raises(ValueError, match="exactly extend"):
         turn.commit(
@@ -703,8 +705,8 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
                 completion_ids=[8],
                 message_spans=[None, None, (4, 6)],
                 is_content=[False, True, False, True, False, True, False],
-                mm_placeholders=[(1, 2)],
-                mm_token_type_id_map={9: 1},
+                mm_placeholders={"image": [(1, 2)]},
+                renderer_mm_placeholders={"image": [(1, 1)]},
             ),
         )
     )
@@ -716,6 +718,60 @@ def test_expanded_prompt_is_attributed_while_bridge_uses_logical_tokens():
     assert list(tool_node.is_content) == [False, True]
 
 
+def test_placeholder_token_ids_outside_placed_media_are_text():
+    """A sampled placeholder id (9) and one in tool text stay text; only the image the
+    renderer placed is a placeholder, even where the text one comes first."""
+    trace = vf.Trace(
+        agent=vf.AgentInfo(config=vf.AgentConfig()),
+        task=vf.TraceTask(type="Task", data=vf.TaskData(idx=0, prompt="x")),
+    )
+    user = vf.UserMessage(content="image")
+    assistant = vf.AssistantMessage(content="sampled pad")
+    graph.prepare_turn(trace, [user]).commit(
+        vf.Response(
+            id="a",
+            created=0,
+            model="t",
+            message=assistant,
+            finish_reason="stop",
+            tokens=TurnTokens(
+                prompt_ids=[1, 9, 9, 3],
+                renderer_prompt_ids=[1, 9, 3],
+                completion_ids=[9, 4],
+                message_spans=[(0, 2)],
+                mm_placeholders={"image": [(1, 2)]},
+                renderer_mm_placeholders={"image": [(1, 1)]},
+            ),
+        )
+    )
+    assert list(trace.branches[0].mm_token_type_ids) == [0, 1, 1, 0, 0, 0]
+
+    # The tool result has a literal placeholder (9) before its image (logical 7).
+    tool = vf.ToolMessage(content="pad text, then image", tool_call_id="call_0")
+    graph.prepare_turn(trace, [user, assistant, tool]).commit(
+        vf.Response(
+            id="b",
+            created=0,
+            model="t",
+            message=vf.AssistantMessage(content="done"),
+            finish_reason="stop",
+            tokens=TurnTokens(
+                prompt_ids=[1, 9, 9, 3, 9, 4, 5, 9, 6, 7, 7, 7, 2],
+                renderer_prompt_ids=[1, 9, 3, 9, 4, 5, 9, 6, 7, 2],
+                bridged=True,
+                completion_ids=[8],
+                message_spans=[None, None, (5, 9)],
+                mm_placeholders={"image": [(1, 2), (9, 3)]},
+                renderer_mm_placeholders={"image": [(1, 1), (8, 1)]},
+            ),
+        )
+    )
+    [branch] = trace.branches
+    assert branch.mm_placeholders == {"image": [(1, 2), (9, 3)]}
+    assert list(branch.mm_token_type_ids) == [0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 0]
+    assert trace.nodes[-2].mm_placeholders == {"image": [(3, 3)]}
+
+
 def test_prompt_supplied_assistant_messages_are_not_sampled_turns():
     task = vf.TaskData(idx=0, prompt="few-shot")
     trace = vf.Trace(

verifiers/v1/clients/train.py

@@ -12,7 +12,7 @@
 import numpy as np
 from openai import OpenAIError
 from renderers import OverlongPromptError, RenderedTokens, Renderer, RendererConfig
-from renderers.base import ToolCallParseStatus, is_multimodal
+from renderers.base import MultiModalData, PlaceholderRange, ToolCallParseStatus
 
 from verifiers.v1.clients.base import build_async_openai
 from verifiers.v1.clients.client import SESSION_ID_HEADER, Client
@@ -25,6 +25,7 @@
 from verifiers.v1.types import (
     AssistantMessage,
     FinishReason,
+    MMPlaceholders,
     Response,
     SamplingMask,
     Tool,
@@ -106,11 +107,28 @@ def serialize_completion(response: Response, model: str) -> dict:
     }
 
 
+def _placeholders_from(
+    ranges: Mapping[str, Any] | None,
+) -> MMPlaceholders | None:
+    """`{modality: [(offset, length)]}` from renderer `PlaceholderRange`s or vLLM's
+    `{"offset", "length"}` dicts."""
+    if ranges is None:
+        return None
+    return {
+        modality: [
+            (r["offset"], r["length"])
+            if isinstance(r, Mapping)
+            else (r.offset, r.length)
+            for r in modality_ranges
+        ]
+        for modality, modality_ranges in ranges.items()
+    }
+
+
 def response_from_generate(
     result: dict,
     model: str,
     bridged_turn: PendingTurn | None = None,
-    mm_token_type_id_map: dict[int, int] | None = None,
 ) -> Response:
     """Parse a `renderers.client.generate` result dict into a typed `Response`,
     mirroring the chat client's `response_from_wire` (plus the token encoding)."""
@@ -146,15 +164,14 @@ def response_from_generate(
         message_spans = bridged_turn.prompt_message_spans(attribution)
     else:
         message_spans = attribution.message_token_spans()
-    raw_mm_placeholders = result.get("mm_placeholders")
+    # Where the renderer placed media in its logical prompt; vLLM's ranges cover the
+    # expanded prompt it returned (None when nothing was expanded, so the two coincide).
+    mm_data = attribution.multi_modal_data if attribution is not None else None
+    renderer_mm_placeholders = _placeholders_from(
+        mm_data.mm_placeholders if mm_data is not None else None
+    )
     mm_placeholders = (
-        sorted(
-            (placeholder["offset"], placeholder["length"])
-            for ranges in raw_mm_placeholders.values()
-            for placeholder in ranges
-        )
-        if raw_mm_placeholders is not None
-        else None
+        _placeholders_from(result.get("mm_placeholders")) or renderer_mm_placeholders
     )
     return Response(
         id=result.get("request_id", ""),
@@ -188,7 +205,7 @@ def response_from_generate(
             if attribution is not None and attribution.is_content is not None
             else None,
             mm_placeholders=mm_placeholders,
-            mm_token_type_id_map=mm_token_type_id_map,
+            renderer_mm_placeholders=renderer_mm_placeholders,
             routed_experts=result.get("routed_experts"),
             sampling_mask=SamplingMask.from_sampling_mask(mask)
             if (mask := result.get("sampling_mask"))
@@ -410,9 +427,6 @@ async def get_response(
 
         async with pool.acquire() as slot:
             renderer = slot.renderer
-            mm_token_type_id_map = (
-                renderer.mm_token_type_id_map if is_multimodal(renderer) else None
-            )
             has_images = _has_multimodal_content(prompt)
             process_multimodal = not has_images
             if has_images and not getattr(
@@ -428,12 +442,29 @@ async def get_response(
             if previous_ids is not None:
                 previous_prompt_ids, previous_completion_ids = previous_ids
 
+                # Token ids can't tell placed media from a sampled or literal placeholder
+                # token, so the bridge gets the prefix's placeholder positions.
+                previous_placeholders = turn.previous_mm_placeholders()
+                bridge_kwargs = (
+                    {
+                        "previous_multi_modal_data": MultiModalData(
+                            mm_placeholders={
+                                modality: [PlaceholderRange(*r) for r in ranges]
+                                for modality, ranges in previous_placeholders.items()
+                            }
+                        )
+                    }
+                    if previous_placeholders
+                    else {}
+                )
+
                 def bridge():
                     return renderer.bridge_to_next_turn(
                         previous_prompt_ids,
                         previous_completion_ids,
                         wire_tail,
                         tools=wire_tools,
+                        **bridge_kwargs,
                         **render_kwargs,
                     )
 
@@ -483,9 +514,7 @@ def bridge():
                 raise ProviderError(str(e), status_code=400) from e
             except OpenAIError as e:
                 raise model_error(e) from e
-        response = response_from_generate(
-            result, model, bridged_turn, mm_token_type_id_map
-        )
+        response = response_from_generate(result, model, bridged_turn)
         # No provider response to relay (we generated), so serialize one for the program; the
         # interception server hands `Response.raw` back regardless of client.
         response.raw = serialize_completion(response, model)