Recover detection JSON after reasoning token limit

app.py

@@ -291,6 +291,33 @@ def _client() -> OpenAI:
     return OpenAI(api_key=api_key, base_url=BASE_URL, timeout=180.0, max_retries=2)
 
 
+def _stream_terminal_problem(event: Any) -> tuple[str, str] | None:
+    event_type = getattr(event, "type", "")
+    response = getattr(event, "response", None)
+    if event_type == "response.incomplete":
+        details = getattr(response, "incomplete_details", None)
+        reason = str(getattr(details, "reason", None) or "unknown_reason")
+        return reason, f"Qwen returned an incomplete response ({reason})."
+    if event_type == "response.failed":
+        error = getattr(response, "error", None) or getattr(event, "error", None)
+        return "response_failed", f"Qwen reported a failed response: {error or 'unknown error'}"
+    return None
+
+
+def _detection_stream(
+    input_messages: list[dict[str, Any]],
+    enable_thinking: bool,
+    max_output_tokens: int,
+) -> Any:
+    return _client().responses.create(
+        model=MODEL,
+        input=input_messages,
+        max_output_tokens=int(max_output_tokens),
+        stream=True,
+        extra_body={"enable_thinking": bool(enable_thinking)},
+    )
+
+
 def _check_rate_limit(session_hash: str | None) -> None:
     session = session_hash or "anonymous"
     now = time.monotonic()
@@ -446,6 +473,7 @@ def chat(
 
     reasoning_parts: list[str] = []
     answer_parts: list[str] = []
+    terminal_problem: tuple[str, str] | None = None
 
     try:
         stream = _client().responses.create(
@@ -457,6 +485,10 @@ def chat(
         )
         for event in stream:
             event_type = getattr(event, "type", "")
+            problem = _stream_terminal_problem(event)
+            if problem:
+                terminal_problem = problem
+                continue
             if event_type == "response.reasoning_summary_text.delta":
                 reasoning_parts.append(str(getattr(event, "delta", "")))
             elif event_type == "response.output_text.delta":
@@ -485,6 +517,25 @@ def chat(
         yield cleared_message, visible_history, conversation
         return
 
+    if terminal_problem:
+        reason, problem = terminal_problem
+        reasoning = "".join(reasoning_parts).strip()
+        partial = "".join(answer_parts).strip()
+        guidance = (
+            " Increase **Maximum output tokens** or shorten the request."
+            if reason == "max_output_tokens"
+            else ""
+        )
+        visible_history[-1] = {
+            "role": "assistant",
+            "content": _streamed_assistant_message(
+                reasoning,
+                (partial + "\n\n" if partial else "") + f"⚠️ {problem}{guidance}",
+            ),
+        }
+        yield cleared_message, visible_history, conversation
+        return
+
     reasoning = "".join(reasoning_parts).strip()
     answer = "".join(answer_parts).strip() or "The model returned no text response."
     visible_history[-1] = {
@@ -762,29 +813,30 @@ Use XYXY coordinates normalized to integers from 0 to 1000. Do not return the al
 For reference, the prompt boxes in normalized XYXY coordinates are: {_normalized_boxes(original_path, annotations)}
 """.strip()
 
+    input_messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": prompt},
+                {"type": "input_image", "image_url": _pil_data_url(annotated_image)},
+            ],
+        }
+    ]
     reasoning_parts: list[str] = []
     answer_parts: list[str] = []
+    terminal_problem: tuple[str, str] | None = None
+    retried_without_thinking = False
     waiting_reasoning = "▌" if enable_thinking else "_Thinking mode is disabled._"
     yield waiting_reasoning, "**Connecting to Qwen…**\n\n```json\n▌\n```", None
 
     try:
-        stream = _client().responses.create(
-            model=MODEL,
-            input=[
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "input_text", "text": prompt},
-                        {"type": "input_image", "image_url": _pil_data_url(annotated_image)},
-                    ],
-                }
-            ],
-            max_output_tokens=int(max_output_tokens),
-            stream=True,
-            extra_body={"enable_thinking": bool(enable_thinking)},
-        )
+        stream = _detection_stream(input_messages, enable_thinking, max_output_tokens)
         for event in stream:
             event_type = getattr(event, "type", "")
+            problem = _stream_terminal_problem(event)
+            if problem:
+                terminal_problem = problem
+                continue
             if event_type == "response.reasoning_summary_text.delta":
                 reasoning_parts.append(str(getattr(event, "delta", "")))
             elif event_type == "response.output_text.delta":
@@ -810,8 +862,54 @@ For reference, the prompt boxes in normalized XYXY coordinates are: {_normalized
         yield reasoning, error_markdown, None
         return
 
+    should_retry = enable_thinking and (
+        (terminal_problem and terminal_problem[0] == "max_output_tokens")
+        or not "".join(answer_parts).strip()
+    )
+    if should_retry:
+        retried_without_thinking = True
+        terminal_problem = None
+        answer_parts.clear()
+        reasoning = "".join(reasoning_parts).strip() or waiting_reasoning
+        yield (
+            reasoning,
+            "**Reasoning reached the output limit. Retrying the final JSON without thinking…**\n\n"
+            "```json\n▌\n```",
+            None,
+        )
+        try:
+            retry_stream = _detection_stream(input_messages, False, max_output_tokens)
+            for event in retry_stream:
+                event_type = getattr(event, "type", "")
+                problem = _stream_terminal_problem(event)
+                if problem:
+                    terminal_problem = problem
+                    continue
+                if event_type != "response.output_text.delta":
+                    continue
+                answer_parts.append(str(getattr(event, "delta", "")))
+                answer = "".join(answer_parts)
+                yield (
+                    reasoning,
+                    "**Recovering final JSON…**\n\n" f"```json\n{answer or '▌'}\n```",
+                    None,
+                )
+        except Exception as error:
+            terminal_problem = ("retry_failed", f"The automatic JSON retry failed: {error}")
+
+    if terminal_problem or not "".join(answer_parts).strip():
+        problem = terminal_problem[1] if terminal_problem else "Qwen returned no final JSON."
+        partial_answer = "".join(answer_parts).strip()
+        details = (
+            f"**⚠️ {problem}**\n\n"
+            + (f"```json\n{partial_answer}\n```\n\n" if partial_answer else "")
+            + "No detection result was fabricated. Try a larger token budget or turn off Thinking mode."
+        )
+        yield "".join(reasoning_parts).strip() or waiting_reasoning, details, None
+        return
+
     reasoning = "".join(reasoning_parts).strip()
-    answer = "".join(answer_parts).strip() or '{"objects": []}'
+    answer = "".join(answer_parts).strip()
     with Image.open(original_path) as image:
         width, height = image.size
 
@@ -826,7 +924,12 @@ For reference, the prompt boxes in normalized XYXY coordinates are: {_normalized
         detections = []
         note = "Qwen's response could not be parsed into boxes; the raw response is shown below."
 
-    details = f"**{note}**\n\n### Model response\n\n```json\n{answer}\n```"
+    recovery_note = (
+        "\n\n_Automatic recovery: the reasoning pass exhausted its token budget, so the final JSON was retried without thinking._"
+        if retried_without_thinking
+        else ""
+    )
+    details = f"**{note}**{recovery_note}\n\n### Model response\n\n```json\n{answer}\n```"
     reasoning_display = reasoning or "_The model returned no reasoning summary._"
     yield reasoning_display, details, (original_path, prompt_annotations + detections)
 
@@ -862,29 +965,30 @@ Return only valid JSON in this exact shape:
 Use tight XYXY bounding boxes with coordinates normalized to integers from 0 to 1000. Return one entry per visible object instance that satisfies the instruction. Use concise labels that distinguish requested categories or attributes. Do not invent objects. If there are no matches, return {{"objects": []}}. Return at most 100 objects.
 """.strip()
 
+    input_messages = [
+        {
+            "role": "user",
+            "content": [
+                {"type": "input_text", "text": prompt},
+                {"type": "input_image", "image_url": _image_data_url(image_path)},
+            ],
+        }
+    ]
     reasoning_parts: list[str] = []
     answer_parts: list[str] = []
+    terminal_problem: tuple[str, str] | None = None
+    retried_without_thinking = False
     waiting_reasoning = "▌" if enable_thinking else "_Thinking mode is disabled._"
     yield waiting_reasoning, "**Connecting to Qwen…**\n\n```json\n▌\n```", None
 
     try:
-        stream = _client().responses.create(
-            model=MODEL,
-            input=[
-                {
-                    "role": "user",
-                    "content": [
-                        {"type": "input_text", "text": prompt},
-                        {"type": "input_image", "image_url": _image_data_url(image_path)},
-                    ],
-                }
-            ],
-            max_output_tokens=int(max_output_tokens),
-            stream=True,
-            extra_body={"enable_thinking": bool(enable_thinking)},
-        )
+        stream = _detection_stream(input_messages, enable_thinking, max_output_tokens)
         for event in stream:
             event_type = getattr(event, "type", "")
+            problem = _stream_terminal_problem(event)
+            if problem:
+                terminal_problem = problem
+                continue
             if event_type == "response.reasoning_summary_text.delta":
                 reasoning_parts.append(str(getattr(event, "delta", "")))
             elif event_type == "response.output_text.delta":
@@ -911,8 +1015,54 @@ Use tight XYXY bounding boxes with coordinates normalized to integers from 0 to
         yield reasoning, details, None
         return
 
+    should_retry = enable_thinking and (
+        (terminal_problem and terminal_problem[0] == "max_output_tokens")
+        or not "".join(answer_parts).strip()
+    )
+    if should_retry:
+        retried_without_thinking = True
+        terminal_problem = None
+        answer_parts.clear()
+        reasoning = "".join(reasoning_parts).strip() or waiting_reasoning
+        yield (
+            reasoning,
+            "**Reasoning reached the output limit. Retrying the final JSON without thinking…**\n\n"
+            "```json\n▌\n```",
+            None,
+        )
+        try:
+            retry_stream = _detection_stream(input_messages, False, max_output_tokens)
+            for event in retry_stream:
+                event_type = getattr(event, "type", "")
+                problem = _stream_terminal_problem(event)
+                if problem:
+                    terminal_problem = problem
+                    continue
+                if event_type != "response.output_text.delta":
+                    continue
+                answer_parts.append(str(getattr(event, "delta", "")))
+                answer = "".join(answer_parts)
+                yield (
+                    reasoning,
+                    "**Recovering final JSON…**\n\n" f"```json\n{answer or '▌'}\n```",
+                    None,
+                )
+        except Exception as error:
+            terminal_problem = ("retry_failed", f"The automatic JSON retry failed: {error}")
+
+    if terminal_problem or not "".join(answer_parts).strip():
+        problem = terminal_problem[1] if terminal_problem else "Qwen returned no final JSON."
+        partial_answer = "".join(answer_parts).strip()
+        details = (
+            f"**⚠️ {problem}**\n\n"
+            + (f"```json\n{partial_answer}\n```\n\n" if partial_answer else "")
+            + "No detection result was fabricated. Try a larger token budget or turn off Thinking mode."
+        )
+        yield "".join(reasoning_parts).strip() or waiting_reasoning, details, None
+        return
+
     reasoning = "".join(reasoning_parts).strip()
-    answer = "".join(answer_parts).strip() or '{"objects": []}'
+    answer = "".join(answer_parts).strip()
     with Image.open(image_path) as image:
         width, height = image.size
 
@@ -925,7 +1075,12 @@ Use tight XYXY bounding boxes with coordinates normalized to integers from 0 to
         detections = []
         note = "Qwen's response could not be parsed into boxes; the raw response is shown below."
 
-    details = f"**{note}**\n\n### Model response\n\n```json\n{answer}\n```"
+    recovery_note = (
+        "\n\n_Automatic recovery: the reasoning pass exhausted its token budget, so the final JSON was retried without thinking._"
+        if retried_without_thinking
+        else ""
+    )
+    details = f"**{note}**{recovery_note}\n\n### Model response\n\n```json\n{answer}\n```"
     reasoning_display = reasoning or "_The model returned no reasoning summary._"
     yield reasoning_display, details, (image_path, detections)
 
@@ -1042,7 +1197,7 @@ with gr.Blocks(title="Free Qwen 3.8 Max") as demo:
                         max_tokens = gr.Slider(
                             minimum=256,
                             maximum=8_192,
-                            value=4_096,
+                            value=8_192,
                             step=256,
                             label="Maximum output tokens",
                         )

tests/test_app.py

@@ -241,3 +241,101 @@ def test_object_detection_streams_and_renders_labeled_boxes(
     assert reasoning == "Scanning scene."
     assert "Rendered 1 detected object" in details
     assert result[1] == [((20, 20, 120, 80), "blue car")]
+
+
+def test_object_detection_retries_without_thinking_after_token_limit(
+    tmp_path, monkeypatch
+) -> None:
+    image_path = tmp_path / "scene.png"
+    Image.new("RGB", (200, 100), "white").save(image_path)
+    calls = []
+    streams = [
+        [
+            SimpleNamespace(
+                type="response.reasoning_summary_text.delta",
+                delta="I found the requested car.",
+            ),
+            SimpleNamespace(
+                type="response.incomplete",
+                response=SimpleNamespace(
+                    incomplete_details=SimpleNamespace(reason="max_output_tokens")
+                ),
+            ),
+        ],
+        [
+            SimpleNamespace(
+                type="response.output_text.delta",
+                delta='{"objects":[{"label":"car","box_2d":[100,200,600,800]}]}',
+            ),
+            SimpleNamespace(type="response.completed", response=SimpleNamespace()),
+        ],
+    ]
+
+    def create(**kwargs):
+        calls.append(kwargs)
+        return iter(streams[len(calls) - 1])
+
+    monkeypatch.setattr(
+        app,
+        "_client",
+        lambda: SimpleNamespace(responses=SimpleNamespace(create=create)),
+    )
+
+    updates = list(
+        app.detect_objects(
+            str(image_path),
+            "Detect the car.",
+            False,
+            True,
+            256,
+            SimpleNamespace(session_hash="object-retry-test"),
+        )
+    )
+
+    assert len(calls) == 2
+    assert calls[0]["extra_body"] == {"enable_thinking": True}
+    assert calls[1]["extra_body"] == {"enable_thinking": False}
+    assert any("Retrying the final JSON" in details for _, details, _ in updates)
+    reasoning, details, result = updates[-1]
+    assert reasoning == "I found the requested car."
+    assert "Automatic recovery" in details
+    assert result[1] == [((20, 20, 120, 80), "car")]
+
+
+def test_detection_does_not_fabricate_empty_json_after_incomplete_response(
+    tmp_path, monkeypatch
+) -> None:
+    image_path = tmp_path / "scene.png"
+    Image.new("RGB", (200, 100), "white").save(image_path)
+    events = [
+        SimpleNamespace(
+            type="response.incomplete",
+            response=SimpleNamespace(
+                incomplete_details=SimpleNamespace(reason="max_output_tokens")
+            ),
+        )
+    ]
+    monkeypatch.setattr(
+        app,
+        "_client",
+        lambda: SimpleNamespace(
+            responses=SimpleNamespace(create=lambda **kwargs: iter(events))
+        ),
+    )
+
+    updates = list(
+        app.detect_objects(
+            str(image_path),
+            "Detect the car.",
+            False,
+            False,
+            256,
+            SimpleNamespace(session_hash="object-incomplete-test"),
+        )
+    )
+
+    _, details, result = updates[-1]
+    assert "max_output_tokens" in details
+    assert "No detection result was fabricated" in details
+    assert '{"objects": []}' not in details
+    assert result is None