[FIX] [litellm_wrapper] omit empty text block for image-only vision calls

The vision message builders always prepended {"type": "text", "text": task},
so an image-only call (empty or None task) produced a user message containing
an empty text content block. Anthropic rejects empty text blocks; the wrapper
already drops empty system blocks in _prepare_messages for the same reason,
and the vision user-text block was the ungated case in that family.

Add a _vision_content(task, image_block) helper that includes the text block
only when task is a non-empty string, and route the anthropic (direct-URL and
base64) and openai vision builders through it. Behavior is unchanged when a
task is present.

Author: ebarkhordar

swarms/utils/litellm_wrapper.py

@@ -864,6 +864,33 @@ def _apply_cache_request_params(
             )
             completion_params["extra_headers"] = headers
 
+    @staticmethod
+    def _vision_content(task: str, image_block: dict) -> list:
+        """
+        Build the content list for a vision user message.
+
+        The text block is included only when ``task`` is a non-empty string.
+        Anthropic rejects content blocks whose text is empty with
+        "text content blocks must be non-empty", so an image-only call
+        (empty or None task) must omit the text block rather than send
+        ``{"type": "text", "text": ""}``. This mirrors the empty-system-block
+        normalization already done in ``_prepare_messages``.
+
+        Args:
+            task (str): The text task/prompt. Omitted from the content when
+                empty, whitespace-only, or None.
+            image_block (dict): The image content block to include.
+
+        Returns:
+            list: ``[text_block, image_block]`` when task has text, else
+                ``[image_block]``.
+        """
+        content = []
+        if isinstance(task, str) and task.strip():
+            content.append({"type": "text", "text": task})
+        content.append(image_block)
+        return content
+
     def anthropic_vision_processing(
         self, task: str, image: str, messages: list
     ) -> list:
@@ -898,15 +925,15 @@ def anthropic_vision_processing(
             messages.append(
                 {
                     "role": "user",
-                    "content": [
-                        {"type": "text", "text": task},
+                    "content": self._vision_content(
+                        task,
                         {
                             "type": "image_url",
                             "image_url": {
                                 "url": image,
                             },
                         },
-                    ],
+                    ),
                 }
             )
         else:
@@ -936,16 +963,16 @@ def anthropic_vision_processing(
             messages.append(
                 {
                     "role": "user",
-                    "content": [
-                        {"type": "text", "text": task},
+                    "content": self._vision_content(
+                        task,
                         {
                             "type": "image_url",
                             "image_url": {
                                 "url": image_url,
                                 "format": mime_type,
                             },
                         },
-                    ],
+                    ),
                 }
             )
 
@@ -1033,10 +1060,7 @@ def openai_vision_processing(
         messages.append(
             {
                 "role": "user",
-                "content": [
-                    {"type": "text", "text": task},
-                    vision_message,
-                ],
+                "content": self._vision_content(task, vision_message),
             }
         )
 

tests/utils/test_litellm_vision_empty_text.py

@@ -0,0 +1,108 @@
+"""Regression tests for vision message construction in ``LiteLLM``.
+
+An image-only vision call (empty, whitespace-only, or ``None`` task) must not
+emit an empty text content block. Anthropic rejects such blocks with
+"text content blocks must be non-empty"; the wrapper already guards this for
+system blocks in ``_prepare_messages`` but historically emitted
+``{"type": "text", "text": ""}`` for the vision user message.
+
+These assertions are on the message structure the wrapper builds (they do not
+make a live provider call).
+"""
+
+import base64
+
+import pytest
+
+from swarms.utils.litellm_wrapper import LiteLLM
+
+# Smallest valid 1x1 PNG, enough for get_image_base64() to encode from a file
+# without any network access.
+_PNG_1x1 = base64.b64decode(
+    "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR4"
+    "2mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg=="
+)
+
+
+@pytest.fixture
+def png_path(tmp_path):
+    path = tmp_path / "pixel.png"
+    path.write_bytes(_PNG_1x1)
+    return str(path)
+
+
+def _user_content(messages):
+    user = next(m for m in messages if m["role"] == "user")
+    return user["content"]
+
+
+def _text_blocks(content):
+    return [
+        b
+        for b in content
+        if isinstance(b, dict) and b.get("type") == "text"
+    ]
+
+
+def _image_blocks(content):
+    return [
+        b
+        for b in content
+        if isinstance(b, dict) and b.get("type") == "image_url"
+    ]
+
+
+@pytest.mark.parametrize("task", ["", "   ", None])
+def test_anthropic_vision_omits_empty_text_block(task, png_path):
+    llm = LiteLLM(model_name="claude-3-5-sonnet-20241022")
+    messages = llm.anthropic_vision_processing(task, png_path, [])
+    content = _user_content(messages)
+    assert _text_blocks(content) == []
+    assert len(_image_blocks(content)) == 1
+
+
+@pytest.mark.parametrize("task", ["", "   ", None])
+def test_openai_vision_omits_empty_text_block(task, png_path):
+    llm = LiteLLM(model_name="gpt-4o")
+    messages = llm.openai_vision_processing(task, png_path, [])
+    content = _user_content(messages)
+    assert _text_blocks(content) == []
+    assert len(_image_blocks(content)) == 1
+
+
+def test_anthropic_direct_url_omits_empty_text_block(monkeypatch):
+    # Force the direct-URL branch so the URL is embedded, not fetched.
+    llm = LiteLLM(model_name="claude-3-5-sonnet-20241022")
+    monkeypatch.setattr(
+        llm, "_should_use_direct_url", lambda image: True
+    )
+    url = "https://example.com/pixel.png"
+    messages = llm.anthropic_vision_processing("", url, [])
+    content = _user_content(messages)
+    assert _text_blocks(content) == []
+    assert _image_blocks(content)[0]["image_url"]["url"] == url
+
+
+def test_anthropic_vision_keeps_nonempty_text_block(png_path):
+    # Over-deletion guard: a real task must still carry its text block.
+    llm = LiteLLM(model_name="claude-3-5-sonnet-20241022")
+    messages = llm.anthropic_vision_processing(
+        "Describe this image", png_path, []
+    )
+    content = _user_content(messages)
+    blocks = _text_blocks(content)
+    assert len(blocks) == 1
+    assert blocks[0]["text"] == "Describe this image"
+    assert len(_image_blocks(content)) == 1
+
+
+def test_openai_vision_keeps_nonempty_text_block(png_path):
+    llm = LiteLLM(model_name="gpt-4o")
+    messages = llm.openai_vision_processing(
+        "Describe this image", png_path, []
+    )
+    content = _user_content(messages)
+    blocks = _text_blocks(content)
+    assert len(blocks) == 1
+    assert blocks[0]["text"] == "Describe this image"
+    assert len(_image_blocks(content)) == 1