fix(integrations): openai/openai-agents: convert input message format

sentry_sdk/integrations/openai.py

@@ -5,8 +5,10 @@
 from sentry_sdk import consts
 from sentry_sdk.ai.monitoring import record_token_usage
 from sentry_sdk.ai.utils import (
+    extract_response_output,
     set_data_normalized,
     normalize_message_roles,
+    transform_openai_content_part,
     truncate_and_annotate_messages,
 )
 from sentry_sdk.consts import SPANDATA
@@ -203,6 +205,21 @@ def _set_input_data(
         and integration.include_prompts
     ):
         normalized_messages = normalize_message_roles(messages)
+        # Transform content parts to standardized format using OpenAI-specific transformer
+        for message in normalized_messages:
+            if isinstance(message, dict) and "content" in message:
+                content = message["content"]
+                if isinstance(content, (list, tuple)):
+                    transformed = []
+                    for item in content:
+                        if isinstance(item, dict):
+                            result = transform_openai_content_part(item)
+                            # If transformation succeeded, use the result; otherwise keep original
+                            transformed.append(result if result is not None else item)
+                        else:
+                            transformed.append(item)
+                    message["content"] = transformed
+
         scope = sentry_sdk.get_current_scope()
         messages_data = truncate_and_annotate_messages(normalized_messages, span, scope)
         if messages_data is not None:
@@ -265,49 +282,64 @@ def _set_output_data(
 
     if hasattr(response, "choices"):
         if should_send_default_pii() and integration.include_prompts:
-            response_text = [
-                choice.message.model_dump()
-                for choice in response.choices
-                if choice.message is not None
-            ]
+            response_text = []  # type: list[str]
+            tool_calls = []  # type: list[Any]
+
+            for choice in response.choices:
+                if choice.message is None:
+                    continue
+
+                # Extract text content
+                content = getattr(choice.message, "content", None)
+                if content is not None:
+                    response_text.append(content)
+
+                # Extract audio transcript if available
+                audio = getattr(choice.message, "audio", None)
+                if audio is not None:
+                    transcript = getattr(audio, "transcript", None)
+                    if transcript is not None:
+                        response_text.append(transcript)
+
+                # Extract tool calls
+                message_tool_calls = getattr(choice.message, "tool_calls", None)
+                if message_tool_calls is not None:
+                    for tool_call in message_tool_calls:
+                        if hasattr(tool_call, "model_dump"):
+                            tool_calls.append(tool_call.model_dump())
+                        elif hasattr(tool_call, "dict"):
+                            tool_calls.append(tool_call.dict())
+
             if len(response_text) > 0:
                 set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, response_text)
 
+            if len(tool_calls) > 0:
+                set_data_normalized(
+                    span,
+                    SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
+                    tool_calls,
+                    unpack=False,
+                )
+
         _calculate_token_usage(messages, response, span, None, integration.count_tokens)
 
         if finish_span:
             span.__exit__(None, None, None)
 
     elif hasattr(response, "output"):
         if should_send_default_pii() and integration.include_prompts:
-            output_messages: "dict[str, list[Any]]" = {
-                "response": [],
-                "tool": [],
-            }
-
-            for output in response.output:
-                if output.type == "function_call":
-                    output_messages["tool"].append(output.dict())
-                elif output.type == "message":
-                    for output_message in output.content:
-                        try:
-                            output_messages["response"].append(output_message.text)
-                        except AttributeError:
-                            # Unknown output message type, just return the json
-                            output_messages["response"].append(output_message.dict())
-
-            if len(output_messages["tool"]) > 0:
+            response_texts, tool_calls = extract_response_output(response.output)
+
+            if len(tool_calls) > 0:
                 set_data_normalized(
                     span,
                     SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS,
-                    output_messages["tool"],
+                    tool_calls,
                     unpack=False,
                 )
 
-            if len(output_messages["response"]) > 0:
-                set_data_normalized(
-                    span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]
-                )
+            if len(response_texts) > 0:
+                set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, response_texts)
 
         _calculate_token_usage(messages, response, span, None, integration.count_tokens)
 

sentry_sdk/integrations/openai_agents/spans/invoke_agent.py

@@ -3,14 +3,19 @@
     get_start_span_function,
     set_data_normalized,
     normalize_message_roles,
+    normalize_message_role,
     truncate_and_annotate_messages,
 )
 from sentry_sdk.consts import OP, SPANDATA
 from sentry_sdk.scope import should_send_default_pii
 from sentry_sdk.utils import safe_serialize
 
 from ..consts import SPAN_ORIGIN
-from ..utils import _set_agent_data, _set_usage_data
+from ..utils import (
+    _set_agent_data,
+    _set_usage_data,
+    _transform_openai_agents_message_content,
+)
 
 from typing import TYPE_CHECKING
 
@@ -49,17 +54,40 @@ def invoke_agent_span(
 
         original_input = kwargs.get("original_input")
         if original_input is not None:
-            message = (
-                original_input
-                if isinstance(original_input, str)
-                else safe_serialize(original_input)
-            )
-            messages.append(
-                {
-                    "content": [{"text": message, "type": "text"}],
-                    "role": "user",
-                }
-            )
+            if isinstance(original_input, str):
+                # String input: wrap in text block
+                messages.append(
+                    {
+                        "content": [{"text": original_input, "type": "text"}],
+                        "role": "user",
+                    }
+                )
+            elif isinstance(original_input, list) and len(original_input) > 0:
+                # Check if list contains message objects (with type="message")
+                # or content parts (input_text, input_image, etc.)
+                first_item = original_input[0]
+                if isinstance(first_item, dict) and first_item.get("type") == "message":
+                    # List of message objects - process each individually
+                    for msg in original_input:
+                        if isinstance(msg, dict) and msg.get("type") == "message":
+                            role = normalize_message_role(msg.get("role", "user"))
+                            content = msg.get("content")
+                            transformed = _transform_openai_agents_message_content(
+                                content
+                            )
+                            if isinstance(transformed, str):
+                                transformed = [{"text": transformed, "type": "text"}]
+                            elif not isinstance(transformed, list):
+                                transformed = [
+                                    {"text": str(transformed), "type": "text"}
+                                ]
+                            messages.append({"content": transformed, "role": role})
+                else:
+                    # List of content parts - transform and wrap as user message
+                    content = _transform_openai_agents_message_content(original_input)
+                    if not isinstance(content, list):
+                        content = [{"text": str(content), "type": "text"}]
+                    messages.append({"content": content, "role": "user"})
 
         if len(messages) > 0:
             normalized_messages = normalize_message_roles(messages)

sentry_sdk/integrations/openai_agents/utils.py

@@ -1,7 +1,9 @@
 import sentry_sdk
 from sentry_sdk.ai.utils import (
     GEN_AI_ALLOWED_MESSAGE_ROLES,
+    extract_response_output,
     normalize_message_roles,
+    parse_data_uri,
     set_data_normalized,
     normalize_message_role,
     truncate_and_annotate_messages,
@@ -27,6 +29,133 @@
     raise DidNotEnable("OpenAI Agents not installed")
 
 
+def _transform_openai_agents_content_part(
+    content_part: "dict[str, Any]",
+) -> "dict[str, Any]":
+    """
+    Transform an OpenAI Agents content part to Sentry-compatible format.
+
+    Handles multimodal content (images, audio, files) by converting them
+    to the standardized format:
+    - base64 encoded data -> type: "blob"
+    - URL references -> type: "uri"
+    - file_id references -> type: "file"
+    """
+    if not isinstance(content_part, dict):
+        return content_part
+
+    part_type = content_part.get("type")
+
+    # Handle input_text (OpenAI Agents SDK text format) -> normalize to standard text format
+    if part_type == "input_text":
+        return {
+            "type": "text",
+            "text": content_part.get("text", ""),
+        }
+
+    # Handle image_url (OpenAI vision format) and input_image (OpenAI Agents SDK format)
+    if part_type in ("image_url", "input_image"):
+        # Get URL from either format
+        if part_type == "image_url":
+            image_url = content_part.get("image_url") or {}
+            url = (
+                image_url.get("url", "")
+                if isinstance(image_url, dict)
+                else str(image_url)
+            )
+        else:
+            # input_image format has image_url directly
+            url = content_part.get("image_url") or ""
+
+        if url.startswith("data:"):
+            try:
+                mime_type, content = parse_data_uri(url)
+                return {
+                    "type": "blob",
+                    "modality": "image",
+                    "mime_type": mime_type,
+                    "content": content,
+                }
+            except ValueError:
+                # If parsing fails, return as URI
+                return {
+                    "type": "uri",
+                    "modality": "image",
+                    "mime_type": "",
+                    "uri": url,
+                }
+        else:
+            return {
+                "type": "uri",
+                "modality": "image",
+                "mime_type": "",
+                "uri": url,
+            }
+
+    # Handle input_audio (OpenAI audio input format)
+    if part_type == "input_audio":
+        input_audio = content_part.get("input_audio") or {}
+        if isinstance(input_audio, dict):
+            audio_format = input_audio.get("format", "")
+            mime_type = f"audio/{audio_format}" if audio_format else ""
+            return {
+                "type": "blob",
+                "modality": "audio",
+                "mime_type": mime_type,
+                "content": input_audio.get("data", ""),
+            }
+        else:
+            return content_part
+
+    # Handle image_file (Assistants API file-based images)
+    if part_type == "image_file":
+        image_file = content_part.get("image_file") or {}
+        if isinstance(image_file, dict):
+            return {
+                "type": "file",
+                "modality": "image",
+                "mime_type": "",
+                "file_id": image_file.get("file_id", ""),
+            }
+        else:
+            return content_part
+
+    # Handle file (document attachments)
+    if part_type == "file":
+        file_data = content_part.get("file") or {}
+        if isinstance(file_data, dict):
+            return {
+                "type": "file",
+                "modality": "document",
+                "mime_type": "",
+                "file_id": file_data.get("file_id", ""),
+            }
+        else:
+            return content_part
+
+    return content_part
+
+
+def _transform_openai_agents_message_content(content: "Any") -> "Any":
+    """
+    Transform OpenAI Agents message content, handling both string content and
+    list of content parts.
+    """
+    if isinstance(content, str):
+        return content
+
+    if isinstance(content, (list, tuple)):
+        transformed = []
+        for item in content:
+            if isinstance(item, dict):
+                transformed.append(_transform_openai_agents_content_part(item))
+            else:
+                transformed.append(item)
+        return transformed
+
+    return content
+
+
 def _capture_exception(exc: "Any") -> None:
     set_span_errored()
 
@@ -128,13 +257,15 @@ def _set_input_data(
         if "role" in message:
             normalized_role = normalize_message_role(message.get("role"))
             content = message.get("content")
+            # Transform content to handle multimodal data (images, audio, files)
+            transformed_content = _transform_openai_agents_message_content(content)
             request_messages.append(
                 {
                     "role": normalized_role,
                     "content": (
-                        [{"type": "text", "text": content}]
-                        if isinstance(content, str)
-                        else content
+                        [{"type": "text", "text": transformed_content}]
+                        if isinstance(transformed_content, str)
+                        else transformed_content
                     ),
                 }
             )
@@ -170,31 +301,13 @@ def _set_output_data(span: "sentry_sdk.tracing.Span", result: "Any") -> None:
     if not should_send_default_pii():
         return
 
-    output_messages: "dict[str, list[Any]]" = {
-        "response": [],
-        "tool": [],
-    }
+    response_texts, tool_calls = extract_response_output(result.output)
 
-    for output in result.output:
-        if output.type == "function_call":
-            output_messages["tool"].append(output.dict())
-        elif output.type == "message":
-            for output_message in output.content:
-                try:
-                    output_messages["response"].append(output_message.text)
-                except AttributeError:
-                    # Unknown output message type, just return the json
-                    output_messages["response"].append(output_message.dict())
-
-    if len(output_messages["tool"]) > 0:
-        span.set_data(
-            SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(output_messages["tool"])
-        )
+    if len(tool_calls) > 0:
+        span.set_data(SPANDATA.GEN_AI_RESPONSE_TOOL_CALLS, safe_serialize(tool_calls))
 
-    if len(output_messages["response"]) > 0:
-        set_data_normalized(
-            span, SPANDATA.GEN_AI_RESPONSE_TEXT, output_messages["response"]
-        )
+    if len(response_texts) > 0:
+        set_data_normalized(span, SPANDATA.GEN_AI_RESPONSE_TEXT, response_texts)
 
 
 def _create_mcp_execute_tool_spans(