diff --git a/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs b/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs index 3d2496b6..468cfca4 100644 --- a/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs +++ b/DevProxy.Abstractions/LanguageModel/OpenAIModels.cs @@ -60,6 +60,22 @@ public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenA return true; } + // Responses API request - has "input" array with objects containing role/content + // Must be checked before embedding request because both have "input" + if (rawRequest.TryGetProperty("input", out var inputProperty) && + inputProperty.ValueKind == JsonValueKind.Array && + inputProperty.GetArrayLength() > 0) + { + var firstItem = inputProperty.EnumerateArray().First(); + if (firstItem.ValueKind == JsonValueKind.Object && + (firstItem.TryGetProperty("role", out _) || firstItem.TryGetProperty("type", out _))) + { + logger.LogDebug("Request is a Responses API request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + } + // Embedding request if (rawRequest.TryGetProperty("input", out _) && rawRequest.TryGetProperty("model", out _) && @@ -112,6 +128,70 @@ public static bool TryGetOpenAIRequest(string content, ILogger logger, out OpenA return false; } } + + /// + /// Tries to parse text generation OpenAI requests (completion, chat completion, and responses API). + /// Used by plugins that only need to handle text-based generation requests, as opposed to + /// embeddings, audio, images, or fine-tuning requests. + /// + public static bool TryGetCompletionLikeRequest(string content, ILogger logger, out OpenAIRequest? request) + { + logger.LogTrace("{Method} called", nameof(TryGetCompletionLikeRequest)); + + request = null; + + if (string.IsNullOrEmpty(content)) + { + logger.LogDebug("Request content is empty or null"); + return false; + } + + try + { + logger.LogDebug("Checking if the request is a completion-like OpenAI request..."); + + var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + + // Completion request + if (rawRequest.TryGetProperty("prompt", out _)) + { + logger.LogDebug("Request is a completion request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Chat completion request + if (rawRequest.TryGetProperty("messages", out _)) + { + logger.LogDebug("Request is a chat completion request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + + // Responses API request - has "input" array with objects containing role/content + if (rawRequest.TryGetProperty("input", out var inputProperty) && + inputProperty.ValueKind == JsonValueKind.Array && + inputProperty.GetArrayLength() > 0) + { + var firstItem = inputProperty.EnumerateArray().First(); + if (firstItem.ValueKind == JsonValueKind.Object && + (firstItem.TryGetProperty("role", out _) || firstItem.TryGetProperty("type", out _))) + { + logger.LogDebug("Request is a Responses API request"); + request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); + return true; + } + } + + logger.LogDebug("Request is not a completion-like OpenAI request."); + return false; + } + catch (JsonException ex) + { + logger.LogDebug(ex, "Failed to deserialize OpenAI request."); + return false; + } + } } public class OpenAIResponse : ILanguageModelCompletionResponse @@ -178,6 +258,23 @@ public class OpenAIResponseUsage public PromptTokenDetails? PromptTokensDetails { get; set; } [JsonPropertyName("total_tokens")] public long TotalTokens { get; set; } + + // Responses API uses different property names (input_tokens, output_tokens) + // These property aliases allow the same class to deserialize both formats. + // When JSON contains "input_tokens", it maps to PromptTokens. + // When JSON contains "output_tokens", it maps to CompletionTokens. + [JsonPropertyName("input_tokens")] + public long InputTokens + { + get => PromptTokens; + set => PromptTokens = value; + } + [JsonPropertyName("output_tokens")] + public long OutputTokens + { + get => CompletionTokens; + set => CompletionTokens = value; + } } public class PromptTokenDetails @@ -409,3 +506,81 @@ public class OpenAIImageData [JsonPropertyName("revised_prompt")] public string? RevisedPrompt { get; set; } } + +#region Responses API + +public class OpenAIResponsesRequest : OpenAIRequest +{ + public IEnumerable? Input { get; set; } + public string? Instructions { get; set; } + [JsonPropertyName("previous_response_id")] + public string? PreviousResponseId { get; set; } + [JsonPropertyName("max_output_tokens")] + public long? MaxOutputTokens { get; set; } +} + +public class OpenAIResponsesInputItem +{ + public string Role { get; set; } = string.Empty; + [JsonConverter(typeof(OpenAIContentPartJsonConverter))] + public object Content { get; set; } = string.Empty; + public string? Type { get; set; } +} + +public class OpenAIResponsesResponse : OpenAIResponse +{ + [JsonPropertyName("created_at")] + public long CreatedAt { get; set; } + public string Status { get; set; } = string.Empty; + public IEnumerable? Output { get; set; } + [JsonPropertyName("previous_response_id")] + public string? PreviousResponseId { get; set; } + + public override string? Response => GetTextFromOutput(); + + private string? GetTextFromOutput() + { + if (Output is null) + { + return null; + } + + var messageOutput = Output.FirstOrDefault(o => + string.Equals(o.Type, "message", StringComparison.OrdinalIgnoreCase)); + if (messageOutput?.Content is null) + { + return null; + } + + var textContent = messageOutput.Content.FirstOrDefault(c => + string.Equals(c.Type, "output_text", StringComparison.OrdinalIgnoreCase)); + return textContent?.Text; + } +} + +public class OpenAIResponsesOutputItem +{ + public string? Type { get; set; } + public string? Id { get; set; } + public string? Role { get; set; } + public IEnumerable? Content { get; set; } + public string? Status { get; set; } +} + +public class OpenAIResponsesOutputContent +{ + public string? Type { get; set; } + public string? Text { get; set; } +} + +public class OpenAIResponsesUsage +{ + [JsonPropertyName("input_tokens")] + public long InputTokens { get; set; } + [JsonPropertyName("output_tokens")] + public long OutputTokens { get; set; } + [JsonPropertyName("total_tokens")] + public long TotalTokens { get; set; } +} + +#endregion diff --git a/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs b/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs index e38e1a29..4d7a205c 100644 --- a/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs +++ b/DevProxy.Plugins/Behavior/LanguageModelFailurePlugin.cs @@ -73,7 +73,7 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo return; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new LoggingContext(e.Session)); return; @@ -116,6 +116,32 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo Logger.LogRequest($"Simulating fault {faultName}", MessageType.Chaos, new LoggingContext(e.Session)); e.Session.SetRequestBodyString(JsonSerializer.Serialize(newRequest, ProxyUtils.JsonSerializerOptions)); } + else if (openAiRequest is OpenAIResponsesRequest responsesRequest) + { + var inputItems = new List(responsesRequest.Input ?? []) + { + new() + { + Role = "user", + Content = faultPrompt + } + }; + var newRequest = new OpenAIResponsesRequest + { + Model = responsesRequest.Model, + Stream = responsesRequest.Stream, + Temperature = responsesRequest.Temperature, + TopP = responsesRequest.TopP, + Instructions = responsesRequest.Instructions, + PreviousResponseId = responsesRequest.PreviousResponseId, + MaxOutputTokens = responsesRequest.MaxOutputTokens, + Input = inputItems + }; + + Logger.LogDebug("Added fault prompt to Responses API input: {Prompt}", faultPrompt); + Logger.LogRequest($"Simulating fault {faultName}", MessageType.Chaos, new LoggingContext(e.Session)); + e.Session.SetRequestBodyString(JsonSerializer.Serialize(newRequest, ProxyUtils.JsonSerializerOptions)); + } else { Logger.LogDebug("Unknown OpenAI request type. Passing request as-is."); @@ -126,45 +152,6 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo Logger.LogTrace("Left {Name}", nameof(BeforeRequestAsync)); } - private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) - { - request = null; - - if (string.IsNullOrEmpty(content)) - { - return false; - } - - try - { - Logger.LogDebug("Checking if the request is an OpenAI request..."); - - var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - - if (rawRequest.TryGetProperty("prompt", out _)) - { - Logger.LogDebug("Request is a completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - if (rawRequest.TryGetProperty("messages", out _)) - { - Logger.LogDebug("Request is a chat completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - Logger.LogDebug("Request is not an OpenAI request."); - return false; - } - catch (JsonException ex) - { - Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); - return false; - } - } - private (string? Name, string? Prompt) GetFault() { var failures = Configuration.Failures?.ToArray() ?? _defaultFailures; diff --git a/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs b/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs index 991ad4da..9858f672 100644 --- a/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs +++ b/DevProxy.Plugins/Behavior/LanguageModelRateLimitingPlugin.cs @@ -99,7 +99,7 @@ public override Task BeforeRequestAsync(ProxyRequestArgs e, CancellationToken ca return Task.CompletedTask; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new LoggingContext(e.Session)); return Task.CompletedTask; @@ -224,7 +224,7 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken return Task.CompletedTask; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogDebug("Skipping non-OpenAI request"); return Task.CompletedTask; @@ -271,45 +271,6 @@ public override Task BeforeResponseAsync(ProxyResponseArgs e, CancellationToken return Task.CompletedTask; } - private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) - { - request = null; - - if (string.IsNullOrEmpty(content)) - { - return false; - } - - try - { - Logger.LogDebug("Checking if the request is an OpenAI request..."); - - var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - - if (rawRequest.TryGetProperty("prompt", out _)) - { - Logger.LogDebug("Request is a completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - if (rawRequest.TryGetProperty("messages", out _)) - { - Logger.LogDebug("Request is a chat completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - Logger.LogDebug("Request is not an OpenAI request."); - return false; - } - catch (JsonException ex) - { - Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); - return false; - } - } - private ThrottlingInfo ShouldThrottle(Request request, string throttlingKey) { var throttleKeyForRequest = BuildThrottleKey(request); diff --git a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs index de231805..eae46b45 100644 --- a/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs +++ b/DevProxy.Plugins/Inspection/OpenAITelemetryPlugin.cs @@ -368,6 +368,9 @@ private void AddResponseTypeSpecificTags(Activity activity, OpenAIRequest openAi case OpenAIFineTuneRequest: AddFineTuneResponseTags(activity, openAiRequest, responseBody); break; + case OpenAIResponsesRequest: + AddResponsesResponseTags(activity, openAiRequest, responseBody); + break; default: throw new InvalidOperationException($"Unsupported OpenAI request type: {openAiRequest.GetType().Name}"); } @@ -402,6 +405,33 @@ private void AddFineTuneResponseTags(Activity activity, OpenAIRequest openAIRequ Logger.LogTrace("AddFineTuneResponseTags() finished"); } + private void AddResponsesResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) + { + Logger.LogTrace("AddResponsesResponseTags() called"); + + var responsesResponse = JsonSerializer.Deserialize(responseBody, ProxyUtils.JsonSerializerOptions); + if (responsesResponse is null) + { + return; + } + + RecordUsageMetrics(activity, openAIRequest, responsesResponse); + + _ = activity.SetTag(SemanticConvention.GEN_AI_RESPONSE_ID, responsesResponse.Id); + + if (!string.IsNullOrEmpty(responsesResponse.Status)) + { + _ = activity.SetTag("ai.response.status", responsesResponse.Status); + } + + if (Configuration.IncludeCompletion && responsesResponse.Response is not null) + { + _ = activity.SetTag(SemanticConvention.GEN_AI_CONTENT_COMPLETION, responsesResponse.Response); + } + + Logger.LogTrace("AddResponsesResponseTags() finished"); + } + private void AddAudioResponseTags(Activity activity, OpenAIRequest openAIRequest, string responseBody) { Logger.LogTrace("AddAudioResponseTags() called"); @@ -562,6 +592,9 @@ private void AddRequestTypeSpecificTags(Activity activity, OpenAIRequest openAiR case OpenAIFineTuneRequest fineTuneRequest: AddFineTuneRequestTags(activity, fineTuneRequest); break; + case OpenAIResponsesRequest responsesRequest: + AddResponsesRequestTags(activity, responsesRequest); + break; default: throw new InvalidOperationException($"Unsupported OpenAI request type: {openAiRequest.GetType().Name}"); } @@ -774,6 +807,43 @@ private void AddFineTuneRequestTags(Activity activity, OpenAIFineTuneRequest fin Logger.LogTrace("AddFineTuneRequestTags() finished"); } + private void AddResponsesRequestTags(Activity activity, OpenAIResponsesRequest responsesRequest) + { + Logger.LogTrace("AddResponsesRequestTags() called"); + + // OpenLIT + _ = activity.SetTag(SemanticConvention.GEN_AI_OPERATION, SemanticConvention.GEN_AI_OPERATION_TYPE_CHAT) + // OpenTelemetry + .SetTag(SemanticConvention.GEN_AI_OPERATION_NAME, "responses"); + + if (Configuration.IncludePrompt && responsesRequest.Input is not null) + { + // Format input items to a more readable form for the span + var formattedInputs = responsesRequest.Input + .Select(i => $"{i.Role}: {i.Content}") + .ToArray(); + + _ = activity.SetTag(SemanticConvention.GEN_AI_CONTENT_PROMPT, string.Join("\n", formattedInputs)); + } + + if (!string.IsNullOrEmpty(responsesRequest.Instructions)) + { + _ = activity.SetTag("ai.request.instructions", responsesRequest.Instructions); + } + + if (!string.IsNullOrEmpty(responsesRequest.PreviousResponseId)) + { + _ = activity.SetTag("ai.request.previous_response_id", responsesRequest.PreviousResponseId); + } + + if (responsesRequest.MaxOutputTokens.HasValue) + { + _ = activity.SetTag(SemanticConvention.GEN_AI_REQUEST_MAX_TOKENS, responsesRequest.MaxOutputTokens.Value); + } + + Logger.LogTrace("AddResponsesRequestTags() finished"); + } + private void AddCommonRequestTags(Activity activity, OpenAIRequest openAiRequest) { Logger.LogTrace("AddCommonRequestTags() called"); @@ -1004,6 +1074,7 @@ private static string GetOperationName(OpenAIRequest request) OpenAIAudioRequest => "audio.transcriptions", OpenAIAudioSpeechRequest => "audio.speech", OpenAIFineTuneRequest => "fine_tuning.jobs", + OpenAIResponsesRequest => "responses", _ => "unknown" }; } diff --git a/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs b/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs index 0a38a7a5..9d20c953 100644 --- a/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs +++ b/DevProxy.Plugins/Mocking/OpenAIMockResponsePlugin.cs @@ -18,6 +18,8 @@ public sealed class OpenAIMockResponsePlugin( ISet urlsToWatch, ILanguageModelClient languageModelClient) : BasePlugin(logger, urlsToWatch) { + private const string ResponsesMessageIdPrefix = "msg_"; + public override string Name => nameof(OpenAIMockResponsePlugin); public override async Task InitializeAsync(InitArgs e, CancellationToken cancellationToken) @@ -58,7 +60,7 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo return; } - if (!TryGetOpenAIRequest(request.BodyString, out var openAiRequest)) + if (!OpenAIRequest.TryGetCompletionLikeRequest(request.BodyString, Logger, out var openAiRequest)) { Logger.LogRequest("Skipping non-OpenAI request", MessageType.Skipped, new LoggingContext(e.Session)); return; @@ -95,6 +97,25 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo var openAiResponse = lmResponse.ConvertToOpenAIResponse(); SendMockResponse(openAiResponse, lmResponse.RequestUrl ?? string.Empty, e); } + else if (openAiRequest is OpenAIResponsesRequest responsesRequest) + { + // Convert Responses API input to chat completion messages + var messages = ConvertResponsesInputToChatMessages(responsesRequest); + if ((await languageModelClient + .GenerateChatCompletionAsync(messages, null, cancellationToken)) is not ILanguageModelCompletionResponse lmResponse) + { + return; + } + if (lmResponse.ErrorMessage is not null) + { + Logger.LogError("Error from local language model: {Error}", lmResponse.ErrorMessage); + return; + } + + // Convert the chat completion response to Responses API format + var responsesResponse = ConvertToResponsesResponse(lmResponse.ConvertToOpenAIResponse()); + SendMockResponse(responsesResponse, lmResponse.RequestUrl ?? string.Empty, e); + } else { Logger.LogError("Unknown OpenAI request type."); @@ -103,43 +124,50 @@ public override async Task BeforeRequestAsync(ProxyRequestArgs e, CancellationTo Logger.LogTrace("Left {Name}", nameof(BeforeRequestAsync)); } - private bool TryGetOpenAIRequest(string content, out OpenAIRequest? request) + private static IEnumerable ConvertResponsesInputToChatMessages(OpenAIResponsesRequest responsesRequest) { - request = null; - - if (string.IsNullOrEmpty(content)) + if (responsesRequest.Input is null) { - return false; + return []; } - try + return responsesRequest.Input.Select(item => new OpenAIChatCompletionMessage { - Logger.LogDebug("Checking if the request is an OpenAI request..."); - - var rawRequest = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - - if (rawRequest.TryGetProperty("prompt", out _)) - { - Logger.LogDebug("Request is a completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } - - if (rawRequest.TryGetProperty("messages", out _)) - { - Logger.LogDebug("Request is a chat completion request"); - request = JsonSerializer.Deserialize(content, ProxyUtils.JsonSerializerOptions); - return true; - } + Role = item.Role, + Content = item.Content + }); + } - Logger.LogDebug("Request is not an OpenAI request."); - return false; - } - catch (JsonException ex) + private static OpenAIResponsesResponse ConvertToResponsesResponse(OpenAIResponse chatResponse) + { + return new OpenAIResponsesResponse { - Logger.LogDebug(ex, "Failed to deserialize OpenAI request."); - return false; - } + Id = chatResponse.Id, + Model = chatResponse.Model, + Object = "response", + Created = chatResponse.Created, + CreatedAt = chatResponse.Created, + Status = "completed", + Usage = chatResponse.Usage, + Output = + [ + new OpenAIResponsesOutputItem + { + Type = "message", + Id = $"{ResponsesMessageIdPrefix}{Guid.NewGuid():N}", + Role = "assistant", + Status = "completed", + Content = + [ + new OpenAIResponsesOutputContent + { + Type = "output_text", + Text = chatResponse.Response + } + ] + } + ] + }; } private void SendMockResponse(OpenAIResponse response, string localLmUrl, ProxyRequestArgs e) where TResponse : OpenAIResponse