package service import ( "bufio" "bytes" "context" "encoding/base64" "fmt" "io" "net/http" "strings" "sync/atomic" "time" "github.com/Wei-Shaw/sub2api/internal/pkg/logger" "github.com/Wei-Shaw/sub2api/internal/util/responseheaders" "github.com/gin-gonic/gin" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) type openAIResponsesImageResult struct { Result string RevisedPrompt string OutputFormat string Size string Background string Quality string Model string } type OpenAIImagesUpstreamError struct { StatusCode int ErrorType string Code string Message string Param string UpstreamRequestID string } func (e *OpenAIImagesUpstreamError) Error() string { if e == nil { return "" } code := strings.TrimSpace(e.Code) if code == "" { code = strings.TrimSpace(e.ErrorType) } message := strings.TrimSpace(e.Message) if code != "" && message != "" { return fmt.Sprintf("openai images upstream error: %s: %s", code, message) } if message != "" { return "openai images upstream error: " + message } if code != "" { return "openai images upstream error: " + code } return "openai images upstream error" } func (e *OpenAIImagesUpstreamError) clientStatusCode() int { if e == nil { return http.StatusBadGateway } if e.StatusCode > 0 { return e.StatusCode } return http.StatusBadGateway } func (e *OpenAIImagesUpstreamError) clientErrorType() string { if e == nil { return "upstream_error" } if trimmed := strings.TrimSpace(e.ErrorType); trimmed != "" { return trimmed } return "upstream_error" } func (e *OpenAIImagesUpstreamError) clientMessage() string { if e == nil { return "Upstream request failed" } if trimmed := strings.TrimSpace(e.Message); trimmed != "" { return trimmed } if trimmed := strings.TrimSpace(e.Code); trimmed != "" { return trimmed } return "Upstream request failed" } func openAIResponsesImageResultKey(itemID string, result openAIResponsesImageResult) string { if strings.TrimSpace(result.Result) != "" { return strings.TrimSpace(result.OutputFormat) + "|" + strings.TrimSpace(result.Result) } return "item:" + strings.TrimSpace(itemID) } func appendOpenAIResponsesImageResultDedup(results *[]openAIResponsesImageResult, seen map[string]struct{}, itemID string, result openAIResponsesImageResult) bool { if results == nil { return false } key := openAIResponsesImageResultKey(itemID, result) if key != "" { if _, exists := seen[key]; exists { return false } seen[key] = struct{}{} } *results = append(*results, result) return true } func mergeOpenAIResponsesImageMeta(dst *openAIResponsesImageResult, src openAIResponsesImageResult) { if dst == nil { return } if trimmed := strings.TrimSpace(src.OutputFormat); trimmed != "" { dst.OutputFormat = trimmed } if trimmed := strings.TrimSpace(src.Size); trimmed != "" { dst.Size = trimmed } if trimmed := strings.TrimSpace(src.Background); trimmed != "" { dst.Background = trimmed } if trimmed := strings.TrimSpace(src.Quality); trimmed != "" { dst.Quality = trimmed } if trimmed := strings.TrimSpace(src.Model); trimmed != "" { dst.Model = trimmed } } func openAIResponsesImageResultSizes(results []openAIResponsesImageResult) []string { if len(results) == 0 { return nil } sizes := make([]string, 0, len(results)) for _, result := range results { if size := strings.TrimSpace(result.Size); size != "" { sizes = append(sizes, size) } } if len(sizes) == 0 { return nil } return sizes } func extractOpenAIResponsesImageMetaFromLifecycleEvent(payload []byte) (openAIResponsesImageResult, int64, bool) { switch gjson.GetBytes(payload, "type").String() { case "response.created", "response.in_progress", "response.completed": default: return openAIResponsesImageResult{}, 0, false } response := gjson.GetBytes(payload, "response") if !response.Exists() { return openAIResponsesImageResult{}, 0, false } meta := openAIResponsesImageResult{ OutputFormat: strings.TrimSpace(response.Get("tools.0.output_format").String()), Size: strings.TrimSpace(response.Get("tools.0.size").String()), Background: strings.TrimSpace(response.Get("tools.0.background").String()), Quality: strings.TrimSpace(response.Get("tools.0.quality").String()), Model: strings.TrimSpace(response.Get("tools.0.model").String()), } return meta, response.Get("created_at").Int(), true } func buildOpenAIImagesStreamPartialPayload( eventType string, b64 string, partialImageIndex int64, responseFormat string, createdAt int64, meta openAIResponsesImageResult, ) []byte { if createdAt <= 0 { createdAt = time.Now().Unix() } payload := []byte(`{"type":"","created_at":0,"partial_image_index":0,"b64_json":""}`) payload, _ = sjson.SetBytes(payload, "type", eventType) payload, _ = sjson.SetBytes(payload, "created_at", createdAt) payload, _ = sjson.SetBytes(payload, "partial_image_index", partialImageIndex) payload, _ = sjson.SetBytes(payload, "b64_json", b64) if strings.EqualFold(strings.TrimSpace(responseFormat), "url") { payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(meta.OutputFormat)+";base64,"+b64) } if meta.Background != "" { payload, _ = sjson.SetBytes(payload, "background", meta.Background) } if meta.OutputFormat != "" { payload, _ = sjson.SetBytes(payload, "output_format", meta.OutputFormat) } if meta.Quality != "" { payload, _ = sjson.SetBytes(payload, "quality", meta.Quality) } if meta.Size != "" { payload, _ = sjson.SetBytes(payload, "size", meta.Size) } if meta.Model != "" { payload, _ = sjson.SetBytes(payload, "model", meta.Model) } return payload } func buildOpenAIImagesStreamCompletedPayload( eventType string, img openAIResponsesImageResult, responseFormat string, createdAt int64, usageRaw []byte, ) []byte { if createdAt <= 0 { createdAt = time.Now().Unix() } payload := []byte(`{"type":"","created_at":0,"b64_json":""}`) payload, _ = sjson.SetBytes(payload, "type", eventType) payload, _ = sjson.SetBytes(payload, "created_at", createdAt) payload, _ = sjson.SetBytes(payload, "b64_json", img.Result) if strings.EqualFold(strings.TrimSpace(responseFormat), "url") { payload, _ = sjson.SetBytes(payload, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result) } if img.Background != "" { payload, _ = sjson.SetBytes(payload, "background", img.Background) } if img.OutputFormat != "" { payload, _ = sjson.SetBytes(payload, "output_format", img.OutputFormat) } if img.Quality != "" { payload, _ = sjson.SetBytes(payload, "quality", img.Quality) } if img.Size != "" { payload, _ = sjson.SetBytes(payload, "size", img.Size) } if img.Model != "" { payload, _ = sjson.SetBytes(payload, "model", img.Model) } if len(usageRaw) > 0 && gjson.ValidBytes(usageRaw) { payload, _ = sjson.SetRawBytes(payload, "usage", usageRaw) } return payload } func openAIImageOutputMIMEType(outputFormat string) string { if outputFormat == "" { return "image/png" } if strings.Contains(outputFormat, "/") { return outputFormat } switch strings.ToLower(strings.TrimSpace(outputFormat)) { case "png": return "image/png" case "jpg", "jpeg": return "image/jpeg" case "webp": return "image/webp" default: return "image/png" } } func openAIImageUploadToDataURL(upload OpenAIImagesUpload) (string, error) { if len(upload.Data) == 0 { return "", fmt.Errorf("upload %q is empty", strings.TrimSpace(upload.FileName)) } contentType := strings.TrimSpace(upload.ContentType) if contentType == "" { contentType = http.DetectContentType(upload.Data) } return "data:" + contentType + ";base64," + base64.StdEncoding.EncodeToString(upload.Data), nil } func buildOpenAIImagesResponsesRequest(parsed *OpenAIImagesRequest, toolModel string) ([]byte, error) { if parsed == nil { return nil, fmt.Errorf("parsed images request is required") } prompt := strings.TrimSpace(parsed.Prompt) if prompt == "" { return nil, fmt.Errorf("prompt is required") } inputImages := make([]string, 0, len(parsed.InputImageURLs)+len(parsed.Uploads)) for _, imageURL := range parsed.InputImageURLs { if trimmed := strings.TrimSpace(imageURL); trimmed != "" { inputImages = append(inputImages, trimmed) } } for _, upload := range parsed.Uploads { dataURL, err := openAIImageUploadToDataURL(upload) if err != nil { return nil, err } inputImages = append(inputImages, dataURL) } if parsed.IsEdits() && len(inputImages) == 0 { return nil, fmt.Errorf("image input is required") } req := []byte(`{"instructions":"","stream":true,"reasoning":{"effort":"medium","summary":"auto"},"parallel_tool_calls":true,"include":["reasoning.encrypted_content"],"model":"","store":false,"tool_choice":{"type":"image_generation"}}`) req, _ = sjson.SetBytes(req, "model", openAIImagesResponsesMainModel) input := []byte(`[{"type":"message","role":"user","content":[{"type":"input_text","text":""}]}]`) input, _ = sjson.SetBytes(input, "0.content.0.text", prompt) for index, imageURL := range inputImages { part := []byte(`{"type":"input_image","image_url":""}`) part, _ = sjson.SetBytes(part, "image_url", imageURL) input, _ = sjson.SetRawBytes(input, fmt.Sprintf("0.content.%d", index+1), part) } req, _ = sjson.SetRawBytes(req, "input", input) action := "generate" if parsed.IsEdits() { action = "edit" } tool := []byte(`{"type":"image_generation","action":"","model":""}`) tool, _ = sjson.SetBytes(tool, "action", action) tool, _ = sjson.SetBytes(tool, "model", strings.TrimSpace(toolModel)) if shouldPassOpenAIImagesN(toolModel, parsed.N) { tool, _ = sjson.SetBytes(tool, "n", parsed.N) } for _, field := range []struct { path string value string }{ {path: "size", value: parsed.Size}, {path: "quality", value: parsed.Quality}, {path: "background", value: parsed.Background}, {path: "output_format", value: parsed.OutputFormat}, {path: "moderation", value: parsed.Moderation}, {path: "style", value: parsed.Style}, } { if trimmed := strings.TrimSpace(field.value); trimmed != "" { tool, _ = sjson.SetBytes(tool, field.path, trimmed) } } if parsed.OutputCompression != nil { tool, _ = sjson.SetBytes(tool, "output_compression", *parsed.OutputCompression) } if parsed.PartialImages != nil { tool, _ = sjson.SetBytes(tool, "partial_images", *parsed.PartialImages) } maskImageURL := strings.TrimSpace(parsed.MaskImageURL) if parsed.MaskUpload != nil { dataURL, err := openAIImageUploadToDataURL(*parsed.MaskUpload) if err != nil { return nil, err } maskImageURL = dataURL } if maskImageURL != "" { tool, _ = sjson.SetBytes(tool, "input_image_mask.image_url", maskImageURL) } req, _ = sjson.SetRawBytes(req, "tools", []byte(`[]`)) req, _ = sjson.SetRawBytes(req, "tools.-1", tool) return req, nil } func shouldPassOpenAIImagesN(model string, n int) bool { if n <= 1 { return false } return !strings.EqualFold(strings.TrimSpace(model), "dall-e-3") } func extractOpenAIImagesFromResponsesCompleted(payload []byte) ([]openAIResponsesImageResult, int64, []byte, openAIResponsesImageResult, error) { if gjson.GetBytes(payload, "type").String() != "response.completed" { return nil, 0, nil, openAIResponsesImageResult{}, fmt.Errorf("unexpected event type") } createdAt := gjson.GetBytes(payload, "response.created_at").Int() if createdAt <= 0 { createdAt = time.Now().Unix() } var ( results []openAIResponsesImageResult firstMeta openAIResponsesImageResult ) output := gjson.GetBytes(payload, "response.output") if output.IsArray() { for _, item := range output.Array() { if item.Get("type").String() != "image_generation_call" { continue } result := strings.TrimSpace(item.Get("result").String()) if result == "" { continue } entry := openAIResponsesImageResult{ Result: result, RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()), OutputFormat: strings.TrimSpace(item.Get("output_format").String()), Size: strings.TrimSpace(item.Get("size").String()), Background: strings.TrimSpace(item.Get("background").String()), Quality: strings.TrimSpace(item.Get("quality").String()), } if len(results) == 0 { firstMeta = entry } results = append(results, entry) } } var usageRaw []byte if usage := gjson.GetBytes(payload, "response.tool_usage.image_gen"); usage.Exists() && usage.IsObject() { usageRaw = []byte(usage.Raw) } return results, createdAt, usageRaw, firstMeta, nil } func extractOpenAIImageFromResponsesOutputItemDone(payload []byte) (openAIResponsesImageResult, string, bool, error) { if gjson.GetBytes(payload, "type").String() != "response.output_item.done" { return openAIResponsesImageResult{}, "", false, fmt.Errorf("unexpected event type") } item := gjson.GetBytes(payload, "item") if !item.Exists() || item.Get("type").String() != "image_generation_call" { return openAIResponsesImageResult{}, "", false, nil } result := strings.TrimSpace(item.Get("result").String()) if result == "" { return openAIResponsesImageResult{}, "", false, nil } entry := openAIResponsesImageResult{ Result: result, RevisedPrompt: strings.TrimSpace(item.Get("revised_prompt").String()), OutputFormat: strings.TrimSpace(item.Get("output_format").String()), Size: strings.TrimSpace(item.Get("size").String()), Background: strings.TrimSpace(item.Get("background").String()), Quality: strings.TrimSpace(item.Get("quality").String()), } return entry, strings.TrimSpace(item.Get("id").String()), true, nil } func collectOpenAIImagesFromResponsesBody(body []byte) ([]openAIResponsesImageResult, int64, []byte, openAIResponsesImageResult, bool, error) { var ( fallbackResults []openAIResponsesImageResult fallbackSeen = make(map[string]struct{}) finalResults []openAIResponsesImageResult finalMeta openAIResponsesImageResult collectErr error createdAt int64 usageRaw []byte foundFinal bool responseMeta openAIResponsesImageResult ) forEachOpenAISSEDataPayload(string(body), func(payload []byte) { if collectErr != nil || len(finalResults) > 0 { return } if !gjson.ValidBytes(payload) { return } if meta, eventCreatedAt, ok := extractOpenAIResponsesImageMetaFromLifecycleEvent(payload); ok { mergeOpenAIResponsesImageMeta(&responseMeta, meta) if eventCreatedAt > 0 { createdAt = eventCreatedAt } } switch gjson.GetBytes(payload, "type").String() { case "response.output_item.done": result, itemID, ok, err := extractOpenAIImageFromResponsesOutputItemDone(payload) if err != nil { collectErr = err return } if ok { mergeOpenAIResponsesImageMeta(&result, responseMeta) appendOpenAIResponsesImageResultDedup(&fallbackResults, fallbackSeen, itemID, result) } case "response.completed": results, completedAt, completedUsageRaw, firstMeta, err := extractOpenAIImagesFromResponsesCompleted(payload) if err != nil { collectErr = err return } foundFinal = true if completedAt > 0 { createdAt = completedAt } if len(completedUsageRaw) > 0 { usageRaw = completedUsageRaw } if len(results) > 0 { mergeOpenAIResponsesImageMeta(&firstMeta, responseMeta) finalResults = results finalMeta = firstMeta return } if len(fallbackResults) > 0 { firstMeta = fallbackResults[0] mergeOpenAIResponsesImageMeta(&firstMeta, responseMeta) finalResults = fallbackResults finalMeta = firstMeta return } } }) if collectErr != nil { return nil, 0, nil, openAIResponsesImageResult{}, false, collectErr } if len(finalResults) > 0 { return finalResults, createdAt, usageRaw, finalMeta, true, nil } if len(fallbackResults) > 0 { firstMeta := fallbackResults[0] mergeOpenAIResponsesImageMeta(&firstMeta, responseMeta) return fallbackResults, createdAt, usageRaw, firstMeta, foundFinal, nil } return nil, createdAt, usageRaw, openAIResponsesImageResult{}, foundFinal, nil } func extractOpenAIImagesUpstreamError(body []byte) *OpenAIImagesUpstreamError { var upstreamErr *OpenAIImagesUpstreamError forEachOpenAISSEDataPayload(string(body), func(payload []byte) { if upstreamErr != nil || !gjson.ValidBytes(payload) { return } upstreamErr = openAIImagesUpstreamErrorFromSSEPayload(payload) }) return upstreamErr } func openAIImagesUpstreamErrorFromSSEPayload(payload []byte) *OpenAIImagesUpstreamError { if !gjson.ValidBytes(payload) { return nil } switch gjson.GetBytes(payload, "type").String() { case "error": return openAIImagesUpstreamErrorFromGJSON(gjson.GetBytes(payload, "error"), "") case "response.failed": response := gjson.GetBytes(payload, "response") return openAIImagesUpstreamErrorFromGJSON(response.Get("error"), response.Get("id").String()) default: return nil } } func openAIImagesUpstreamErrorFromGJSON(errorObj gjson.Result, upstreamRequestID string) *OpenAIImagesUpstreamError { if !errorObj.Exists() { return nil } code := strings.TrimSpace(errorObj.Get("code").String()) errType := strings.TrimSpace(errorObj.Get("type").String()) message := strings.TrimSpace(errorObj.Get("message").String()) param := strings.TrimSpace(errorObj.Get("param").String()) statusCode := http.StatusBadGateway if strings.EqualFold(code, "moderation_blocked") || strings.EqualFold(errType, "image_generation_user_error") { statusCode = http.StatusBadRequest } if message == "" { message = "Upstream request failed" } return &OpenAIImagesUpstreamError{ StatusCode: statusCode, ErrorType: errType, Code: code, Message: sanitizeUpstreamErrorMessage(message), Param: param, UpstreamRequestID: strings.TrimSpace(upstreamRequestID), } } func buildOpenAIImagesAPIResponse( results []openAIResponsesImageResult, createdAt int64, usageRaw []byte, firstMeta openAIResponsesImageResult, responseFormat string, ) ([]byte, error) { if createdAt <= 0 { createdAt = time.Now().Unix() } out := []byte(`{"created":0,"data":[]}`) out, _ = sjson.SetBytes(out, "created", createdAt) format := strings.ToLower(strings.TrimSpace(responseFormat)) if format == "" { format = "b64_json" } for _, img := range results { item := []byte(`{}`) if format == "url" { item, _ = sjson.SetBytes(item, "url", "data:"+openAIImageOutputMIMEType(img.OutputFormat)+";base64,"+img.Result) } else { item, _ = sjson.SetBytes(item, "b64_json", img.Result) } if img.RevisedPrompt != "" { item, _ = sjson.SetBytes(item, "revised_prompt", img.RevisedPrompt) } out, _ = sjson.SetRawBytes(out, "data.-1", item) } if firstMeta.Background != "" { out, _ = sjson.SetBytes(out, "background", firstMeta.Background) } if firstMeta.OutputFormat != "" { out, _ = sjson.SetBytes(out, "output_format", firstMeta.OutputFormat) } if firstMeta.Quality != "" { out, _ = sjson.SetBytes(out, "quality", firstMeta.Quality) } if firstMeta.Size != "" { out, _ = sjson.SetBytes(out, "size", firstMeta.Size) } if firstMeta.Model != "" { out, _ = sjson.SetBytes(out, "model", firstMeta.Model) } if len(usageRaw) > 0 && gjson.ValidBytes(usageRaw) { out, _ = sjson.SetRawBytes(out, "usage", usageRaw) } return out, nil } func openAIImagesStreamPrefix(parsed *OpenAIImagesRequest) string { if parsed != nil && parsed.IsEdits() { return "image_edit" } return "image_generation" } func buildOpenAIImagesStreamErrorBody(message string) []byte { body := []byte(`{"type":"error","error":{"type":"upstream_error","message":""}}`) if strings.TrimSpace(message) == "" { message = "upstream request failed" } body, _ = sjson.SetBytes(body, "error.message", message) return body } func buildOpenAIImagesStreamErrorBodyFromUpstream(err *OpenAIImagesUpstreamError) []byte { if err == nil { return buildOpenAIImagesStreamErrorBody("") } body := buildOpenAIImagesStreamErrorBody(err.clientMessage()) body, _ = sjson.SetBytes(body, "error.type", err.clientErrorType()) if code := strings.TrimSpace(err.Code); code != "" { body, _ = sjson.SetBytes(body, "error.code", code) } if param := strings.TrimSpace(err.Param); param != "" { body, _ = sjson.SetBytes(body, "error.param", param) } return body } func writeOpenAIImagesUpstreamErrorResponse(c *gin.Context, err *OpenAIImagesUpstreamError) bool { if c == nil || c.Writer == nil || c.Writer.Written() || err == nil { return false } errorObj := gin.H{ "type": err.clientErrorType(), "message": err.clientMessage(), } if code := strings.TrimSpace(err.Code); code != "" { errorObj["code"] = code } if param := strings.TrimSpace(err.Param); param != "" { errorObj["param"] = param } c.JSON(err.clientStatusCode(), gin.H{ "error": errorObj, }) return true } func (s *OpenAIGatewayService) writeOpenAIImagesStreamEvent(c *gin.Context, flusher http.Flusher, eventName string, payload []byte) error { if strings.TrimSpace(eventName) != "" { if _, err := fmt.Fprintf(c.Writer, "event: %s\n", eventName); err != nil { return err } } if _, err := fmt.Fprintf(c.Writer, "data: %s\n\n", payload); err != nil { return err } flusher.Flush() return nil } func (s *OpenAIGatewayService) tryWriteOpenAIImagesStreamEvent( c *gin.Context, flusher http.Flusher, clientDisconnected *bool, lastWriteAt *time.Time, eventName string, payload []byte, ) bool { if clientDisconnected != nil && *clientDisconnected { return false } if err := s.writeOpenAIImagesStreamEvent(c, flusher, eventName, payload); err != nil { if clientDisconnected != nil { *clientDisconnected = true } logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images stream client disconnected, continue draining upstream for billing") return false } if lastWriteAt != nil { *lastWriteAt = time.Now() } return true } func (s *OpenAIGatewayService) handleOpenAIImagesOAuthNonStreamingResponse( resp *http.Response, c *gin.Context, responseFormat string, fallbackModel string, ) (OpenAIUsage, int, []string, error) { body, err := ReadUpstreamResponseBody(resp.Body, s.cfg, c, openAITooLargeError) if err != nil { return OpenAIUsage{}, 0, nil, err } var usage OpenAIUsage forEachOpenAISSEDataPayload(string(body), func(data []byte) { s.parseSSEUsageBytes(data, &usage) }) results, createdAt, usageRaw, firstMeta, _, err := collectOpenAIImagesFromResponsesBody(body) if err != nil { return OpenAIUsage{}, 0, nil, err } if len(results) == 0 { if upstreamErr := extractOpenAIImagesUpstreamError(body); upstreamErr != nil { setOpsUpstreamError(c, upstreamErr.clientStatusCode(), upstreamErr.clientMessage(), "") writeOpenAIImagesUpstreamErrorResponse(c, upstreamErr) return OpenAIUsage{}, 0, nil, upstreamErr } return OpenAIUsage{}, 0, nil, fmt.Errorf("upstream did not return image output") } if strings.TrimSpace(firstMeta.Model) == "" { firstMeta.Model = strings.TrimSpace(fallbackModel) } responseBody, err := buildOpenAIImagesAPIResponse(results, createdAt, usageRaw, firstMeta, responseFormat) if err != nil { return OpenAIUsage{}, 0, nil, err } responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) c.Data(resp.StatusCode, "application/json; charset=utf-8", responseBody) return usage, len(results), openAIResponsesImageResultSizes(results), nil } func (s *OpenAIGatewayService) handleOpenAIImagesOAuthStreamingResponse( resp *http.Response, c *gin.Context, startTime time.Time, responseFormat string, streamPrefix string, fallbackModel string, ) (OpenAIUsage, int, []string, *int, error) { responseheaders.WriteFilteredHeaders(c.Writer.Header(), resp.Header, s.responseHeaderFilter) c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") c.Status(resp.StatusCode) flusher, ok := c.Writer.(http.Flusher) if !ok { return OpenAIUsage{}, 0, nil, nil, fmt.Errorf("streaming is not supported by response writer") } format := strings.ToLower(strings.TrimSpace(responseFormat)) if format == "" { format = "b64_json" } usage := OpenAIUsage{} imageCount := 0 var imageOutputSizes []string var firstTokenMs *int emitted := make(map[string]struct{}) pendingResults := make([]openAIResponsesImageResult, 0, 1) pendingSeen := make(map[string]struct{}) streamMeta := openAIResponsesImageResult{Model: strings.TrimSpace(fallbackModel)} var createdAt int64 clientDisconnected := false lastDownstreamWriteAt := time.Now() var sseData openAISSEDataAccumulator var processDataErr error processDataDone := false processData := func(dataBytes []byte) { if processDataDone || processDataErr != nil { return } if firstTokenMs == nil { ms := int(time.Since(startTime).Milliseconds()) firstTokenMs = &ms } s.parseSSEUsageBytes(dataBytes, &usage) if !gjson.ValidBytes(dataBytes) { return } if meta, eventCreatedAt, ok := extractOpenAIResponsesImageMetaFromLifecycleEvent(dataBytes); ok { mergeOpenAIResponsesImageMeta(&streamMeta, meta) if eventCreatedAt > 0 { createdAt = eventCreatedAt } } switch gjson.GetBytes(dataBytes, "type").String() { case "response.image_generation_call.partial_image": b64 := strings.TrimSpace(gjson.GetBytes(dataBytes, "partial_image_b64").String()) if b64 == "" { return } eventName := streamPrefix + ".partial_image" partialMeta := streamMeta mergeOpenAIResponsesImageMeta(&partialMeta, openAIResponsesImageResult{ OutputFormat: strings.TrimSpace(gjson.GetBytes(dataBytes, "output_format").String()), Background: strings.TrimSpace(gjson.GetBytes(dataBytes, "background").String()), }) payload := buildOpenAIImagesStreamPartialPayload( eventName, b64, gjson.GetBytes(dataBytes, "partial_image_index").Int(), format, createdAt, partialMeta, ) s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, eventName, payload) case "response.output_item.done": img, itemID, ok, extractErr := extractOpenAIImageFromResponsesOutputItemDone(dataBytes) if extractErr != nil { s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(extractErr.Error())) processDataErr = extractErr processDataDone = true return } if !ok { return } mergeOpenAIResponsesImageMeta(&streamMeta, img) mergeOpenAIResponsesImageMeta(&img, streamMeta) key := openAIResponsesImageResultKey(itemID, img) if _, exists := emitted[key]; exists { return } if _, exists := pendingSeen[key]; exists { return } pendingSeen[key] = struct{}{} pendingResults = append(pendingResults, img) case "response.completed": results, _, usageRaw, firstMeta, extractErr := extractOpenAIImagesFromResponsesCompleted(dataBytes) if extractErr != nil { s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(extractErr.Error())) processDataErr = extractErr processDataDone = true return } mergeOpenAIResponsesImageMeta(&streamMeta, firstMeta) finalResults := make([]openAIResponsesImageResult, 0, len(results)+len(pendingResults)) finalSeen := make(map[string]struct{}) for _, img := range results { mergeOpenAIResponsesImageMeta(&img, streamMeta) appendOpenAIResponsesImageResultDedup(&finalResults, finalSeen, "", img) } for _, img := range pendingResults { mergeOpenAIResponsesImageMeta(&img, streamMeta) appendOpenAIResponsesImageResultDedup(&finalResults, finalSeen, "", img) } if len(finalResults) == 0 { outputErr := fmt.Errorf("upstream did not return image output") s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(outputErr.Error())) processDataErr = outputErr processDataDone = true return } eventName := streamPrefix + ".completed" for _, img := range finalResults { key := openAIResponsesImageResultKey("", img) if _, exists := emitted[key]; exists { continue } payload := buildOpenAIImagesStreamCompletedPayload(eventName, img, format, createdAt, usageRaw) emitted[key] = struct{}{} s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, eventName, payload) } imageCount = len(emitted) imageOutputSizes = openAIResponsesImageResultSizes(finalResults) processDataDone = true case "error", "response.failed": if upstreamErr := openAIImagesUpstreamErrorFromSSEPayload(dataBytes); upstreamErr != nil { if !clientDisconnected { s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBodyFromUpstream(upstreamErr)) } setOpsUpstreamError(c, upstreamErr.clientStatusCode(), upstreamErr.clientMessage(), "") processDataErr = upstreamErr processDataDone = true return } } } processLine := func(line []byte) (bool, error) { if len(line) == 0 { return false, nil } sseData.AddLine(string(line), processData) if processDataErr != nil { return true, processDataErr } return processDataDone, nil } flushData := func() (bool, error) { sseData.Flush(processData) if processDataErr != nil { return true, processDataErr } return processDataDone, nil } finalizePending := func() error { if imageCount > 0 { return nil } if len(pendingResults) > 0 { eventName := streamPrefix + ".completed" for _, img := range pendingResults { mergeOpenAIResponsesImageMeta(&img, streamMeta) key := openAIResponsesImageResultKey("", img) if _, exists := emitted[key]; exists { continue } payload := buildOpenAIImagesStreamCompletedPayload(eventName, img, format, createdAt, nil) emitted[key] = struct{}{} s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, eventName, payload) } imageCount = len(emitted) imageOutputSizes = openAIResponsesImageResultSizes(pendingResults) return nil } streamErr := fmt.Errorf("stream disconnected before image generation completed") s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(streamErr.Error())) return streamErr } streamInterval := s.openAIImageStreamDataInterval() keepaliveInterval := s.openAIImageStreamKeepaliveInterval() if streamInterval <= 0 && keepaliveInterval <= 0 { reader := bufio.NewReader(resp.Body) for { line, err := reader.ReadBytes('\n') done, processErr := processLine(line) if processErr != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, processErr } if done { return usage, imageCount, imageOutputSizes, firstTokenMs, nil } if err == io.EOF { break } if err != nil { if done, processErr := flushData(); processErr != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, processErr } else if done { return usage, imageCount, imageOutputSizes, firstTokenMs, nil } s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(err.Error())) return usage, imageCount, imageOutputSizes, firstTokenMs, err } } if done, processErr := flushData(); processErr != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, processErr } else if done { return usage, imageCount, imageOutputSizes, firstTokenMs, nil } if err := finalizePending(); err != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, err } return usage, imageCount, imageOutputSizes, firstTokenMs, nil } type readEvent struct { line []byte err error } events := make(chan readEvent, 16) done := make(chan struct{}) sendEvent := func(ev readEvent) bool { select { case events <- ev: return true case <-done: return false } } var lastReadAt int64 atomic.StoreInt64(&lastReadAt, time.Now().UnixNano()) go func() { defer close(events) reader := bufio.NewReader(resp.Body) for { line, err := reader.ReadBytes('\n') if len(line) > 0 { atomic.StoreInt64(&lastReadAt, time.Now().UnixNano()) } if len(line) > 0 && !sendEvent(readEvent{line: line}) { return } if err == io.EOF { return } if err != nil { _ = sendEvent(readEvent{err: err}) return } } }() defer close(done) var intervalTicker *time.Ticker if streamInterval > 0 { intervalTicker = time.NewTicker(streamInterval) defer intervalTicker.Stop() } var intervalCh <-chan time.Time if intervalTicker != nil { intervalCh = intervalTicker.C } var keepaliveTicker *time.Ticker if keepaliveInterval > 0 { keepaliveTicker = time.NewTicker(keepaliveInterval) defer keepaliveTicker.Stop() } var keepaliveCh <-chan time.Time if keepaliveTicker != nil { keepaliveCh = keepaliveTicker.C } for { select { case ev, ok := <-events: if !ok { if done, processErr := flushData(); processErr != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, processErr } else if done { return usage, imageCount, imageOutputSizes, firstTokenMs, nil } if err := finalizePending(); err != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, err } return usage, imageCount, imageOutputSizes, firstTokenMs, nil } if ev.err != nil { if done, processErr := flushData(); processErr != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, processErr } else if done { return usage, imageCount, imageOutputSizes, firstTokenMs, nil } s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(ev.err.Error())) return usage, imageCount, imageOutputSizes, firstTokenMs, ev.err } done, processErr := processLine(ev.line) if processErr != nil { return usage, imageCount, imageOutputSizes, firstTokenMs, processErr } if done { return usage, imageCount, imageOutputSizes, firstTokenMs, nil } case <-intervalCh: lastRead := time.Unix(0, atomic.LoadInt64(&lastReadAt)) if time.Since(lastRead) < streamInterval { continue } if clientDisconnected { return usage, imageCount, imageOutputSizes, firstTokenMs, fmt.Errorf("image stream incomplete after timeout") } logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images responses stream data interval timeout: interval=%s", streamInterval) s.tryWriteOpenAIImagesStreamEvent(c, flusher, &clientDisconnected, &lastDownstreamWriteAt, "error", buildOpenAIImagesStreamErrorBody(fmt.Sprintf("upstream image stream idle for %s", streamInterval))) return usage, imageCount, imageOutputSizes, firstTokenMs, fmt.Errorf("image stream data interval timeout") case <-keepaliveCh: if clientDisconnected || time.Since(lastDownstreamWriteAt) < keepaliveInterval { continue } if _, writeErr := io.WriteString(c.Writer, ":\n\n"); writeErr != nil { clientDisconnected = true logger.LegacyPrintf("service.openai_gateway", "[OpenAI] Images responses stream client disconnected during keepalive, continue draining upstream for billing") continue } flusher.Flush() lastDownstreamWriteAt = time.Now() } } } func (s *OpenAIGatewayService) forwardOpenAIImagesOAuth( ctx context.Context, c *gin.Context, account *Account, parsed *OpenAIImagesRequest, channelMappedModel string, ) (*OpenAIForwardResult, error) { startTime := time.Now() requestModel := strings.TrimSpace(parsed.Model) if mapped := strings.TrimSpace(channelMappedModel); mapped != "" { requestModel = mapped } if requestModel == "" { requestModel = "gpt-image-2" } if err := validateOpenAIImagesModel(requestModel); err != nil { return nil, err } logger.LegacyPrintf( "service.openai_gateway", "[OpenAI] Images request routing request_model=%s endpoint=%s account_type=%s uploads=%d", requestModel, parsed.Endpoint, account.Type, len(parsed.Uploads), ) upstreamCtx, releaseUpstreamCtx := detachUpstreamContext(ctx) defer releaseUpstreamCtx() token, _, err := s.GetAccessToken(upstreamCtx, account) if err != nil { return nil, err } responsesBody, err := buildOpenAIImagesResponsesRequest(parsed, requestModel) if err != nil { return nil, err } upstreamReq, err := s.buildUpstreamRequest(upstreamCtx, c, account, responsesBody, token, true, parsed.StickySessionSeed(), false) if err != nil { return nil, err } upstreamReq.Header.Set("Content-Type", "application/json") upstreamReq.Header.Set("Accept", "text/event-stream") proxyURL := "" if account.ProxyID != nil && account.Proxy != nil { proxyURL = account.Proxy.URL() } upstreamStart := time.Now() resp, err := s.httpUpstream.Do(upstreamReq, proxyURL, account.ID, account.Concurrency) SetOpsLatencyMs(c, OpsUpstreamLatencyMsKey, time.Since(upstreamStart).Milliseconds()) if err != nil { safeErr := sanitizeUpstreamErrorMessage(err.Error()) setOpsUpstreamError(c, 0, safeErr, "") appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: 0, UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()), Kind: "request_error", Message: safeErr, }) return nil, fmt.Errorf("upstream request failed: %s", safeErr) } if resp.StatusCode >= 400 { respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 2<<20)) _ = resp.Body.Close() resp.Body = io.NopCloser(bytes.NewReader(respBody)) upstreamMsg := strings.TrimSpace(extractUpstreamErrorMessage(respBody)) upstreamMsg = sanitizeUpstreamErrorMessage(upstreamMsg) if s.shouldFailoverOpenAIUpstreamResponse(resp.StatusCode, upstreamMsg, respBody) { appendOpsUpstreamError(c, OpsUpstreamErrorEvent{ Platform: account.Platform, AccountID: account.ID, AccountName: account.Name, UpstreamStatusCode: resp.StatusCode, UpstreamRequestID: resp.Header.Get("x-request-id"), UpstreamURL: safeUpstreamURL(upstreamReq.URL.String()), Kind: "failover", Message: upstreamMsg, }) s.handleFailoverSideEffects(upstreamCtx, resp, account, requestModel) return nil, &UpstreamFailoverError{ StatusCode: resp.StatusCode, ResponseBody: respBody, RetryableOnSameAccount: account.IsPoolMode() && account.IsPoolModeRetryableStatus(resp.StatusCode), } } return s.handleErrorResponse(upstreamCtx, resp, c, account, responsesBody) } defer func() { _ = resp.Body.Close() }() var ( usage OpenAIUsage imageCount int imageOutputSizes []string firstTokenMs *int ) if parsed.Stream { usage, imageCount, imageOutputSizes, firstTokenMs, err = s.handleOpenAIImagesOAuthStreamingResponse(resp, c, startTime, parsed.ResponseFormat, openAIImagesStreamPrefix(parsed), requestModel) if err != nil { if imageCount > 0 { return &OpenAIForwardResult{ RequestID: resp.Header.Get("x-request-id"), Usage: usage, Model: requestModel, UpstreamModel: requestModel, Stream: parsed.Stream, ResponseHeaders: resp.Header.Clone(), Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, ImageCount: imageCount, ImageSize: parsed.SizeTier, ImageInputSize: parsed.Size, ImageOutputSizes: imageOutputSizes, }, err } return nil, err } } else { usage, imageCount, imageOutputSizes, err = s.handleOpenAIImagesOAuthNonStreamingResponse(resp, c, parsed.ResponseFormat, requestModel) if err != nil { return nil, err } } if imageCount <= 0 { imageCount = parsed.N } return &OpenAIForwardResult{ RequestID: resp.Header.Get("x-request-id"), Usage: usage, Model: requestModel, UpstreamModel: requestModel, Stream: parsed.Stream, ResponseHeaders: resp.Header.Clone(), Duration: time.Since(startTime), FirstTokenMs: firstTokenMs, ImageCount: imageCount, ImageSize: parsed.SizeTier, ImageInputSize: parsed.Size, ImageOutputSizes: imageOutputSizes, }, nil }