fix(ws): exclude terminal events from first-token detection

isOpenAIWSTokenEvent classified response.completed / response.done as
token events. When upstream finishes a request without ever emitting
a recognizable delta (e.g. cached completions or models that skip
incremental output), firstTokenMs was then filled at the terminal
event's timestamp, so the first-token latency metric effectively
reported total request duration.

Terminal events are already handled separately by
isOpenAIWSTerminalEvent. Treating them as token events makes the two
classifiers overlap, which violates the implicit invariant that the
token-event and terminal-event sets are disjoint.

The metric only affects ForwardResult.FirstTokenMs (logging and
observability) — billing and routing are unchanged.

Add regression tests for both directions:

* TestIsOpenAIWSTokenEvent_TerminalEventsExcluded covers each
  classification branch.
* TestIsOpenAIWSTokenEvent_DisjointWithTerminal asserts the
  disjoint-set invariant for every known terminal event.

Both new tests fail when the old `return eventType == "response.completed"
|| eventType == "response.done"` is restored.

Fixes #2651

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Pluviobyte 2026-05-29 01:33:42 +00:00
parent 89d96f4b25
commit 8a999f438d
No known key found for this signature in database
2 changed files with 79 additions and 1 deletions

View File

@ -3915,7 +3915,10 @@ func isOpenAIWSTokenEvent(eventType string) bool {
if strings.HasPrefix(eventType, "response.output") {
return true
}
return eventType == "response.completed" || eventType == "response.done"
// 终止事件response.completed/done/failed/...)由 isOpenAIWSTerminalEvent 单独处理。
// 不能把它们当作 token event否则当上游没有可识别的 delta 时,
// firstTokenMs 会被填到终止时刻,等于把"总耗时"误报为"首 token 延迟"。
return false
}
func replaceOpenAIWSMessageModel(message []byte, fromModel, toModel string) []byte {

View File

@ -0,0 +1,75 @@
package service
import (
"testing"
"github.com/stretchr/testify/require"
)
// TestIsOpenAIWSTokenEvent_TerminalEventsExcluded 覆盖 isOpenAIWSTokenEvent 的回归用例。
// 重点验证终止事件response.completed / response.done不再被当作 token event
// 否则当上游没有可识别的 delta 时firstTokenMs 会被填到终止时刻,
// 等于把"总耗时"误报为"首 token 延迟"issue #2651
func TestIsOpenAIWSTokenEvent_TerminalEventsExcluded(t *testing.T) {
cases := []struct {
name string
eventType string
want bool
}{
{name: "empty", eventType: "", want: false},
{name: "whitespace_trimmed_empty", eventType: " ", want: false},
{name: "response.created", eventType: "response.created", want: false},
{name: "response.in_progress", eventType: "response.in_progress", want: false},
{name: "response.output_item.added", eventType: "response.output_item.added", want: false},
{name: "response.output_item.done", eventType: "response.output_item.done", want: false},
{name: "terminal_response.completed", eventType: "response.completed", want: false},
{name: "terminal_response.done", eventType: "response.done", want: false},
{name: "terminal_response.completed_padded", eventType: " response.completed ", want: false},
{name: "terminal_response.done_padded", eventType: " response.done ", want: false},
{name: "delta_text", eventType: "response.output_text.delta", want: true},
{name: "delta_audio_transcript", eventType: "response.audio_transcript.delta", want: true},
{name: "delta_function_call_arguments", eventType: "response.function_call_arguments.delta", want: true},
{name: "output_text_done", eventType: "response.output_text.done", want: true},
{name: "output_text_annotation_added", eventType: "response.output_text.annotation.added", want: true},
{name: "output_audio_done", eventType: "response.output_audio.done", want: true},
{name: "reasoning_summary_delta", eventType: "response.reasoning_summary_text.delta", want: true},
{name: "unrelated_event_error", eventType: "error", want: false},
{name: "unknown_event_without_match", eventType: "response.reasoning_summary_part.added", want: false},
}
for _, tc := range cases {
tc := tc
t.Run(tc.name, func(t *testing.T) {
got := isOpenAIWSTokenEvent(tc.eventType)
require.Equal(t, tc.want, got, "isOpenAIWSTokenEvent(%q)", tc.eventType)
})
}
}
// TestIsOpenAIWSTokenEvent_DisjointWithTerminal 守护「token 事件集合与终止事件集合互斥」的不变量。
// firstTokenMs 的计算依赖于 isTokenEvent && !isTerminalEvent
// 若两者再次出现交集,则 issue #2651 描述的 latency 误报会重现。
func TestIsOpenAIWSTokenEvent_DisjointWithTerminal(t *testing.T) {
terminalEvents := []string{
"response.completed",
"response.done",
"response.failed",
"response.incomplete",
"response.cancelled",
"response.canceled",
}
for _, ev := range terminalEvents {
ev := ev
t.Run(ev, func(t *testing.T) {
require.True(t, isOpenAIWSTerminalEvent(ev), "expected terminal event %q to be classified as terminal", ev)
require.False(t, isOpenAIWSTokenEvent(ev), "terminal event %q must NOT be classified as token event (issue #2651)", ev)
})
}
}