fix(risk-control): Agent 工具循环中同一用户消息重复审计去重
末尾 role 检查方案：当 messages / input / contents 数组末尾一项不是用户消息 （而是 assistant、tool / function_call_output 等）时，直接跳过内容审计， 从而避免 Agent 工具循环中同一用户输入被反复审计、计费、写日志。 Fixes #2678
This commit is contained in:
wucm667
parent
16793d3af0
commit
199a5bcc69
@ -48,44 +48,44 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string,
|
||||
if !messages.IsArray() {
|
||||
return
|
||||
}
|
||||
var lastParts []string
|
||||
var lastImages []string
|
||||
messages.ForEach(func(_, msg gjson.Result) bool {
|
||||
if strings.ToLower(strings.TrimSpace(msg.Get("role").String())) == role {
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
collectContentValue(msg.Get("content"), &candidate, &candidateImages)
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) != "" || len(candidateImages) > 0 {
|
||||
lastParts = candidate
|
||||
lastImages = candidateImages
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
*parts = append(*parts, lastParts...)
|
||||
*images = append(*images, lastImages...)
|
||||
array := messages.Array()
|
||||
if len(array) == 0 {
|
||||
return
|
||||
}
|
||||
last := array[len(array)-1]
|
||||
if strings.ToLower(strings.TrimSpace(last.Get("role").String())) != role {
|
||||
return
|
||||
}
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
collectContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||
return
|
||||
}
|
||||
*parts = append(*parts, candidate...)
|
||||
*images = append(*images, candidateImages...)
|
||||
}
|
||||
|
||||
func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
|
||||
if !messages.IsArray() {
|
||||
return
|
||||
}
|
||||
var lastParts []string
|
||||
var lastImages []string
|
||||
messages.ForEach(func(_, msg gjson.Result) bool {
|
||||
if strings.ToLower(strings.TrimSpace(msg.Get("role").String())) == "user" {
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
collectAnthropicUserContentValue(msg.Get("content"), &candidate, &candidateImages)
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) != "" || len(candidateImages) > 0 {
|
||||
lastParts = candidate
|
||||
lastImages = candidateImages
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
*parts = append(*parts, lastParts...)
|
||||
*images = append(*images, lastImages...)
|
||||
array := messages.Array()
|
||||
if len(array) == 0 {
|
||||
return
|
||||
}
|
||||
last := array[len(array)-1]
|
||||
if strings.ToLower(strings.TrimSpace(last.Get("role").String())) != "user" {
|
||||
return
|
||||
}
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||
return
|
||||
}
|
||||
*parts = append(*parts, candidate...)
|
||||
*images = append(*images, candidateImages...)
|
||||
}
|
||||
|
||||
func collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||
@ -128,18 +128,17 @@ func collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]st
|
||||
case input.Type == gjson.String:
|
||||
addModerationText(parts, input.String())
|
||||
case input.IsArray():
|
||||
var last gjson.Result
|
||||
input.ForEach(func(_, item gjson.Result) bool {
|
||||
if isResponsesUserTextItem(item) {
|
||||
last = item
|
||||
}
|
||||
return true
|
||||
})
|
||||
if last.Exists() {
|
||||
collectContentValue(last.Get("content"), parts, images)
|
||||
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
|
||||
collectContentValue(last, parts, images)
|
||||
}
|
||||
array := input.Array()
|
||||
if len(array) == 0 {
|
||||
return
|
||||
}
|
||||
last := array[len(array)-1]
|
||||
if !isResponsesUserTextItem(last) {
|
||||
return
|
||||
}
|
||||
collectContentValue(last.Get("content"), parts, images)
|
||||
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
|
||||
collectContentValue(last, parts, images)
|
||||
}
|
||||
case input.IsObject():
|
||||
if isResponsesUserTextItem(input) {
|
||||
@ -176,29 +175,29 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]
|
||||
if !contents.IsArray() {
|
||||
return
|
||||
}
|
||||
var lastParts []string
|
||||
var lastImages []string
|
||||
contents.ForEach(func(_, content gjson.Result) bool {
|
||||
role := strings.ToLower(strings.TrimSpace(content.Get("role").String()))
|
||||
if role == "" || role == "user" {
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
if arr := content.Get("parts"); arr.IsArray() {
|
||||
arr.ForEach(func(_, part gjson.Result) bool {
|
||||
addModerationText(&candidate, part.Get("text").String())
|
||||
addGeminiModerationImage(&candidateImages, part)
|
||||
return true
|
||||
})
|
||||
}
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) != "" || len(candidateImages) > 0 {
|
||||
lastParts = candidate
|
||||
lastImages = candidateImages
|
||||
}
|
||||
}
|
||||
return true
|
||||
})
|
||||
*parts = append(*parts, lastParts...)
|
||||
*images = append(*images, lastImages...)
|
||||
array := contents.Array()
|
||||
if len(array) == 0 {
|
||||
return
|
||||
}
|
||||
last := array[len(array)-1]
|
||||
role := strings.ToLower(strings.TrimSpace(last.Get("role").String()))
|
||||
if role != "" && role != "user" {
|
||||
return
|
||||
}
|
||||
var candidate []string
|
||||
var candidateImages []string
|
||||
if arr := last.Get("parts"); arr.IsArray() {
|
||||
arr.ForEach(func(_, part gjson.Result) bool {
|
||||
addModerationText(&candidate, part.Get("text").String())
|
||||
addGeminiModerationImage(&candidateImages, part)
|
||||
return true
|
||||
})
|
||||
}
|
||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||
return
|
||||
}
|
||||
*parts = append(*parts, candidate...)
|
||||
*images = append(*images, candidateImages...)
|
||||
}
|
||||
|
||||
func collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||
|
||||
179
backend/internal/service/content_moderation_input_test.go
Normal file
179
backend/internal/service/content_moderation_input_test.go
Normal file
@ -0,0 +1,179 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
// 当数组末尾不是用户消息时（典型场景：Agent 工具循环结束于 tool/assistant），
|
||||
// 应直接跳过审计——不再回溯查找历史中的某条用户消息。
|
||||
|
||||
func TestExtractContentModerationInput_AnthropicAgentToolLoopSkipsAudit(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"messages": [
|
||||
{"role":"user","content":"调用一下天气工具"},
|
||||
{"role":"assistant","content":[{"type":"tool_use","id":"tool_1","name":"weather","input":{}}]},
|
||||
{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_1","content":"晴 25 度"}]}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||
|
||||
require.Empty(t, input.Text)
|
||||
require.Empty(t, input.Images)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_AnthropicFirstTurnExtractsUser(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"messages": [
|
||||
{"role":"user","content":"Q1"}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||
|
||||
require.Equal(t, "Q1", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_AnthropicMultiTurnExtractsLatestUser(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"messages": [
|
||||
{"role":"user","content":"Q1"},
|
||||
{"role":"assistant","content":"A1"},
|
||||
{"role":"user","content":"Q2"}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||
|
||||
require.Equal(t, "Q2", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_AnthropicStreamResendExtractsResend(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"messages": [
|
||||
{"role":"user","content":"原问题"},
|
||||
{"role":"assistant","content":"部分回答……"},
|
||||
{"role":"user","content":"重发"}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||
|
||||
require.Equal(t, "重发", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_OpenAIChatAgentToolLoopSkipsAudit(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"messages": [
|
||||
{"role":"system","content":"sys"},
|
||||
{"role":"user","content":"列出我的订单"},
|
||||
{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"orders","arguments":"{}"}}]},
|
||||
{"role":"tool","tool_call_id":"call_1","content":"[]"}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
|
||||
|
||||
require.Empty(t, input.Text)
|
||||
require.Empty(t, input.Images)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_OpenAIChatMultiTurnExtractsLatestUser(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"messages": [
|
||||
{"role":"user","content":"Q1"},
|
||||
{"role":"assistant","content":"A1"},
|
||||
{"role":"user","content":"Q2"}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
|
||||
|
||||
require.Equal(t, "Q2", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_GeminiAgentToolLoopSkipsAudit(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"contents": [
|
||||
{"role":"user","parts":[{"text":"查询天气"}]},
|
||||
{"role":"model","parts":[{"functionCall":{"name":"weather","args":{}}}]},
|
||||
{"role":"user","parts":[{"functionResponse":{"name":"weather","response":{"temp":25}}}]}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolGemini, body)
|
||||
|
||||
require.Empty(t, input.Text)
|
||||
require.Empty(t, input.Images)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_GeminiFirstTurnExtractsUser(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"contents": [
|
||||
{"role":"user","parts":[{"text":"你好"}]}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolGemini, body)
|
||||
|
||||
require.Equal(t, "你好", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_GeminiMultiTurnExtractsLatestUser(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"contents": [
|
||||
{"role":"user","parts":[{"text":"Q1"}]},
|
||||
{"role":"model","parts":[{"text":"A1"}]},
|
||||
{"role":"user","parts":[{"text":"Q2"}]}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolGemini, body)
|
||||
|
||||
require.Equal(t, "Q2", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_ResponsesAgentToolLoopSkipsAudit(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"input":[
|
||||
{"type":"message","role":"user","content":[{"type":"input_text","text":"运行测试"}]},
|
||||
{"type":"function_call","call_id":"call_1","name":"run_tests","arguments":"{}"},
|
||||
{"type":"function_call_output","call_id":"call_1","output":"all passed"}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||||
|
||||
require.Empty(t, input.Text)
|
||||
require.Empty(t, input.Images)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_ResponsesLastUserMessageExtracted(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"input":[
|
||||
{"type":"message","role":"user","content":[{"type":"input_text","text":"first"}]},
|
||||
{"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer"}]},
|
||||
{"type":"message","role":"user","content":[{"type":"input_text","text":"latest"}]}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||||
|
||||
require.Equal(t, "latest", input.Text)
|
||||
}
|
||||
|
||||
func TestExtractContentModerationInput_ResponsesLastIsAssistantSkipped(t *testing.T) {
|
||||
body := []byte(`{
|
||||
"input":[
|
||||
{"type":"message","role":"user","content":[{"type":"input_text","text":"q1"}]},
|
||||
{"type":"message","role":"assistant","content":[{"type":"output_text","text":"a1"}]}
|
||||
]
|
||||
}`)
|
||||
|
||||
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||||
|
||||
require.Empty(t, input.Text)
|
||||
require.Empty(t, input.Images)
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user