fix(risk-control): Agent 工具循环中同一用户消息重复审计去重
末尾 role 检查方案:当 messages / input / contents 数组末尾一项不是用户消息 (而是 assistant、tool / function_call_output 等)时,直接跳过内容审计, 从而避免 Agent 工具循环中同一用户输入被反复审计、计费、写日志。 Fixes #2678
This commit is contained in:
parent
16793d3af0
commit
199a5bcc69
@ -48,44 +48,44 @@ func collectLastRoleMessage(messages gjson.Result, role string, parts *[]string,
|
|||||||
if !messages.IsArray() {
|
if !messages.IsArray() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var lastParts []string
|
array := messages.Array()
|
||||||
var lastImages []string
|
if len(array) == 0 {
|
||||||
messages.ForEach(func(_, msg gjson.Result) bool {
|
return
|
||||||
if strings.ToLower(strings.TrimSpace(msg.Get("role").String())) == role {
|
}
|
||||||
var candidate []string
|
last := array[len(array)-1]
|
||||||
var candidateImages []string
|
if strings.ToLower(strings.TrimSpace(last.Get("role").String())) != role {
|
||||||
collectContentValue(msg.Get("content"), &candidate, &candidateImages)
|
return
|
||||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) != "" || len(candidateImages) > 0 {
|
}
|
||||||
lastParts = candidate
|
var candidate []string
|
||||||
lastImages = candidateImages
|
var candidateImages []string
|
||||||
}
|
collectContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||||
}
|
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||||
return true
|
return
|
||||||
})
|
}
|
||||||
*parts = append(*parts, lastParts...)
|
*parts = append(*parts, candidate...)
|
||||||
*images = append(*images, lastImages...)
|
*images = append(*images, candidateImages...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
|
func collectLastAnthropicUserMessage(messages gjson.Result, parts *[]string, images *[]string) {
|
||||||
if !messages.IsArray() {
|
if !messages.IsArray() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var lastParts []string
|
array := messages.Array()
|
||||||
var lastImages []string
|
if len(array) == 0 {
|
||||||
messages.ForEach(func(_, msg gjson.Result) bool {
|
return
|
||||||
if strings.ToLower(strings.TrimSpace(msg.Get("role").String())) == "user" {
|
}
|
||||||
var candidate []string
|
last := array[len(array)-1]
|
||||||
var candidateImages []string
|
if strings.ToLower(strings.TrimSpace(last.Get("role").String())) != "user" {
|
||||||
collectAnthropicUserContentValue(msg.Get("content"), &candidate, &candidateImages)
|
return
|
||||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) != "" || len(candidateImages) > 0 {
|
}
|
||||||
lastParts = candidate
|
var candidate []string
|
||||||
lastImages = candidateImages
|
var candidateImages []string
|
||||||
}
|
collectAnthropicUserContentValue(last.Get("content"), &candidate, &candidateImages)
|
||||||
}
|
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||||
return true
|
return
|
||||||
})
|
}
|
||||||
*parts = append(*parts, lastParts...)
|
*parts = append(*parts, candidate...)
|
||||||
*images = append(*images, lastImages...)
|
*images = append(*images, candidateImages...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
func collectAnthropicUserContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||||
@ -128,18 +128,17 @@ func collectLastResponsesInput(input gjson.Result, parts *[]string, images *[]st
|
|||||||
case input.Type == gjson.String:
|
case input.Type == gjson.String:
|
||||||
addModerationText(parts, input.String())
|
addModerationText(parts, input.String())
|
||||||
case input.IsArray():
|
case input.IsArray():
|
||||||
var last gjson.Result
|
array := input.Array()
|
||||||
input.ForEach(func(_, item gjson.Result) bool {
|
if len(array) == 0 {
|
||||||
if isResponsesUserTextItem(item) {
|
return
|
||||||
last = item
|
}
|
||||||
}
|
last := array[len(array)-1]
|
||||||
return true
|
if !isResponsesUserTextItem(last) {
|
||||||
})
|
return
|
||||||
if last.Exists() {
|
}
|
||||||
collectContentValue(last.Get("content"), parts, images)
|
collectContentValue(last.Get("content"), parts, images)
|
||||||
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
|
if last.Get("type").String() == "input_text" || last.Get("text").Exists() {
|
||||||
collectContentValue(last, parts, images)
|
collectContentValue(last, parts, images)
|
||||||
}
|
|
||||||
}
|
}
|
||||||
case input.IsObject():
|
case input.IsObject():
|
||||||
if isResponsesUserTextItem(input) {
|
if isResponsesUserTextItem(input) {
|
||||||
@ -176,29 +175,29 @@ func collectLastGeminiContent(contents gjson.Result, parts *[]string, images *[]
|
|||||||
if !contents.IsArray() {
|
if !contents.IsArray() {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
var lastParts []string
|
array := contents.Array()
|
||||||
var lastImages []string
|
if len(array) == 0 {
|
||||||
contents.ForEach(func(_, content gjson.Result) bool {
|
return
|
||||||
role := strings.ToLower(strings.TrimSpace(content.Get("role").String()))
|
}
|
||||||
if role == "" || role == "user" {
|
last := array[len(array)-1]
|
||||||
var candidate []string
|
role := strings.ToLower(strings.TrimSpace(last.Get("role").String()))
|
||||||
var candidateImages []string
|
if role != "" && role != "user" {
|
||||||
if arr := content.Get("parts"); arr.IsArray() {
|
return
|
||||||
arr.ForEach(func(_, part gjson.Result) bool {
|
}
|
||||||
addModerationText(&candidate, part.Get("text").String())
|
var candidate []string
|
||||||
addGeminiModerationImage(&candidateImages, part)
|
var candidateImages []string
|
||||||
return true
|
if arr := last.Get("parts"); arr.IsArray() {
|
||||||
})
|
arr.ForEach(func(_, part gjson.Result) bool {
|
||||||
}
|
addModerationText(&candidate, part.Get("text").String())
|
||||||
if normalizeContentModerationText(strings.Join(candidate, "\n")) != "" || len(candidateImages) > 0 {
|
addGeminiModerationImage(&candidateImages, part)
|
||||||
lastParts = candidate
|
return true
|
||||||
lastImages = candidateImages
|
})
|
||||||
}
|
}
|
||||||
}
|
if normalizeContentModerationText(strings.Join(candidate, "\n")) == "" && len(candidateImages) == 0 {
|
||||||
return true
|
return
|
||||||
})
|
}
|
||||||
*parts = append(*parts, lastParts...)
|
*parts = append(*parts, candidate...)
|
||||||
*images = append(*images, lastImages...)
|
*images = append(*images, candidateImages...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
func collectContentValue(value gjson.Result, parts *[]string, images *[]string) {
|
||||||
|
|||||||
179
backend/internal/service/content_moderation_input_test.go
Normal file
179
backend/internal/service/content_moderation_input_test.go
Normal file
@ -0,0 +1,179 @@
|
|||||||
|
package service
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/stretchr/testify/require"
|
||||||
|
)
|
||||||
|
|
||||||
|
// 当数组末尾不是用户消息时(典型场景:Agent 工具循环结束于 tool/assistant),
|
||||||
|
// 应直接跳过审计——不再回溯查找历史中的某条用户消息。
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_AnthropicAgentToolLoopSkipsAudit(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"messages": [
|
||||||
|
{"role":"user","content":"调用一下天气工具"},
|
||||||
|
{"role":"assistant","content":[{"type":"tool_use","id":"tool_1","name":"weather","input":{}}]},
|
||||||
|
{"role":"user","content":[{"type":"tool_result","tool_use_id":"tool_1","content":"晴 25 度"}]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||||
|
|
||||||
|
require.Empty(t, input.Text)
|
||||||
|
require.Empty(t, input.Images)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_AnthropicFirstTurnExtractsUser(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"messages": [
|
||||||
|
{"role":"user","content":"Q1"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||||
|
|
||||||
|
require.Equal(t, "Q1", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_AnthropicMultiTurnExtractsLatestUser(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"messages": [
|
||||||
|
{"role":"user","content":"Q1"},
|
||||||
|
{"role":"assistant","content":"A1"},
|
||||||
|
{"role":"user","content":"Q2"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||||
|
|
||||||
|
require.Equal(t, "Q2", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_AnthropicStreamResendExtractsResend(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"messages": [
|
||||||
|
{"role":"user","content":"原问题"},
|
||||||
|
{"role":"assistant","content":"部分回答……"},
|
||||||
|
{"role":"user","content":"重发"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolAnthropicMessages, body)
|
||||||
|
|
||||||
|
require.Equal(t, "重发", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_OpenAIChatAgentToolLoopSkipsAudit(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"messages": [
|
||||||
|
{"role":"system","content":"sys"},
|
||||||
|
{"role":"user","content":"列出我的订单"},
|
||||||
|
{"role":"assistant","content":null,"tool_calls":[{"id":"call_1","type":"function","function":{"name":"orders","arguments":"{}"}}]},
|
||||||
|
{"role":"tool","tool_call_id":"call_1","content":"[]"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
|
||||||
|
|
||||||
|
require.Empty(t, input.Text)
|
||||||
|
require.Empty(t, input.Images)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_OpenAIChatMultiTurnExtractsLatestUser(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"messages": [
|
||||||
|
{"role":"user","content":"Q1"},
|
||||||
|
{"role":"assistant","content":"A1"},
|
||||||
|
{"role":"user","content":"Q2"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIChat, body)
|
||||||
|
|
||||||
|
require.Equal(t, "Q2", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_GeminiAgentToolLoopSkipsAudit(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"contents": [
|
||||||
|
{"role":"user","parts":[{"text":"查询天气"}]},
|
||||||
|
{"role":"model","parts":[{"functionCall":{"name":"weather","args":{}}}]},
|
||||||
|
{"role":"user","parts":[{"functionResponse":{"name":"weather","response":{"temp":25}}}]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolGemini, body)
|
||||||
|
|
||||||
|
require.Empty(t, input.Text)
|
||||||
|
require.Empty(t, input.Images)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_GeminiFirstTurnExtractsUser(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"contents": [
|
||||||
|
{"role":"user","parts":[{"text":"你好"}]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolGemini, body)
|
||||||
|
|
||||||
|
require.Equal(t, "你好", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_GeminiMultiTurnExtractsLatestUser(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"contents": [
|
||||||
|
{"role":"user","parts":[{"text":"Q1"}]},
|
||||||
|
{"role":"model","parts":[{"text":"A1"}]},
|
||||||
|
{"role":"user","parts":[{"text":"Q2"}]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolGemini, body)
|
||||||
|
|
||||||
|
require.Equal(t, "Q2", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_ResponsesAgentToolLoopSkipsAudit(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"input":[
|
||||||
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"运行测试"}]},
|
||||||
|
{"type":"function_call","call_id":"call_1","name":"run_tests","arguments":"{}"},
|
||||||
|
{"type":"function_call_output","call_id":"call_1","output":"all passed"}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||||||
|
|
||||||
|
require.Empty(t, input.Text)
|
||||||
|
require.Empty(t, input.Images)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_ResponsesLastUserMessageExtracted(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"input":[
|
||||||
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"first"}]},
|
||||||
|
{"type":"message","role":"assistant","content":[{"type":"output_text","text":"answer"}]},
|
||||||
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"latest"}]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||||||
|
|
||||||
|
require.Equal(t, "latest", input.Text)
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestExtractContentModerationInput_ResponsesLastIsAssistantSkipped(t *testing.T) {
|
||||||
|
body := []byte(`{
|
||||||
|
"input":[
|
||||||
|
{"type":"message","role":"user","content":[{"type":"input_text","text":"q1"}]},
|
||||||
|
{"type":"message","role":"assistant","content":[{"type":"output_text","text":"a1"}]}
|
||||||
|
]
|
||||||
|
}`)
|
||||||
|
|
||||||
|
input := ExtractContentModerationInput(ContentModerationProtocolOpenAIResponses, body)
|
||||||
|
|
||||||
|
require.Empty(t, input.Text)
|
||||||
|
require.Empty(t, input.Images)
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user