From 23f3d426c6b700e0e66141b4cdd678a8e1e02b4f Mon Sep 17 00:00:00 2001 From: lyen1688 Date: Tue, 26 May 2026 13:58:02 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=94=AF=E6=8C=81=E5=86=85=E5=AE=B9?= =?UTF-8?q?=E5=AE=A1=E8=AE=A1=E9=A3=8E=E9=99=A9=E9=98=88=E5=80=BC=E9=85=8D?= =?UTF-8?q?=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../admin/content_moderation_handler.go | 2 + .../internal/service/content_moderation.go | 6 + .../service/content_moderation_test.go | 31 +++++ frontend/src/api/admin/riskControl.ts | 2 + frontend/src/i18n/locales/en.ts | 6 + frontend/src/i18n/locales/zh.ts | 6 + frontend/src/views/admin/RiskControlView.vue | 127 +++++++++++++++++- .../admin/__tests__/RiskControlView.spec.ts | 37 +++++ 8 files changed, 216 insertions(+), 1 deletion(-) diff --git a/backend/internal/handler/admin/content_moderation_handler.go b/backend/internal/handler/admin/content_moderation_handler.go index defcd29d..e11ea6eb 100644 --- a/backend/internal/handler/admin/content_moderation_handler.go +++ b/backend/internal/handler/admin/content_moderation_handler.go @@ -34,6 +34,7 @@ type contentModerationConfigRequest struct { AllGroups *bool `json:"all_groups"` GroupIDs *[]int64 `json:"group_ids"` RecordNonHits *bool `json:"record_non_hits"` + Thresholds *map[string]float64 `json:"thresholds"` WorkerCount *int `json:"worker_count"` QueueSize *int `json:"queue_size"` BlockStatus *int `json:"block_status"` @@ -94,6 +95,7 @@ func (h *ContentModerationHandler) UpdateConfig(c *gin.Context) { AllGroups: req.AllGroups, GroupIDs: req.GroupIDs, RecordNonHits: req.RecordNonHits, + Thresholds: req.Thresholds, WorkerCount: req.WorkerCount, QueueSize: req.QueueSize, BlockStatus: req.BlockStatus, diff --git a/backend/internal/service/content_moderation.go b/backend/internal/service/content_moderation.go index b5a889e1..a5a84d7b 100644 --- a/backend/internal/service/content_moderation.go +++ b/backend/internal/service/content_moderation.go @@ -177,6 +177,7 @@ type ContentModerationConfigView struct { AllGroups bool `json:"all_groups"` GroupIDs []int64 `json:"group_ids"` RecordNonHits bool `json:"record_non_hits"` + Thresholds map[string]float64 `json:"thresholds"` WorkerCount int `json:"worker_count"` QueueSize int `json:"queue_size"` BlockStatus int `json:"block_status"` @@ -249,6 +250,7 @@ type UpdateContentModerationConfigInput struct { AllGroups *bool `json:"all_groups"` GroupIDs *[]int64 `json:"group_ids"` RecordNonHits *bool `json:"record_non_hits"` + Thresholds *map[string]float64 `json:"thresholds"` WorkerCount *int `json:"worker_count"` QueueSize *int `json:"queue_size"` BlockStatus *int `json:"block_status"` @@ -607,6 +609,9 @@ func (s *ContentModerationService) UpdateConfig(ctx context.Context, input Updat if input.RecordNonHits != nil { cfg.RecordNonHits = *input.RecordNonHits } + if input.Thresholds != nil { + cfg.Thresholds = mergeContentModerationThresholds(ContentModerationDefaultThresholds(), *input.Thresholds) + } if input.ClearAPIKey { cfg.APIKey = "" cfg.APIKeys = []string{} @@ -1894,6 +1899,7 @@ func (s *ContentModerationService) configView(cfg *ContentModerationConfig) *Con AllGroups: cfg.AllGroups, GroupIDs: append([]int64(nil), cfg.GroupIDs...), RecordNonHits: cfg.RecordNonHits, + Thresholds: cloneFloatMap(cfg.Thresholds), WorkerCount: cfg.WorkerCount, QueueSize: cfg.QueueSize, BlockStatus: cfg.BlockStatus, diff --git a/backend/internal/service/content_moderation_test.go b/backend/internal/service/content_moderation_test.go index 60a99318..20fce3ec 100644 --- a/backend/internal/service/content_moderation_test.go +++ b/backend/internal/service/content_moderation_test.go @@ -726,6 +726,37 @@ func TestContentModerationUpdateConfig_ReplacesAPIKeysWhenRequested(t *testing.T require.Equal(t, []string{"sk-new-only"}, saved.apiKeys()) } +func TestContentModerationUpdateConfig_SavesCustomThresholds(t *testing.T) { + cfg := defaultContentModerationConfig() + rawCfg, err := json.Marshal(cfg) + require.NoError(t, err) + + repo := &contentModerationTestSettingRepo{values: map[string]string{ + SettingKeyContentModerationConfig: string(rawCfg), + }} + svc := NewContentModerationService(repo, nil, nil, nil, nil, nil, nil) + thresholds := map[string]float64{ + "sexual": 0.72, + "harassment": 1.25, + "unknown": 0.01, + } + + view, err := svc.UpdateConfig(context.Background(), UpdateContentModerationConfigInput{ + Thresholds: &thresholds, + }) + + require.NoError(t, err) + require.Equal(t, 0.72, view.Thresholds["sexual"]) + require.Equal(t, 1.0, view.Thresholds["harassment"]) + require.NotContains(t, view.Thresholds, "unknown") + + var saved ContentModerationConfig + require.NoError(t, json.Unmarshal([]byte(repo.values[SettingKeyContentModerationConfig]), &saved)) + require.Equal(t, 0.72, saved.Thresholds["sexual"]) + require.Equal(t, 1.0, saved.Thresholds["harassment"]) + require.NotContains(t, saved.Thresholds, "unknown") +} + func TestExtractContentModerationInput_AnthropicImageSourceOnlyParticipatesInMemory(t *testing.T) { body := []byte(`{ "messages": [ diff --git a/frontend/src/api/admin/riskControl.ts b/frontend/src/api/admin/riskControl.ts index fbba96be..521114c2 100644 --- a/frontend/src/api/admin/riskControl.ts +++ b/frontend/src/api/admin/riskControl.ts @@ -24,6 +24,7 @@ export interface ContentModerationConfig { all_groups: boolean group_ids: number[] record_non_hits: boolean + thresholds: Record worker_count: number queue_size: number block_status: number @@ -98,6 +99,7 @@ export interface UpdateContentModerationConfig { all_groups?: boolean group_ids?: number[] record_non_hits?: boolean + thresholds?: Record worker_count?: number queue_size?: number block_status?: number diff --git a/frontend/src/i18n/locales/en.ts b/frontend/src/i18n/locales/en.ts index 53176a93..c016e4e3 100644 --- a/frontend/src/i18n/locales/en.ts +++ b/frontend/src/i18n/locales/en.ts @@ -2564,11 +2564,17 @@ export default { lastCleanup: 'Last cleanup: {time}', cleanupStats: 'Last cleanup deleted {hit} hits and {nonHit} non-hits', riskSwitchOff: 'System switch off', + riskThresholds: 'Risk Thresholds', + riskThresholdsHint: 'Adjust hit thresholds by OpenAI Moderations category. Scores greater than or equal to the threshold count as hits.', + riskThresholdDefault: 'Default {value}', + riskThresholdReset: 'Restore defaults', + riskThresholdPercent: 'Threshold percentage', tabs: { basic: 'Basic', scope: 'Scope', runtime: 'Runtime', response: 'Hit Notice', + riskThresholds: 'Risk Thresholds', keywords: 'Keyword Block', retention: 'Retention', }, diff --git a/frontend/src/i18n/locales/zh.ts b/frontend/src/i18n/locales/zh.ts index b293ec67..1af82e1a 100644 --- a/frontend/src/i18n/locales/zh.ts +++ b/frontend/src/i18n/locales/zh.ts @@ -2641,11 +2641,17 @@ export default { lastCleanup: '上次清理：{time}', cleanupStats: '上次清理删除命中 {hit} 条，未命中 {nonHit} 条', riskSwitchOff: '系统开关关闭', + riskThresholds: '风险阈值', + riskThresholdsHint: '按 OpenAI Moderations 分类调整命中阈值，分数达到或超过阈值即视为命中。', + riskThresholdDefault: '默认 {value}', + riskThresholdReset: '恢复默认阈值', + riskThresholdPercent: '阈值百分比', tabs: { basic: '基础', scope: '审计范围', runtime: '运行队列', response: '命中通知', + riskThresholds: '风险阈值', keywords: '关键词拦截', retention: '日志保留', }, diff --git a/frontend/src/views/admin/RiskControlView.vue b/frontend/src/views/admin/RiskControlView.vue index 4d56b492..36a04756 100644 --- a/frontend/src/views/admin/RiskControlView.vue +++ b/frontend/src/views/admin/RiskControlView.vue @@ -794,6 +794,63 @@ +
+
+
+

{{ t('admin.riskControl.riskThresholds') }}

+

{{ t('admin.riskControl.riskThresholdsHint') }}

+
+ +
+ +
+
+
+
+ +

+ {{ t('admin.riskControl.riskThresholdDefault', { value: formatThresholdPercent(row.defaultValue) }) }} +

+
+ + {{ formatThresholdPercent(row.value) }} + +
+
+ +
+ + % +
+
+
+
+
+
= { + harassment: 98, + 'harassment/threatening': 90, + hate: 65, + 'hate/threatening': 65, + illicit: 95, + 'illicit/violent': 95, + 'self-harm': 65, + 'self-harm/intent': 85, + 'self-harm/instructions': 65, + sexual: 65, + 'sexual/minors': 65, + violence: 95, + 'violence/graphic': 95, +} +const riskThresholdCategories = Object.keys(riskThresholdDefaults) const { t } = useI18n() const appStore = useAppStore() @@ -1054,6 +1132,7 @@ const configForm = reactive({ hit_retention_days: 180, non_hit_retention_days: 3, pre_hash_check_enabled: false, + thresholds: { ...riskThresholdDefaults } as Record, blocked_keywords_text: '', keyword_blocking_mode: 'keyword_and_api' as KeywordBlockingMode, model_filter_type: 'all' as ContentModerationModelFilterType, @@ -1081,6 +1160,7 @@ const settingsTabs = computed>(() => [ { id: 'scope', label: t('admin.riskControl.tabs.scope') }, { id: 'runtime', label: t('admin.riskControl.tabs.runtime') }, { id: 'response', label: t('admin.riskControl.tabs.response') }, + { id: 'riskThresholds', label: t('admin.riskControl.tabs.riskThresholds') }, { id: 'keywords', label: t('admin.riskControl.tabs.keywords') }, { id: 'retention', label: t('admin.riskControl.tabs.retention') }, ]) @@ -1373,6 +1453,14 @@ const moderationScoreRows = computed(() => { .sort((a, b) => b.score - a.score) }) +const riskThresholdRows = computed(() => ( + riskThresholdCategories.map((category) => ({ + category, + value: configForm.thresholds[category] ?? riskThresholdDefaults[category], + defaultValue: riskThresholdDefaults[category], + })) +)) + const inputDetailText = computed(() => { if (!inputDetailRow.value) return '-' return inputDetailRow.value.input_excerpt || inputDetailRow.value.error || '-' @@ -1445,6 +1533,7 @@ function applyConfig(config: ContentModerationConfig) { configForm.hit_retention_days = config.hit_retention_days || 180 configForm.non_hit_retention_days = Math.min(Math.max(config.non_hit_retention_days || 3, 1), 3) configForm.pre_hash_check_enabled = config.pre_hash_check_enabled ?? false + configForm.thresholds = riskThresholdsFromConfig(config.thresholds) configForm.blocked_keywords_text = Array.isArray(config.blocked_keywords) ? config.blocked_keywords.join('\n') : '' configForm.keyword_blocking_mode = normalizeKeywordBlockingMode(config.keyword_blocking_mode) const modelFilter = normalizeModelFilter(config.model_filter) @@ -1524,6 +1613,7 @@ async function saveConfig() { hit_retention_days: Number(configForm.hit_retention_days) || 180, non_hit_retention_days: Math.min(Math.max(Number(configForm.non_hit_retention_days) || 3, 1), 3), pre_hash_check_enabled: configForm.pre_hash_check_enabled, + thresholds: buildRiskThresholdPayload(), blocked_keywords: blockedKeywordList.value, keyword_blocking_mode: configForm.keyword_blocking_mode, model_filter: modelFilterPayload, @@ -1988,6 +2078,41 @@ function buildModelFilterPayload(): ContentModerationModelFilter { } } +function riskThresholdsFromConfig(thresholds: Record | null | undefined): Record { + const out: Record = { ...riskThresholdDefaults } + for (const category of riskThresholdCategories) { + const value = thresholds?.[category] + if (Number.isFinite(value)) { + out[category] = clampPercent(Number(value) * 100) + } + } + return out +} + +function buildRiskThresholdPayload(): Record { + const payload: Record = {} + for (const category of riskThresholdCategories) { + payload[category] = Number((clampPercent(configForm.thresholds[category]) / 100).toFixed(4)) + } + return payload +} + +function resetRiskThresholds() { + configForm.thresholds = { ...riskThresholdDefaults } +} + +function clampPercent(value: unknown): number { + const numeric = Number(value) + if (!Number.isFinite(numeric)) { + return 0 + } + return Math.min(100, Math.max(0, numeric)) +} + +function formatThresholdPercent(value: number): string { + return `${clampPercent(value).toFixed(1)}%` +} + function parseBlockedKeywords(value: string): string[] { const seen = new Set() const out: string[] = [] diff --git a/frontend/src/views/admin/__tests__/RiskControlView.spec.ts b/frontend/src/views/admin/__tests__/RiskControlView.spec.ts index b528a278..3c6aa0e9 100644 --- a/frontend/src/views/admin/__tests__/RiskControlView.spec.ts +++ b/frontend/src/views/admin/__tests__/RiskControlView.spec.ts @@ -93,6 +93,10 @@ const baseConfig = (): ContentModerationConfig => ({ pre_hash_check_enabled: false, blocked_keywords: [], keyword_blocking_mode: 'keyword_and_api', + thresholds: { + harassment: 0.98, + sexual: 0.65, + }, model_filter: { type: 'all', models: [], @@ -224,4 +228,37 @@ describe('admin RiskControlView', () => { })) expect(showError).not.toHaveBeenCalled() }) + + it('submits edited risk control thresholds when saving moderation config', async () => { + const wrapper = mount(RiskControlView, { + global: { + stubs: { + AppLayout: AppLayoutStub, + BaseDialog: BaseDialogStub, + Icon: true, + Select: true, + Toggle: true, + Pagination: true, + ModelWhitelistSelector: ModelWhitelistSelectorStub, + }, + }, + }) + + await flushPromises() + + await findButtonByText(wrapper, 'admin.riskControl.openSettings').trigger('click') + await findButtonByText(wrapper, 'admin.riskControl.tabs.riskThresholds').trigger('click') + await wrapper.get('[data-test="risk-threshold-sexual"]').setValue('72') + await wrapper.get('[data-test="risk-threshold-harassment"]').setValue('99') + await findButtonByText(wrapper, 'admin.riskControl.saveConfig').trigger('click') + await flushPromises() + + expect(updateConfig).toHaveBeenCalledWith(expect.objectContaining({ + thresholds: expect.objectContaining({ + sexual: 0.72, + harassment: 0.99, + }), + })) + expect(showError).not.toHaveBeenCalled() + }) })