Merge pull request #2790 from Arron196/from-arron-main

修复 Ops SLA 本地限制错误统计
This commit is contained in:
Wesley Liddick 2026-05-26 20:21:11 +08:00 committed by GitHub
commit 4b9b63443f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 542 additions and 22 deletions

View File

@ -781,6 +781,7 @@ func (h *GatewayHandler) Messages(c *gin.Context) {
// Beta policy block: return 400 immediately, no failover // Beta policy block: return 400 immediately, no failover
var betaBlockedErr *service.BetaBlockedError var betaBlockedErr *service.BetaBlockedError
if errors.As(err, &betaBlockedErr) { if errors.As(err, &betaBlockedErr) {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalPolicyDenied)
h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", betaBlockedErr.Message) h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", betaBlockedErr.Message)
return return
} }

View File

@ -53,6 +53,8 @@ const (
opsCodeUserNotFound = "USER_NOT_FOUND" opsCodeUserNotFound = "USER_NOT_FOUND"
opsCodeAPIKeyQuotaExhausted = "API_KEY_QUOTA_EXHAUSTED" opsCodeAPIKeyQuotaExhausted = "API_KEY_QUOTA_EXHAUSTED"
opsCodeAPIKeyQueryDeprecated = "api_key_in_query_deprecated" opsCodeAPIKeyQueryDeprecated = "api_key_in_query_deprecated"
opsCodeGroupDeleted = "GROUP_DELETED"
opsCodeGroupDisabled = "GROUP_DISABLED"
) )
const ( const (
@ -1012,6 +1014,8 @@ func parseOpsErrorResponse(body []byte) parsedOpsError {
var code string var code string
if v, ok := errObj["code"]; ok { if v, ok := errObj["code"]; ok {
switch n := v.(type) { switch n := v.(type) {
case string:
code = strings.TrimSpace(n)
case float64: case float64:
code = strconvItoa(int(n)) code = strconvItoa(int(n))
case int: case int:
@ -1188,14 +1192,19 @@ func isOpsClientAuthError(code string, msg string) bool {
opsCodeAPIKeyExpired, opsCodeAPIKeyExpired,
opsCodeAPIKeyDisabled, opsCodeAPIKeyDisabled,
opsCodeUserNotFound, opsCodeUserNotFound,
opsCodeUserInactive: opsCodeUserInactive,
opsCodeGroupDeleted,
opsCodeGroupDisabled:
return true return true
} }
return strings.Contains(msg, "invalid api key") || return strings.Contains(msg, "invalid api key") ||
strings.Contains(msg, "api key is required") || strings.Contains(msg, "api key is required") ||
strings.Contains(msg, "api key is disabled") || strings.Contains(msg, "api key is disabled") ||
strings.Contains(msg, "user associated with api key not found") || strings.Contains(msg, "user associated with api key not found") ||
strings.Contains(msg, "user account is not active") strings.Contains(msg, "user account is not active") ||
strings.Contains(msg, "api key 所属分组已删除") ||
strings.Contains(msg, "api key 所属分组已停用") ||
strings.Contains(msg, "api key is not assigned to any group")
} }
func isOpsLocalBusinessLimitError(code string, msg string) bool { func isOpsLocalBusinessLimitError(code string, msg string) bool {
@ -1211,6 +1220,7 @@ func isOpsLocalBusinessLimitError(code string, msg string) bool {
return strings.Contains(msg, "api key in query parameter is deprecated") || return strings.Contains(msg, "api key in query parameter is deprecated") ||
strings.Contains(msg, "query parameter api_key is deprecated") || strings.Contains(msg, "query parameter api_key is deprecated") ||
strings.Contains(msg, "no active subscription found for this group") || strings.Contains(msg, "no active subscription found for this group") ||
strings.Contains(msg, "subscription is invalid or expired") ||
strings.Contains(msg, opsErrInsufficientBalance) || strings.Contains(msg, opsErrInsufficientBalance) ||
strings.Contains(msg, "insufficient account balance") || strings.Contains(msg, "insufficient account balance") ||
strings.Contains(msg, "api key group platform is not gemini") || strings.Contains(msg, "api key group platform is not gemini") ||
@ -1221,7 +1231,22 @@ func isOpsLocalBusinessLimitError(code string, msg string) bool {
strings.Contains(msg, "daily usage limit exceeded") || strings.Contains(msg, "daily usage limit exceeded") ||
strings.Contains(msg, "weekly usage limit exceeded") || strings.Contains(msg, "weekly usage limit exceeded") ||
strings.Contains(msg, "monthly usage limit exceeded") || strings.Contains(msg, "monthly usage limit exceeded") ||
strings.Contains(msg, "requests-per-minute limit exceeded") strings.Contains(msg, "usage quota exhausted for this platform") ||
strings.Contains(msg, "requests-per-minute limit exceeded") ||
strings.Contains(msg, "too many pending requests") ||
strings.Contains(msg, "concurrency limit exceeded") ||
strings.Contains(msg, "image generation concurrency limit exceeded") ||
strings.Contains(msg, "this group is restricted to claude code clients") ||
strings.Contains(msg, "this group does not allow /v1/messages dispatch") ||
strings.Contains(msg, "image generation is not enabled for this group") ||
strings.Contains(msg, "token counting is not supported for this platform") ||
strings.Contains(msg, "images api is not supported for this platform") ||
(strings.Contains(msg, "model ") && strings.Contains(msg, " not in whitelist")) ||
(strings.Contains(msg, "beta feature ") && strings.Contains(msg, " is not allowed")) ||
(strings.Contains(msg, "openai service_tier=") && strings.Contains(msg, " is not allowed for model")) ||
strings.Contains(msg, "this account only allows codex official clients") ||
strings.Contains(msg, "openai wsv1 is temporarily unsupported") ||
strings.Contains(msg, "openai codex passthrough requires a non-empty instructions field")
} }
func hasOpsUpstreamErrorContext(c *gin.Context) bool { func hasOpsUpstreamErrorContext(c *gin.Context) bool {

View File

@ -288,6 +288,34 @@ func TestClassifyOpsAuthClientErrorsExcludedFromSLA(t *testing.T) {
code: "USER_INACTIVE", code: "USER_INACTIVE",
status: http.StatusUnauthorized, status: http.StatusUnauthorized,
}, },
{
name: "deleted local API key group",
errType: "api_error",
message: "API Key 所属分组已删除",
code: "GROUP_DELETED",
status: http.StatusForbidden,
},
{
name: "disabled local API key group",
errType: "api_error",
message: "API Key 所属分组已停用",
code: "GROUP_DISABLED",
status: http.StatusForbidden,
},
{
name: "google deleted API key group message without semantic code",
errType: "api_error",
message: "API Key 所属分组已删除",
code: "403",
status: http.StatusForbidden,
},
{
name: "anthropic unassigned API key group",
errType: "permission_error",
message: "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.",
code: "",
status: http.StatusForbidden,
},
{ {
name: "google invalid API key", name: "google invalid API key",
errType: "api_error", errType: "api_error",
@ -389,6 +417,15 @@ func TestClassifyOpsLocalBusinessLimitErrorsExcludedFromSLA(t *testing.T) {
wantErrType: "api_error", wantErrType: "api_error",
wantPhase: "request", wantPhase: "request",
}, },
{
name: "gateway subscription invalid cache recheck",
errType: "billing_error",
message: "subscription is invalid or expired",
code: "billing_error",
status: http.StatusForbidden,
wantErrType: "billing_error",
wantPhase: "request",
},
{ {
name: "google insufficient account balance", name: "google insufficient account balance",
errType: "api_error", errType: "api_error",
@ -443,6 +480,132 @@ func TestClassifyOpsLocalBusinessLimitErrorsExcludedFromSLA(t *testing.T) {
wantErrType: "api_error", wantErrType: "api_error",
wantPhase: "request", wantPhase: "request",
}, },
{
name: "user platform daily quota exhausted",
errType: "api_error",
message: "Daily usage quota exhausted for this platform.",
code: "rate_limit_exceeded",
status: http.StatusTooManyRequests,
wantErrType: "api_error",
wantPhase: "request",
},
{
name: "local pending queue limit",
errType: "rate_limit_error",
message: "Too many pending requests, please retry later",
code: "",
status: http.StatusTooManyRequests,
wantErrType: "rate_limit_error",
wantPhase: "request",
},
{
name: "local concurrency limit",
errType: "rate_limit_error",
message: "Concurrency limit exceeded for user, please retry later",
code: "",
status: http.StatusTooManyRequests,
wantErrType: "rate_limit_error",
wantPhase: "request",
},
{
name: "group claude code only feature gate",
errType: "permission_error",
message: "This group is restricted to Claude Code clients (/v1/messages only)",
code: "",
status: http.StatusForbidden,
wantErrType: "api_error",
wantPhase: "request",
},
{
name: "group image generation feature gate",
errType: "permission_error",
message: "Image generation is not enabled for this group",
code: "",
status: http.StatusForbidden,
wantErrType: "api_error",
wantPhase: "request",
},
{
name: "route token counting platform unsupported",
errType: "not_found_error",
message: "Token counting is not supported for this platform",
code: "",
status: http.StatusNotFound,
wantErrType: "not_found_error",
wantPhase: "request",
},
{
name: "route images API platform unsupported",
errType: "not_found_error",
message: "Images API is not supported for this platform",
code: "",
status: http.StatusNotFound,
wantErrType: "not_found_error",
wantPhase: "request",
},
{
name: "antigravity model whitelist feature gate",
errType: "permission_error",
message: "model claude-3-5-sonnet not in whitelist",
code: "",
status: http.StatusForbidden,
wantErrType: "api_error",
wantPhase: "request",
},
{
name: "google antigravity model whitelist feature gate",
errType: "api_error",
message: "model gemini-2.5-pro not in whitelist",
code: "403",
status: http.StatusForbidden,
wantErrType: "api_error",
wantPhase: "request",
},
{
name: "claude beta policy block",
errType: "invalid_request_error",
message: "beta feature interleaved-thinking-2025-05-14 is not allowed",
code: "",
status: http.StatusBadRequest,
wantErrType: "invalid_request_error",
wantPhase: "request",
},
{
name: "openai fast policy block",
errType: "permission_error",
message: "openai service_tier=priority is not allowed for model gpt-5.5",
code: "",
status: http.StatusForbidden,
wantErrType: "api_error",
wantPhase: "request",
},
{
name: "codex official client policy block",
errType: "forbidden_error",
message: "This account only allows Codex official clients",
code: "",
status: http.StatusForbidden,
wantErrType: "forbidden_error",
wantPhase: "request",
},
{
name: "openai wsv1 unsupported feature gate",
errType: "invalid_request_error",
message: "OpenAI WSv1 is temporarily unsupported. Please enable responses_websockets_v2.",
code: "",
status: http.StatusBadRequest,
wantErrType: "invalid_request_error",
wantPhase: "request",
},
{
name: "openai passthrough instructions policy block",
errType: "forbidden_error",
message: "OpenAI codex passthrough requires a non-empty instructions field",
code: "",
status: http.StatusForbidden,
wantErrType: "forbidden_error",
wantPhase: "request",
},
} }
for _, tt := range tests { for _, tt := range tests {
@ -479,6 +642,22 @@ func TestClassifyOpsIPRestrictionAccessDeniedExcludedFromSLA(t *testing.T) {
require.Equal(t, "client_request", errorSource) require.Equal(t, "client_request", errorSource)
} }
func TestClassifyOpsClientBusinessLimitedMarkerExcludesCustomPolicyDenialFromSLA(t *testing.T) {
gin.SetMode(gin.TestMode)
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalPolicyDenied)
errType := normalizeOpsErrorType("invalid_request_error", "")
phase, isBusinessLimited, errorOwner, errorSource := classifyOpsErrorLog(c, errType, "custom admin policy message", "", http.StatusBadRequest)
require.Equal(t, "invalid_request_error", errType)
require.Equal(t, "auth", phase)
require.True(t, isBusinessLimited)
require.Equal(t, "client", errorOwner)
require.Equal(t, "client_request", errorSource)
}
func TestClassifyOpsOtherErrorsStillCountForSLA(t *testing.T) { func TestClassifyOpsOtherErrorsStillCountForSLA(t *testing.T) {
gin.SetMode(gin.TestMode) gin.SetMode(gin.TestMode)
rec := httptest.NewRecorder() rec := httptest.NewRecorder()
@ -583,6 +762,78 @@ func TestClassifyOpsUpstreamAuthTextStillCountsForSLA(t *testing.T) {
code: "API_KEY_QUOTA_EXHAUSTED", code: "API_KEY_QUOTA_EXHAUSTED",
status: http.StatusTooManyRequests, status: http.StatusTooManyRequests,
}, },
{
name: "provider deleted group shaped error",
message: "API Key 所属分组已删除",
code: "GROUP_DELETED",
status: http.StatusForbidden,
},
{
name: "provider unassigned group shaped error",
message: "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.",
code: "403",
status: http.StatusForbidden,
},
{
name: "provider local quota shaped error",
message: "Daily usage quota exhausted for this platform.",
code: "rate_limit_exceeded",
status: http.StatusTooManyRequests,
},
{
name: "provider feature gate shaped error",
message: "Image generation is not enabled for this group",
code: "403",
status: http.StatusForbidden,
},
{
name: "provider token counting unsupported shaped error",
message: "Token counting is not supported for this platform",
code: "404",
status: http.StatusNotFound,
},
{
name: "provider image API unsupported shaped error",
message: "Images API is not supported for this platform",
code: "404",
status: http.StatusNotFound,
},
{
name: "provider antigravity whitelist shaped error",
message: "model claude-3-5-sonnet not in whitelist",
code: "403",
status: http.StatusForbidden,
},
{
name: "provider beta policy shaped error",
message: "beta feature interleaved-thinking-2025-05-14 is not allowed",
code: "400",
status: http.StatusBadRequest,
},
{
name: "provider openai fast policy shaped error",
message: "openai service_tier=priority is not allowed for model gpt-5.5",
code: "403",
status: http.StatusForbidden,
},
{
name: "provider codex client policy shaped error",
message: "This account only allows Codex official clients",
code: "403",
status: http.StatusForbidden,
},
{
name: "provider wsv1 unsupported shaped error",
message: "OpenAI WSv1 is temporarily unsupported. Please enable responses_websockets_v2.",
code: "400",
status: http.StatusBadRequest,
},
{
name: "provider passthrough instructions shaped error",
message: "OpenAI codex passthrough requires a non-empty instructions field",
code: "403",
status: http.StatusForbidden,
},
} }
for _, tt := range tests { for _, tt := range tests {
@ -628,6 +879,14 @@ func TestClassifyOpsUpstreamNoAvailableTextStillCountsForSLA(t *testing.T) {
require.Equal(t, "upstream_http", errorSource) require.Equal(t, "upstream_http", errorSource)
} }
func TestParseOpsErrorResponsePreservesNestedStringCode(t *testing.T) {
parsed := parseOpsErrorResponse([]byte(`{"error":{"type":"permission_error","code":"GROUP_DELETED","message":"API Key 所属分组已删除"}}`))
require.Equal(t, "permission_error", parsed.ErrorType)
require.Equal(t, "GROUP_DELETED", parsed.Code)
require.Equal(t, "API Key 所属分组已删除", parsed.Message)
}
func TestSetOpsEndpointContext_SetsContextKeys(t *testing.T) { func TestSetOpsEndpointContext_SetsContextKeys(t *testing.T) {
gin.SetMode(gin.TestMode) gin.SetMode(gin.TestMode)
rec := httptest.NewRecorder() rec := httptest.NewRecorder()

View File

@ -263,6 +263,7 @@ func abortIfAPIKeyGroupUnavailable(c *gin.Context, apiKey *service.APIKey) bool
if ok { if ok {
return false return false
} }
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
AbortWithError(c, 403, code, message) AbortWithError(c, 403, code, message)
return true return true
} }

View File

@ -55,6 +55,7 @@ func APIKeyAuthWithSubscriptionGoogle(apiKeyService *service.APIKeyService, subs
return return
} }
if _, message, ok := validateAPIKeyGroupAvailable(apiKey); !ok { if _, message, ok := validateAPIKeyGroupAvailable(apiKey); !ok {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
abortWithGoogleError(c, 403, message) abortWithGoogleError(c, 403, message)
return return
} }

View File

@ -373,6 +373,68 @@ func TestApiKeyAuthWithSubscriptionGoogle_InvalidKey(t *testing.T) {
require.Equal(t, "UNAUTHENTICATED", resp.Error.Status) require.Equal(t, "UNAUTHENTICATED", resp.Error.Status)
} }
func TestApiKeyAuthWithSubscriptionGoogle_MarksUnavailableGroupBusinessLimited(t *testing.T) {
gin.SetMode(gin.TestMode)
groupID := int64(101)
user := &service.User{
ID: 7,
Role: service.RoleUser,
Status: service.StatusActive,
Balance: 10,
Concurrency: 3,
}
apiKey := &service.APIKey{
ID: 100,
UserID: user.ID,
GroupID: &groupID,
Key: "google-group-deleted",
Status: service.StatusActive,
User: user,
Group: &service.Group{
ID: groupID,
Name: "deleted",
Status: "deleted",
Platform: service.PlatformGemini,
Hydrated: true,
},
}
r := gin.New()
var markedBusinessLimited bool
var businessLimitedReason string
r.Use(func(c *gin.Context) {
c.Next()
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
businessLimitedReason, _ = v.(string)
}
})
apiKeyService := newTestAPIKeyService(fakeAPIKeyRepo{
getByKey: func(ctx context.Context, key string) (*service.APIKey, error) {
if key != apiKey.Key {
return nil, service.ErrAPIKeyNotFound
}
clone := *apiKey
return &clone, nil
},
})
r.Use(APIKeyAuthWithSubscriptionGoogle(apiKeyService, nil, &config.Config{RunMode: config.RunModeSimple}))
r.GET("/v1beta/test", func(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) })
req := httptest.NewRequest(http.MethodGet, "/v1beta/test", nil)
req.Header.Set("x-goog-api-key", apiKey.Key)
rec := httptest.NewRecorder()
r.ServeHTTP(rec, req)
require.Equal(t, http.StatusForbidden, rec.Code)
var resp googleErrorResponse
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
require.Equal(t, "API Key 所属分组已删除", resp.Error.Message)
require.True(t, markedBusinessLimited)
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable, businessLimitedReason)
}
func TestApiKeyAuthWithSubscriptionGoogle_RepoError(t *testing.T) { func TestApiKeyAuthWithSubscriptionGoogle_RepoError(t *testing.T) {
gin.SetMode(gin.TestMode) gin.SetMode(gin.TestMode)

View File

@ -317,6 +317,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
group *service.Group group *service.Group
wantStatus int wantStatus int
wantCode string wantCode string
wantMarked bool
}{ }{
{ {
name: "active group passes", name: "active group passes",
@ -340,6 +341,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
}, },
wantStatus: http.StatusForbidden, wantStatus: http.StatusForbidden,
wantCode: "GROUP_DISABLED", wantCode: "GROUP_DISABLED",
wantMarked: true,
}, },
{ {
name: "deleted status group is forbidden", name: "deleted status group is forbidden",
@ -352,12 +354,14 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
}, },
wantStatus: http.StatusForbidden, wantStatus: http.StatusForbidden,
wantCode: "GROUP_DELETED", wantCode: "GROUP_DELETED",
wantMarked: true,
}, },
{ {
name: "missing group edge is forbidden", name: "missing group edge is forbidden",
group: nil, group: nil,
wantStatus: http.StatusForbidden, wantStatus: http.StatusForbidden,
wantCode: "GROUP_DELETED", wantCode: "GROUP_DELETED",
wantMarked: true,
}, },
} }
@ -383,7 +387,20 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
} }
cfg := &config.Config{RunMode: config.RunModeStandard} cfg := &config.Config{RunMode: config.RunModeStandard}
apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg) apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg)
router := newAuthTestRouter(apiKeyService, nil, cfg) router := gin.New()
var markedBusinessLimited bool
var businessLimitedReason string
router.Use(func(c *gin.Context) {
c.Next()
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
businessLimitedReason, _ = v.(string)
}
})
router.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(apiKeyService, nil, cfg)))
router.GET("/t", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"ok": true})
})
w := httptest.NewRecorder() w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/t", nil) req := httptest.NewRequest(http.MethodGet, "/t", nil)
@ -394,10 +411,57 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
if tt.wantCode != "" { if tt.wantCode != "" {
require.Contains(t, w.Body.String(), tt.wantCode) require.Contains(t, w.Body.String(), tt.wantCode)
} }
require.Equal(t, tt.wantMarked, markedBusinessLimited)
if tt.wantMarked {
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable, businessLimitedReason)
}
}) })
} }
} }
func TestRequireGroupAssignmentMarksUngroupedKeyBusinessLimited(t *testing.T) {
gin.SetMode(gin.TestMode)
settingService := service.NewSettingService(fakeSettingRepo{
values: map[string]string{
service.SettingKeyAllowUngroupedKeyScheduling: "false",
},
}, &config.Config{})
apiKey := &service.APIKey{
ID: 100,
Key: "ungrouped-key",
Status: service.StatusActive,
}
router := gin.New()
var markedBusinessLimited bool
var businessLimitedReason string
router.Use(func(c *gin.Context) {
c.Next()
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
businessLimitedReason, _ = v.(string)
}
})
router.Use(func(c *gin.Context) {
c.Set(string(ContextKeyAPIKey), apiKey)
c.Next()
})
router.Use(RequireGroupAssignment(settingService, AnthropicErrorWriter))
router.GET("/t", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"ok": true})
})
w := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/t", nil)
router.ServeHTTP(w, req)
require.Equal(t, http.StatusForbidden, w.Code)
require.Contains(t, w.Body.String(), "not assigned to any group")
require.True(t, markedBusinessLimited)
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned, businessLimitedReason)
}
func TestAPIKeyAuthIPRestrictionDoesNotTrustForwardedClientIPByDefault(t *testing.T) { func TestAPIKeyAuthIPRestrictionDoesNotTrustForwardedClientIPByDefault(t *testing.T) {
gin.SetMode(gin.TestMode) gin.SetMode(gin.TestMode)
@ -771,6 +835,41 @@ type stubUserSubscriptionRepo struct {
resetMonthly func(ctx context.Context, id int64, start time.Time) error resetMonthly func(ctx context.Context, id int64, start time.Time) error
} }
type fakeSettingRepo struct {
values map[string]string
}
func (r fakeSettingRepo) Get(ctx context.Context, key string) (*service.Setting, error) {
return nil, errors.New("not implemented")
}
func (r fakeSettingRepo) GetValue(ctx context.Context, key string) (string, error) {
if v, ok := r.values[key]; ok {
return v, nil
}
return "", service.ErrSettingNotFound
}
func (r fakeSettingRepo) Set(ctx context.Context, key, value string) error {
return errors.New("not implemented")
}
func (r fakeSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
return nil, errors.New("not implemented")
}
func (r fakeSettingRepo) SetMultiple(ctx context.Context, settings map[string]string) error {
return errors.New("not implemented")
}
func (r fakeSettingRepo) GetAll(ctx context.Context) (map[string]string, error) {
return nil, errors.New("not implemented")
}
func (r fakeSettingRepo) Delete(ctx context.Context, key string) error {
return errors.New("not implemented")
}
func (r *stubUserSubscriptionRepo) Create(ctx context.Context, sub *service.UserSubscription) error { func (r *stubUserSubscriptionRepo) Create(ctx context.Context, sub *service.UserSubscription) error {
return errors.New("not implemented") return errors.New("not implemented")
} }

View File

@ -115,6 +115,7 @@ func RequireGroupAssignment(settingService *service.SettingService, writeError G
c.Next() c.Next()
return return
} }
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned)
writeError(c, http.StatusForbidden, "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.") writeError(c, http.StatusForbidden, "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.")
c.Abort() c.Abort()
} }

View File

@ -51,6 +51,7 @@ func RegisterGatewayRoutes(
// /v1/messages/count_tokens: OpenAI groups get 404 // /v1/messages/count_tokens: OpenAI groups get 404
gateway.POST("/messages/count_tokens", func(c *gin.Context) { gateway.POST("/messages/count_tokens", func(c *gin.Context) {
if getGroupPlatform(c) == service.PlatformOpenAI { if getGroupPlatform(c) == service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{ c.JSON(http.StatusNotFound, gin.H{
"type": "error", "type": "error",
"error": gin.H{ "error": gin.H{
@ -90,6 +91,7 @@ func RegisterGatewayRoutes(
}) })
gateway.POST("/images/generations", func(c *gin.Context) { gateway.POST("/images/generations", func(c *gin.Context) {
if getGroupPlatform(c) != service.PlatformOpenAI { if getGroupPlatform(c) != service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{ c.JSON(http.StatusNotFound, gin.H{
"error": gin.H{ "error": gin.H{
"type": "not_found_error", "type": "not_found_error",
@ -102,6 +104,7 @@ func RegisterGatewayRoutes(
}) })
gateway.POST("/images/edits", func(c *gin.Context) { gateway.POST("/images/edits", func(c *gin.Context) {
if getGroupPlatform(c) != service.PlatformOpenAI { if getGroupPlatform(c) != service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{ c.JSON(http.StatusNotFound, gin.H{
"error": gin.H{ "error": gin.H{
"type": "not_found_error", "type": "not_found_error",
@ -157,6 +160,7 @@ func RegisterGatewayRoutes(
}) })
r.POST("/images/generations", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) { r.POST("/images/generations", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
if getGroupPlatform(c) != service.PlatformOpenAI { if getGroupPlatform(c) != service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{ c.JSON(http.StatusNotFound, gin.H{
"error": gin.H{ "error": gin.H{
"type": "not_found_error", "type": "not_found_error",
@ -169,6 +173,7 @@ func RegisterGatewayRoutes(
}) })
r.POST("/images/edits", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) { r.POST("/images/edits", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
if getGroupPlatform(c) != service.PlatformOpenAI { if getGroupPlatform(c) != service.PlatformOpenAI {
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusNotFound, gin.H{ c.JSON(http.StatusNotFound, gin.H{
"error": gin.H{ "error": gin.H{
"type": "not_found_error", "type": "not_found_error",

View File

@ -1362,6 +1362,7 @@ func (s *AntigravityGatewayService) Forward(ctx context.Context, c *gin.Context,
originalModel := claudeReq.Model originalModel := claudeReq.Model
mappedModel := s.getMappedModel(account, claudeReq.Model) mappedModel := s.getMappedModel(account, claudeReq.Model)
if mappedModel == "" { if mappedModel == "" {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
return nil, s.writeClaudeError(c, http.StatusForbidden, "permission_error", fmt.Sprintf("model %s not in whitelist", claudeReq.Model)) return nil, s.writeClaudeError(c, http.StatusForbidden, "permission_error", fmt.Sprintf("model %s not in whitelist", claudeReq.Model))
} }
// 应用 thinking 模式自动后缀:如果 thinking 开启且目标是 claude-sonnet-4-5自动改为 thinking 版本 // 应用 thinking 模式自动后缀:如果 thinking 开启且目标是 claude-sonnet-4-5自动改为 thinking 版本
@ -2112,6 +2113,7 @@ func (s *AntigravityGatewayService) ForwardGemini(ctx context.Context, c *gin.Co
mappedModel := s.getMappedModel(account, originalModel) mappedModel := s.getMappedModel(account, originalModel)
if mappedModel == "" { if mappedModel == "" {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
return nil, s.writeGoogleError(c, http.StatusForbidden, fmt.Sprintf("model %s not in whitelist", originalModel)) return nil, s.writeGoogleError(c, http.StatusForbidden, fmt.Sprintf("model %s not in whitelist", originalModel))
} }
billingModel := mappedModel billingModel := mappedModel

View File

@ -193,6 +193,7 @@ func (s *OpenAIGatewayService) ForwardAsChatCompletions(
if policyErr != nil { if policyErr != nil {
var blocked *OpenAIFastBlockedError var blocked *OpenAIFastBlockedError
if errors.As(policyErr, &blocked) { if errors.As(policyErr, &blocked) {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
writeChatCompletionsError(c, http.StatusForbidden, "permission_error", blocked.Message) writeChatCompletionsError(c, http.StatusForbidden, "permission_error", blocked.Message)
} }
return nil, policyErr return nil, policyErr

View File

@ -93,6 +93,7 @@ func (s *OpenAIGatewayService) forwardAsRawChatCompletions(
if policyErr != nil { if policyErr != nil {
var blocked *OpenAIFastBlockedError var blocked *OpenAIFastBlockedError
if errors.As(policyErr, &blocked) { if errors.As(policyErr, &blocked) {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
writeChatCompletionsError(c, http.StatusForbidden, "permission_error", blocked.Message) writeChatCompletionsError(c, http.StatusForbidden, "permission_error", blocked.Message)
} }
return nil, policyErr return nil, policyErr

View File

@ -231,6 +231,7 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic(
if policyErr != nil { if policyErr != nil {
var blocked *OpenAIFastBlockedError var blocked *OpenAIFastBlockedError
if errors.As(policyErr, &blocked) { if errors.As(policyErr, &blocked) {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
writeAnthropicError(c, http.StatusForbidden, "forbidden_error", blocked.Message) writeAnthropicError(c, http.StatusForbidden, "forbidden_error", blocked.Message)
} }
return nil, policyErr return nil, policyErr

View File

@ -2080,6 +2080,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
apiKeyID := getAPIKeyIDFromContext(c) apiKeyID := getAPIKeyIDFromContext(c)
logCodexCLIOnlyDetection(ctx, c, account, apiKeyID, restrictionResult, body) logCodexCLIOnlyDetection(ctx, c, account, apiKeyID, restrictionResult, body)
if restrictionResult.Enabled && !restrictionResult.Matched { if restrictionResult.Enabled && !restrictionResult.Matched {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
c.JSON(http.StatusForbidden, gin.H{ c.JSON(http.StatusForbidden, gin.H{
"error": gin.H{ "error": gin.H{
"type": "forbidden_error", "type": "forbidden_error",
@ -2123,6 +2124,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
// 当前仅支持 WSv2WSv1 命中时直接返回错误,避免出现“配置可开但行为不确定”。 // 当前仅支持 WSv2WSv1 命中时直接返回错误,避免出现“配置可开但行为不确定”。
if wsDecision.Transport == OpenAIUpstreamTransportResponsesWebsocket { if wsDecision.Transport == OpenAIUpstreamTransportResponsesWebsocket {
if c != nil { if c != nil {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusBadRequest, gin.H{ c.JSON(http.StatusBadRequest, gin.H{
"error": gin.H{ "error": gin.H{
"type": "invalid_request_error", "type": "invalid_request_error",
@ -2163,7 +2165,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
} }
codexImageGenerationBridgeEnabled := isCodexCLI && imageGenerationAllowed && s.isCodexImageGenerationBridgeEnabled(ctx, account, apiKey) codexImageGenerationBridgeEnabled := isCodexCLI && imageGenerationAllowed && s.isCodexImageGenerationBridgeEnabled(ctx, account, apiKey)
if IsImageGenerationIntentMap(openAIResponsesEndpoint, reqModel, reqBody) && !imageGenerationAllowed { if IsImageGenerationIntentMap(openAIResponsesEndpoint, reqModel, reqBody) && !imageGenerationAllowed {
setOpsUpstreamError(c, http.StatusForbidden, ImageGenerationPermissionMessage(), "") MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusForbidden, gin.H{ c.JSON(http.StatusForbidden, gin.H{
"error": gin.H{ "error": gin.H{
"type": "permission_error", "type": "permission_error",
@ -2492,7 +2494,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
} }
if IsImageGenerationIntentMap(openAIResponsesEndpoint, reqModel, reqBody) && !imageGenerationAllowed { if IsImageGenerationIntentMap(openAIResponsesEndpoint, reqModel, reqBody) && !imageGenerationAllowed {
setOpsUpstreamError(c, http.StatusForbidden, ImageGenerationPermissionMessage(), "") MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusForbidden, gin.H{ c.JSON(http.StatusForbidden, gin.H{
"error": gin.H{ "error": gin.H{
"type": "permission_error", "type": "permission_error",
@ -2949,17 +2951,7 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough(
if account != nil && account.Type == AccountTypeOAuth { if account != nil && account.Type == AccountTypeOAuth {
if rejectReason := detectOpenAIPassthroughInstructionsRejectReason(reqModel, body); rejectReason != "" { if rejectReason := detectOpenAIPassthroughInstructionsRejectReason(reqModel, body); rejectReason != "" {
rejectMsg := "OpenAI codex passthrough requires a non-empty instructions field" rejectMsg := "OpenAI codex passthrough requires a non-empty instructions field"
setOpsUpstreamError(c, http.StatusForbidden, rejectMsg, "") MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
Platform: account.Platform,
AccountID: account.ID,
AccountName: account.Name,
UpstreamStatusCode: http.StatusForbidden,
Passthrough: true,
Kind: "request_error",
Message: rejectMsg,
Detail: rejectReason,
})
logOpenAIPassthroughInstructionsRejected(ctx, c, account, reqModel, rejectReason, body) logOpenAIPassthroughInstructionsRejected(ctx, c, account, reqModel, rejectReason, body)
c.JSON(http.StatusForbidden, gin.H{ c.JSON(http.StatusForbidden, gin.H{
"error": gin.H{ "error": gin.H{
@ -3010,7 +3002,7 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough(
apiKey := getAPIKeyFromContext(c) apiKey := getAPIKeyFromContext(c)
if IsImageGenerationIntent(openAIResponsesEndpoint, reqModel, body) && !GroupAllowsImageGeneration(apiKeyGroup(apiKey)) { if IsImageGenerationIntent(openAIResponsesEndpoint, reqModel, body) && !GroupAllowsImageGeneration(apiKeyGroup(apiKey)) {
setOpsUpstreamError(c, http.StatusForbidden, ImageGenerationPermissionMessage(), "") MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
c.JSON(http.StatusForbidden, gin.H{ c.JSON(http.StatusForbidden, gin.H{
"error": gin.H{ "error": gin.H{
"type": "permission_error", "type": "permission_error",
@ -6322,6 +6314,7 @@ func writeOpenAIFastPolicyBlockedResponse(c *gin.Context, err *OpenAIFastBlocked
if c == nil || err == nil { if c == nil || err == nil {
return return
} }
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
c.JSON(http.StatusForbidden, gin.H{ c.JSON(http.StatusForbidden, gin.H{
"error": gin.H{ "error": gin.H{
"type": "permission_error", "type": "permission_error",

View File

@ -2612,6 +2612,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", policyErr) return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", policyErr)
} }
if blocked != nil { if blocked != nil {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
// Send a Realtime-style error event to the client first, then // Send a Realtime-style error event to the client first, then
// signal the handler to close the connection with PolicyViolation. // signal the handler to close the connection with PolicyViolation.
// We intentionally do NOT forward this frame upstream. // We intentionally do NOT forward this frame upstream.

View File

@ -280,6 +280,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
return fmt.Errorf("apply openai fast policy on first ws frame: %w", policyErr) return fmt.Errorf("apply openai fast policy on first ws frame: %w", policyErr)
} }
if blocked != nil { if blocked != nil {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
// coder/websocket@v1.8.14 Conn.Write is synchronous: it acquires // coder/websocket@v1.8.14 Conn.Write is synchronous: it acquires
// writeFrameMu, writes the entire frame, and Flushes the underlying // writeFrameMu, writes the entire frame, and Flushes the underlying
// bufio writer before returning (write.go:42 → write.go:307-311). // bufio writer before returning (write.go:42 → write.go:307-311).
@ -442,6 +443,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
return out, blocked, policyErr return out, blocked, policyErr
}, },
onBlock: func(blocked *OpenAIFastBlockedError) { onBlock: func(blocked *OpenAIFastBlockedError) {
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
// See note above on Conn.Write being synchronous w.r.t. flush; // See note above on Conn.Write being synchronous w.r.t. flush;
// no explicit flush is required to ensure the error event lands // no explicit flush is required to ensure the error event lands
// before the close frame. // before the close frame.

View File

@ -538,7 +538,8 @@ SELECT
COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 429), 0) AS upstream_429, COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 429), 0) AS upstream_429,
COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 529), 0) AS upstream_529 COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 529), 0) AS upstream_529
FROM ops_error_logs FROM ops_error_logs
WHERE created_at >= $1 AND created_at < $2` WHERE created_at >= $1 AND created_at < $2
AND is_count_tokens = FALSE`
if err := c.db.QueryRowContext(ctx, q, start, end).Scan( if err := c.db.QueryRowContext(ctx, q, start, end).Scan(
&errorTotal, &errorTotal,

View File

@ -0,0 +1,60 @@
package service
import (
"context"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/DATA-DOG/go-sqlmock"
"github.com/gin-gonic/gin"
"github.com/stretchr/testify/require"
)
func TestWriteOpenAIFastPolicyBlockedResponseMarksBusinessLimited(t *testing.T) {
gin.SetMode(gin.TestMode)
rec := httptest.NewRecorder()
c, _ := gin.CreateTestContext(rec)
writeOpenAIFastPolicyBlockedResponse(c, &OpenAIFastBlockedError{Message: "custom fast policy block"})
require.Equal(t, http.StatusForbidden, rec.Code)
require.True(t, HasOpsClientBusinessLimited(c))
reason, ok := c.Get(OpsClientBusinessLimitedReasonKey)
require.True(t, ok)
require.Equal(t, OpsClientBusinessLimitedReasonLocalPolicyDenied, reason)
}
func TestOpsMetricsCollectorQueryErrorCountsExcludesCountTokens(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
collector := &OpsMetricsCollector{db: db}
start := time.Date(2026, 5, 26, 10, 0, 0, 0, time.UTC)
end := start.Add(time.Hour)
mock.ExpectQuery(`(?s)FROM ops_error_logs\s+WHERE created_at >= \$1 AND created_at < \$2\s+AND is_count_tokens = FALSE`).
WithArgs(start, end).
WillReturnRows(sqlmock.NewRows([]string{
"error_total",
"business_limited",
"error_sla",
"upstream_excl",
"upstream_429",
"upstream_529",
}).AddRow(int64(5), int64(2), int64(3), int64(1), int64(1), int64(1)))
errorTotal, businessLimited, errorSLA, upstreamExcl429529, upstream429, upstream529, err := collector.queryErrorCounts(context.Background(), start, end)
require.NoError(t, err)
require.Equal(t, int64(5), errorTotal)
require.Equal(t, int64(2), businessLimited)
require.Equal(t, int64(3), errorSLA)
require.Equal(t, int64(1), upstreamExcl429529)
require.Equal(t, int64(1), upstream429)
require.Equal(t, int64(1), upstream529)
require.NoError(t, mock.ExpectationsWereMet())
mock.ExpectClose()
require.NoError(t, db.Close())
require.NoError(t, mock.ExpectationsWereMet())
}

View File

@ -34,9 +34,13 @@ const (
// Client-side configuration denials should remain visible in ops_error_logs, // Client-side configuration denials should remain visible in ops_error_logs,
// but should be excluded from SLA/error-rate calculations. // but should be excluded from SLA/error-rate calculations.
OpsClientBusinessLimitedKey = "ops_client_business_limited" OpsClientBusinessLimitedKey = "ops_client_business_limited"
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason" OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction" OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable"
OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned"
OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate"
OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied"
) )
func SetOpsLatencyMs(c *gin.Context, key string, value int64) { func SetOpsLatencyMs(c *gin.Context, key string, value int64) {