Merge pull request #2790 from Arron196/from-arron-main
修复 Ops SLA 本地限制错误统计
This commit is contained in:
commit
4b9b63443f
@ -781,6 +781,7 @@ func (h *GatewayHandler) Messages(c *gin.Context) {
|
||||
// Beta policy block: return 400 immediately, no failover
|
||||
var betaBlockedErr *service.BetaBlockedError
|
||||
if errors.As(err, &betaBlockedErr) {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
h.errorResponse(c, http.StatusBadRequest, "invalid_request_error", betaBlockedErr.Message)
|
||||
return
|
||||
}
|
||||
|
||||
@ -53,6 +53,8 @@ const (
|
||||
opsCodeUserNotFound = "USER_NOT_FOUND"
|
||||
opsCodeAPIKeyQuotaExhausted = "API_KEY_QUOTA_EXHAUSTED"
|
||||
opsCodeAPIKeyQueryDeprecated = "api_key_in_query_deprecated"
|
||||
opsCodeGroupDeleted = "GROUP_DELETED"
|
||||
opsCodeGroupDisabled = "GROUP_DISABLED"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -1012,6 +1014,8 @@ func parseOpsErrorResponse(body []byte) parsedOpsError {
|
||||
var code string
|
||||
if v, ok := errObj["code"]; ok {
|
||||
switch n := v.(type) {
|
||||
case string:
|
||||
code = strings.TrimSpace(n)
|
||||
case float64:
|
||||
code = strconvItoa(int(n))
|
||||
case int:
|
||||
@ -1188,14 +1192,19 @@ func isOpsClientAuthError(code string, msg string) bool {
|
||||
opsCodeAPIKeyExpired,
|
||||
opsCodeAPIKeyDisabled,
|
||||
opsCodeUserNotFound,
|
||||
opsCodeUserInactive:
|
||||
opsCodeUserInactive,
|
||||
opsCodeGroupDeleted,
|
||||
opsCodeGroupDisabled:
|
||||
return true
|
||||
}
|
||||
return strings.Contains(msg, "invalid api key") ||
|
||||
strings.Contains(msg, "api key is required") ||
|
||||
strings.Contains(msg, "api key is disabled") ||
|
||||
strings.Contains(msg, "user associated with api key not found") ||
|
||||
strings.Contains(msg, "user account is not active")
|
||||
strings.Contains(msg, "user account is not active") ||
|
||||
strings.Contains(msg, "api key 所属分组已删除") ||
|
||||
strings.Contains(msg, "api key 所属分组已停用") ||
|
||||
strings.Contains(msg, "api key is not assigned to any group")
|
||||
}
|
||||
|
||||
func isOpsLocalBusinessLimitError(code string, msg string) bool {
|
||||
@ -1211,6 +1220,7 @@ func isOpsLocalBusinessLimitError(code string, msg string) bool {
|
||||
return strings.Contains(msg, "api key in query parameter is deprecated") ||
|
||||
strings.Contains(msg, "query parameter api_key is deprecated") ||
|
||||
strings.Contains(msg, "no active subscription found for this group") ||
|
||||
strings.Contains(msg, "subscription is invalid or expired") ||
|
||||
strings.Contains(msg, opsErrInsufficientBalance) ||
|
||||
strings.Contains(msg, "insufficient account balance") ||
|
||||
strings.Contains(msg, "api key group platform is not gemini") ||
|
||||
@ -1221,7 +1231,22 @@ func isOpsLocalBusinessLimitError(code string, msg string) bool {
|
||||
strings.Contains(msg, "daily usage limit exceeded") ||
|
||||
strings.Contains(msg, "weekly usage limit exceeded") ||
|
||||
strings.Contains(msg, "monthly usage limit exceeded") ||
|
||||
strings.Contains(msg, "requests-per-minute limit exceeded")
|
||||
strings.Contains(msg, "usage quota exhausted for this platform") ||
|
||||
strings.Contains(msg, "requests-per-minute limit exceeded") ||
|
||||
strings.Contains(msg, "too many pending requests") ||
|
||||
strings.Contains(msg, "concurrency limit exceeded") ||
|
||||
strings.Contains(msg, "image generation concurrency limit exceeded") ||
|
||||
strings.Contains(msg, "this group is restricted to claude code clients") ||
|
||||
strings.Contains(msg, "this group does not allow /v1/messages dispatch") ||
|
||||
strings.Contains(msg, "image generation is not enabled for this group") ||
|
||||
strings.Contains(msg, "token counting is not supported for this platform") ||
|
||||
strings.Contains(msg, "images api is not supported for this platform") ||
|
||||
(strings.Contains(msg, "model ") && strings.Contains(msg, " not in whitelist")) ||
|
||||
(strings.Contains(msg, "beta feature ") && strings.Contains(msg, " is not allowed")) ||
|
||||
(strings.Contains(msg, "openai service_tier=") && strings.Contains(msg, " is not allowed for model")) ||
|
||||
strings.Contains(msg, "this account only allows codex official clients") ||
|
||||
strings.Contains(msg, "openai wsv1 is temporarily unsupported") ||
|
||||
strings.Contains(msg, "openai codex passthrough requires a non-empty instructions field")
|
||||
}
|
||||
|
||||
func hasOpsUpstreamErrorContext(c *gin.Context) bool {
|
||||
|
||||
@ -288,6 +288,34 @@ func TestClassifyOpsAuthClientErrorsExcludedFromSLA(t *testing.T) {
|
||||
code: "USER_INACTIVE",
|
||||
status: http.StatusUnauthorized,
|
||||
},
|
||||
{
|
||||
name: "deleted local API key group",
|
||||
errType: "api_error",
|
||||
message: "API Key 所属分组已删除",
|
||||
code: "GROUP_DELETED",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "disabled local API key group",
|
||||
errType: "api_error",
|
||||
message: "API Key 所属分组已停用",
|
||||
code: "GROUP_DISABLED",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "google deleted API key group message without semantic code",
|
||||
errType: "api_error",
|
||||
message: "API Key 所属分组已删除",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "anthropic unassigned API key group",
|
||||
errType: "permission_error",
|
||||
message: "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "google invalid API key",
|
||||
errType: "api_error",
|
||||
@ -389,6 +417,15 @@ func TestClassifyOpsLocalBusinessLimitErrorsExcludedFromSLA(t *testing.T) {
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "gateway subscription invalid cache recheck",
|
||||
errType: "billing_error",
|
||||
message: "subscription is invalid or expired",
|
||||
code: "billing_error",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "billing_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "google insufficient account balance",
|
||||
errType: "api_error",
|
||||
@ -443,6 +480,132 @@ func TestClassifyOpsLocalBusinessLimitErrorsExcludedFromSLA(t *testing.T) {
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "user platform daily quota exhausted",
|
||||
errType: "api_error",
|
||||
message: "Daily usage quota exhausted for this platform.",
|
||||
code: "rate_limit_exceeded",
|
||||
status: http.StatusTooManyRequests,
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "local pending queue limit",
|
||||
errType: "rate_limit_error",
|
||||
message: "Too many pending requests, please retry later",
|
||||
code: "",
|
||||
status: http.StatusTooManyRequests,
|
||||
wantErrType: "rate_limit_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "local concurrency limit",
|
||||
errType: "rate_limit_error",
|
||||
message: "Concurrency limit exceeded for user, please retry later",
|
||||
code: "",
|
||||
status: http.StatusTooManyRequests,
|
||||
wantErrType: "rate_limit_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "group claude code only feature gate",
|
||||
errType: "permission_error",
|
||||
message: "This group is restricted to Claude Code clients (/v1/messages only)",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "group image generation feature gate",
|
||||
errType: "permission_error",
|
||||
message: "Image generation is not enabled for this group",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "route token counting platform unsupported",
|
||||
errType: "not_found_error",
|
||||
message: "Token counting is not supported for this platform",
|
||||
code: "",
|
||||
status: http.StatusNotFound,
|
||||
wantErrType: "not_found_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "route images API platform unsupported",
|
||||
errType: "not_found_error",
|
||||
message: "Images API is not supported for this platform",
|
||||
code: "",
|
||||
status: http.StatusNotFound,
|
||||
wantErrType: "not_found_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "antigravity model whitelist feature gate",
|
||||
errType: "permission_error",
|
||||
message: "model claude-3-5-sonnet not in whitelist",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "google antigravity model whitelist feature gate",
|
||||
errType: "api_error",
|
||||
message: "model gemini-2.5-pro not in whitelist",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "claude beta policy block",
|
||||
errType: "invalid_request_error",
|
||||
message: "beta feature interleaved-thinking-2025-05-14 is not allowed",
|
||||
code: "",
|
||||
status: http.StatusBadRequest,
|
||||
wantErrType: "invalid_request_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "openai fast policy block",
|
||||
errType: "permission_error",
|
||||
message: "openai service_tier=priority is not allowed for model gpt-5.5",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "api_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "codex official client policy block",
|
||||
errType: "forbidden_error",
|
||||
message: "This account only allows Codex official clients",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "forbidden_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "openai wsv1 unsupported feature gate",
|
||||
errType: "invalid_request_error",
|
||||
message: "OpenAI WSv1 is temporarily unsupported. Please enable responses_websockets_v2.",
|
||||
code: "",
|
||||
status: http.StatusBadRequest,
|
||||
wantErrType: "invalid_request_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
{
|
||||
name: "openai passthrough instructions policy block",
|
||||
errType: "forbidden_error",
|
||||
message: "OpenAI codex passthrough requires a non-empty instructions field",
|
||||
code: "",
|
||||
status: http.StatusForbidden,
|
||||
wantErrType: "forbidden_error",
|
||||
wantPhase: "request",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@ -479,6 +642,22 @@ func TestClassifyOpsIPRestrictionAccessDeniedExcludedFromSLA(t *testing.T) {
|
||||
require.Equal(t, "client_request", errorSource)
|
||||
}
|
||||
|
||||
func TestClassifyOpsClientBusinessLimitedMarkerExcludesCustomPolicyDenialFromSLA(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
|
||||
errType := normalizeOpsErrorType("invalid_request_error", "")
|
||||
phase, isBusinessLimited, errorOwner, errorSource := classifyOpsErrorLog(c, errType, "custom admin policy message", "", http.StatusBadRequest)
|
||||
|
||||
require.Equal(t, "invalid_request_error", errType)
|
||||
require.Equal(t, "auth", phase)
|
||||
require.True(t, isBusinessLimited)
|
||||
require.Equal(t, "client", errorOwner)
|
||||
require.Equal(t, "client_request", errorSource)
|
||||
}
|
||||
|
||||
func TestClassifyOpsOtherErrorsStillCountForSLA(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
@ -583,6 +762,78 @@ func TestClassifyOpsUpstreamAuthTextStillCountsForSLA(t *testing.T) {
|
||||
code: "API_KEY_QUOTA_EXHAUSTED",
|
||||
status: http.StatusTooManyRequests,
|
||||
},
|
||||
{
|
||||
name: "provider deleted group shaped error",
|
||||
message: "API Key 所属分组已删除",
|
||||
code: "GROUP_DELETED",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "provider unassigned group shaped error",
|
||||
message: "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "provider local quota shaped error",
|
||||
message: "Daily usage quota exhausted for this platform.",
|
||||
code: "rate_limit_exceeded",
|
||||
status: http.StatusTooManyRequests,
|
||||
},
|
||||
{
|
||||
name: "provider feature gate shaped error",
|
||||
message: "Image generation is not enabled for this group",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "provider token counting unsupported shaped error",
|
||||
message: "Token counting is not supported for this platform",
|
||||
code: "404",
|
||||
status: http.StatusNotFound,
|
||||
},
|
||||
{
|
||||
name: "provider image API unsupported shaped error",
|
||||
message: "Images API is not supported for this platform",
|
||||
code: "404",
|
||||
status: http.StatusNotFound,
|
||||
},
|
||||
{
|
||||
name: "provider antigravity whitelist shaped error",
|
||||
message: "model claude-3-5-sonnet not in whitelist",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "provider beta policy shaped error",
|
||||
message: "beta feature interleaved-thinking-2025-05-14 is not allowed",
|
||||
code: "400",
|
||||
status: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "provider openai fast policy shaped error",
|
||||
message: "openai service_tier=priority is not allowed for model gpt-5.5",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "provider codex client policy shaped error",
|
||||
message: "This account only allows Codex official clients",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
{
|
||||
name: "provider wsv1 unsupported shaped error",
|
||||
message: "OpenAI WSv1 is temporarily unsupported. Please enable responses_websockets_v2.",
|
||||
code: "400",
|
||||
status: http.StatusBadRequest,
|
||||
},
|
||||
{
|
||||
name: "provider passthrough instructions shaped error",
|
||||
message: "OpenAI codex passthrough requires a non-empty instructions field",
|
||||
code: "403",
|
||||
status: http.StatusForbidden,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
@ -628,6 +879,14 @@ func TestClassifyOpsUpstreamNoAvailableTextStillCountsForSLA(t *testing.T) {
|
||||
require.Equal(t, "upstream_http", errorSource)
|
||||
}
|
||||
|
||||
func TestParseOpsErrorResponsePreservesNestedStringCode(t *testing.T) {
|
||||
parsed := parseOpsErrorResponse([]byte(`{"error":{"type":"permission_error","code":"GROUP_DELETED","message":"API Key 所属分组已删除"}}`))
|
||||
|
||||
require.Equal(t, "permission_error", parsed.ErrorType)
|
||||
require.Equal(t, "GROUP_DELETED", parsed.Code)
|
||||
require.Equal(t, "API Key 所属分组已删除", parsed.Message)
|
||||
}
|
||||
|
||||
func TestSetOpsEndpointContext_SetsContextKeys(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
|
||||
@ -263,6 +263,7 @@ func abortIfAPIKeyGroupUnavailable(c *gin.Context, apiKey *service.APIKey) bool
|
||||
if ok {
|
||||
return false
|
||||
}
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
|
||||
AbortWithError(c, 403, code, message)
|
||||
return true
|
||||
}
|
||||
|
||||
@ -55,6 +55,7 @@ func APIKeyAuthWithSubscriptionGoogle(apiKeyService *service.APIKeyService, subs
|
||||
return
|
||||
}
|
||||
if _, message, ok := validateAPIKeyGroupAvailable(apiKey); !ok {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable)
|
||||
abortWithGoogleError(c, 403, message)
|
||||
return
|
||||
}
|
||||
|
||||
@ -373,6 +373,68 @@ func TestApiKeyAuthWithSubscriptionGoogle_InvalidKey(t *testing.T) {
|
||||
require.Equal(t, "UNAUTHENTICATED", resp.Error.Status)
|
||||
}
|
||||
|
||||
func TestApiKeyAuthWithSubscriptionGoogle_MarksUnavailableGroupBusinessLimited(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
groupID := int64(101)
|
||||
user := &service.User{
|
||||
ID: 7,
|
||||
Role: service.RoleUser,
|
||||
Status: service.StatusActive,
|
||||
Balance: 10,
|
||||
Concurrency: 3,
|
||||
}
|
||||
apiKey := &service.APIKey{
|
||||
ID: 100,
|
||||
UserID: user.ID,
|
||||
GroupID: &groupID,
|
||||
Key: "google-group-deleted",
|
||||
Status: service.StatusActive,
|
||||
User: user,
|
||||
Group: &service.Group{
|
||||
ID: groupID,
|
||||
Name: "deleted",
|
||||
Status: "deleted",
|
||||
Platform: service.PlatformGemini,
|
||||
Hydrated: true,
|
||||
},
|
||||
}
|
||||
|
||||
r := gin.New()
|
||||
var markedBusinessLimited bool
|
||||
var businessLimitedReason string
|
||||
r.Use(func(c *gin.Context) {
|
||||
c.Next()
|
||||
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
|
||||
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
|
||||
businessLimitedReason, _ = v.(string)
|
||||
}
|
||||
})
|
||||
apiKeyService := newTestAPIKeyService(fakeAPIKeyRepo{
|
||||
getByKey: func(ctx context.Context, key string) (*service.APIKey, error) {
|
||||
if key != apiKey.Key {
|
||||
return nil, service.ErrAPIKeyNotFound
|
||||
}
|
||||
clone := *apiKey
|
||||
return &clone, nil
|
||||
},
|
||||
})
|
||||
r.Use(APIKeyAuthWithSubscriptionGoogle(apiKeyService, nil, &config.Config{RunMode: config.RunModeSimple}))
|
||||
r.GET("/v1beta/test", func(c *gin.Context) { c.JSON(200, gin.H{"ok": true}) })
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/v1beta/test", nil)
|
||||
req.Header.Set("x-goog-api-key", apiKey.Key)
|
||||
rec := httptest.NewRecorder()
|
||||
r.ServeHTTP(rec, req)
|
||||
|
||||
require.Equal(t, http.StatusForbidden, rec.Code)
|
||||
var resp googleErrorResponse
|
||||
require.NoError(t, json.Unmarshal(rec.Body.Bytes(), &resp))
|
||||
require.Equal(t, "API Key 所属分组已删除", resp.Error.Message)
|
||||
require.True(t, markedBusinessLimited)
|
||||
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable, businessLimitedReason)
|
||||
}
|
||||
|
||||
func TestApiKeyAuthWithSubscriptionGoogle_RepoError(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
|
||||
@ -317,6 +317,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
|
||||
group *service.Group
|
||||
wantStatus int
|
||||
wantCode string
|
||||
wantMarked bool
|
||||
}{
|
||||
{
|
||||
name: "active group passes",
|
||||
@ -340,6 +341,7 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
|
||||
},
|
||||
wantStatus: http.StatusForbidden,
|
||||
wantCode: "GROUP_DISABLED",
|
||||
wantMarked: true,
|
||||
},
|
||||
{
|
||||
name: "deleted status group is forbidden",
|
||||
@ -352,12 +354,14 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
|
||||
},
|
||||
wantStatus: http.StatusForbidden,
|
||||
wantCode: "GROUP_DELETED",
|
||||
wantMarked: true,
|
||||
},
|
||||
{
|
||||
name: "missing group edge is forbidden",
|
||||
group: nil,
|
||||
wantStatus: http.StatusForbidden,
|
||||
wantCode: "GROUP_DELETED",
|
||||
wantMarked: true,
|
||||
},
|
||||
}
|
||||
|
||||
@ -383,7 +387,20 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
|
||||
}
|
||||
cfg := &config.Config{RunMode: config.RunModeStandard}
|
||||
apiKeyService := service.NewAPIKeyService(apiKeyRepo, nil, nil, nil, nil, nil, cfg)
|
||||
router := newAuthTestRouter(apiKeyService, nil, cfg)
|
||||
router := gin.New()
|
||||
var markedBusinessLimited bool
|
||||
var businessLimitedReason string
|
||||
router.Use(func(c *gin.Context) {
|
||||
c.Next()
|
||||
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
|
||||
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
|
||||
businessLimitedReason, _ = v.(string)
|
||||
}
|
||||
})
|
||||
router.Use(gin.HandlerFunc(NewAPIKeyAuthMiddleware(apiKeyService, nil, cfg)))
|
||||
router.GET("/t", func(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
})
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/t", nil)
|
||||
@ -394,10 +411,57 @@ func TestAPIKeyAuthRejectsUnavailableGroup(t *testing.T) {
|
||||
if tt.wantCode != "" {
|
||||
require.Contains(t, w.Body.String(), tt.wantCode)
|
||||
}
|
||||
require.Equal(t, tt.wantMarked, markedBusinessLimited)
|
||||
if tt.wantMarked {
|
||||
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable, businessLimitedReason)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRequireGroupAssignmentMarksUngroupedKeyBusinessLimited(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
settingService := service.NewSettingService(fakeSettingRepo{
|
||||
values: map[string]string{
|
||||
service.SettingKeyAllowUngroupedKeyScheduling: "false",
|
||||
},
|
||||
}, &config.Config{})
|
||||
apiKey := &service.APIKey{
|
||||
ID: 100,
|
||||
Key: "ungrouped-key",
|
||||
Status: service.StatusActive,
|
||||
}
|
||||
|
||||
router := gin.New()
|
||||
var markedBusinessLimited bool
|
||||
var businessLimitedReason string
|
||||
router.Use(func(c *gin.Context) {
|
||||
c.Next()
|
||||
markedBusinessLimited = service.HasOpsClientBusinessLimited(c)
|
||||
if v, ok := c.Get(service.OpsClientBusinessLimitedReasonKey); ok {
|
||||
businessLimitedReason, _ = v.(string)
|
||||
}
|
||||
})
|
||||
router.Use(func(c *gin.Context) {
|
||||
c.Set(string(ContextKeyAPIKey), apiKey)
|
||||
c.Next()
|
||||
})
|
||||
router.Use(RequireGroupAssignment(settingService, AnthropicErrorWriter))
|
||||
router.GET("/t", func(c *gin.Context) {
|
||||
c.JSON(http.StatusOK, gin.H{"ok": true})
|
||||
})
|
||||
|
||||
w := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "/t", nil)
|
||||
router.ServeHTTP(w, req)
|
||||
|
||||
require.Equal(t, http.StatusForbidden, w.Code)
|
||||
require.Contains(t, w.Body.String(), "not assigned to any group")
|
||||
require.True(t, markedBusinessLimited)
|
||||
require.Equal(t, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned, businessLimitedReason)
|
||||
}
|
||||
|
||||
func TestAPIKeyAuthIPRestrictionDoesNotTrustForwardedClientIPByDefault(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
|
||||
@ -771,6 +835,41 @@ type stubUserSubscriptionRepo struct {
|
||||
resetMonthly func(ctx context.Context, id int64, start time.Time) error
|
||||
}
|
||||
|
||||
type fakeSettingRepo struct {
|
||||
values map[string]string
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) Get(ctx context.Context, key string) (*service.Setting, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) GetValue(ctx context.Context, key string) (string, error) {
|
||||
if v, ok := r.values[key]; ok {
|
||||
return v, nil
|
||||
}
|
||||
return "", service.ErrSettingNotFound
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) Set(ctx context.Context, key, value string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) GetMultiple(ctx context.Context, keys []string) (map[string]string, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) SetMultiple(ctx context.Context, settings map[string]string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) GetAll(ctx context.Context) (map[string]string, error) {
|
||||
return nil, errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (r fakeSettingRepo) Delete(ctx context.Context, key string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func (r *stubUserSubscriptionRepo) Create(ctx context.Context, sub *service.UserSubscription) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
@ -115,6 +115,7 @@ func RequireGroupAssignment(settingService *service.SettingService, writeError G
|
||||
c.Next()
|
||||
return
|
||||
}
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned)
|
||||
writeError(c, http.StatusForbidden, "API Key is not assigned to any group and cannot be used. Please contact the administrator to assign it to a group.")
|
||||
c.Abort()
|
||||
}
|
||||
|
||||
@ -51,6 +51,7 @@ func RegisterGatewayRoutes(
|
||||
// /v1/messages/count_tokens: OpenAI groups get 404
|
||||
gateway.POST("/messages/count_tokens", func(c *gin.Context) {
|
||||
if getGroupPlatform(c) == service.PlatformOpenAI {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"type": "error",
|
||||
"error": gin.H{
|
||||
@ -90,6 +91,7 @@ func RegisterGatewayRoutes(
|
||||
})
|
||||
gateway.POST("/images/generations", func(c *gin.Context) {
|
||||
if getGroupPlatform(c) != service.PlatformOpenAI {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "not_found_error",
|
||||
@ -102,6 +104,7 @@ func RegisterGatewayRoutes(
|
||||
})
|
||||
gateway.POST("/images/edits", func(c *gin.Context) {
|
||||
if getGroupPlatform(c) != service.PlatformOpenAI {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "not_found_error",
|
||||
@ -157,6 +160,7 @@ func RegisterGatewayRoutes(
|
||||
})
|
||||
r.POST("/images/generations", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
|
||||
if getGroupPlatform(c) != service.PlatformOpenAI {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "not_found_error",
|
||||
@ -169,6 +173,7 @@ func RegisterGatewayRoutes(
|
||||
})
|
||||
r.POST("/images/edits", bodyLimit, clientRequestID, opsErrorLogger, endpointNorm, gin.HandlerFunc(apiKeyAuth), requireGroupAnthropic, func(c *gin.Context) {
|
||||
if getGroupPlatform(c) != service.PlatformOpenAI {
|
||||
service.MarkOpsClientBusinessLimited(c, service.OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusNotFound, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "not_found_error",
|
||||
|
||||
@ -1362,6 +1362,7 @@ func (s *AntigravityGatewayService) Forward(ctx context.Context, c *gin.Context,
|
||||
originalModel := claudeReq.Model
|
||||
mappedModel := s.getMappedModel(account, claudeReq.Model)
|
||||
if mappedModel == "" {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
return nil, s.writeClaudeError(c, http.StatusForbidden, "permission_error", fmt.Sprintf("model %s not in whitelist", claudeReq.Model))
|
||||
}
|
||||
// 应用 thinking 模式自动后缀:如果 thinking 开启且目标是 claude-sonnet-4-5,自动改为 thinking 版本
|
||||
@ -2112,6 +2113,7 @@ func (s *AntigravityGatewayService) ForwardGemini(ctx context.Context, c *gin.Co
|
||||
|
||||
mappedModel := s.getMappedModel(account, originalModel)
|
||||
if mappedModel == "" {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
return nil, s.writeGoogleError(c, http.StatusForbidden, fmt.Sprintf("model %s not in whitelist", originalModel))
|
||||
}
|
||||
billingModel := mappedModel
|
||||
|
||||
@ -193,6 +193,7 @@ func (s *OpenAIGatewayService) ForwardAsChatCompletions(
|
||||
if policyErr != nil {
|
||||
var blocked *OpenAIFastBlockedError
|
||||
if errors.As(policyErr, &blocked) {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
writeChatCompletionsError(c, http.StatusForbidden, "permission_error", blocked.Message)
|
||||
}
|
||||
return nil, policyErr
|
||||
|
||||
@ -93,6 +93,7 @@ func (s *OpenAIGatewayService) forwardAsRawChatCompletions(
|
||||
if policyErr != nil {
|
||||
var blocked *OpenAIFastBlockedError
|
||||
if errors.As(policyErr, &blocked) {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
writeChatCompletionsError(c, http.StatusForbidden, "permission_error", blocked.Message)
|
||||
}
|
||||
return nil, policyErr
|
||||
|
||||
@ -231,6 +231,7 @@ func (s *OpenAIGatewayService) ForwardAsAnthropic(
|
||||
if policyErr != nil {
|
||||
var blocked *OpenAIFastBlockedError
|
||||
if errors.As(policyErr, &blocked) {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
writeAnthropicError(c, http.StatusForbidden, "forbidden_error", blocked.Message)
|
||||
}
|
||||
return nil, policyErr
|
||||
|
||||
@ -2080,6 +2080,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
apiKeyID := getAPIKeyIDFromContext(c)
|
||||
logCodexCLIOnlyDetection(ctx, c, account, apiKeyID, restrictionResult, body)
|
||||
if restrictionResult.Enabled && !restrictionResult.Matched {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "forbidden_error",
|
||||
@ -2123,6 +2124,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
// 当前仅支持 WSv2;WSv1 命中时直接返回错误,避免出现“配置可开但行为不确定”。
|
||||
if wsDecision.Transport == OpenAIUpstreamTransportResponsesWebsocket {
|
||||
if c != nil {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusBadRequest, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "invalid_request_error",
|
||||
@ -2163,7 +2165,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
}
|
||||
codexImageGenerationBridgeEnabled := isCodexCLI && imageGenerationAllowed && s.isCodexImageGenerationBridgeEnabled(ctx, account, apiKey)
|
||||
if IsImageGenerationIntentMap(openAIResponsesEndpoint, reqModel, reqBody) && !imageGenerationAllowed {
|
||||
setOpsUpstreamError(c, http.StatusForbidden, ImageGenerationPermissionMessage(), "")
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "permission_error",
|
||||
@ -2492,7 +2494,7 @@ func (s *OpenAIGatewayService) Forward(ctx context.Context, c *gin.Context, acco
|
||||
}
|
||||
|
||||
if IsImageGenerationIntentMap(openAIResponsesEndpoint, reqModel, reqBody) && !imageGenerationAllowed {
|
||||
setOpsUpstreamError(c, http.StatusForbidden, ImageGenerationPermissionMessage(), "")
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "permission_error",
|
||||
@ -2949,17 +2951,7 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough(
|
||||
if account != nil && account.Type == AccountTypeOAuth {
|
||||
if rejectReason := detectOpenAIPassthroughInstructionsRejectReason(reqModel, body); rejectReason != "" {
|
||||
rejectMsg := "OpenAI codex passthrough requires a non-empty instructions field"
|
||||
setOpsUpstreamError(c, http.StatusForbidden, rejectMsg, "")
|
||||
appendOpsUpstreamError(c, OpsUpstreamErrorEvent{
|
||||
Platform: account.Platform,
|
||||
AccountID: account.ID,
|
||||
AccountName: account.Name,
|
||||
UpstreamStatusCode: http.StatusForbidden,
|
||||
Passthrough: true,
|
||||
Kind: "request_error",
|
||||
Message: rejectMsg,
|
||||
Detail: rejectReason,
|
||||
})
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
logOpenAIPassthroughInstructionsRejected(ctx, c, account, reqModel, rejectReason, body)
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": gin.H{
|
||||
@ -3010,7 +3002,7 @@ func (s *OpenAIGatewayService) forwardOpenAIPassthrough(
|
||||
|
||||
apiKey := getAPIKeyFromContext(c)
|
||||
if IsImageGenerationIntent(openAIResponsesEndpoint, reqModel, body) && !GroupAllowsImageGeneration(apiKeyGroup(apiKey)) {
|
||||
setOpsUpstreamError(c, http.StatusForbidden, ImageGenerationPermissionMessage(), "")
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalFeatureGate)
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "permission_error",
|
||||
@ -6322,6 +6314,7 @@ func writeOpenAIFastPolicyBlockedResponse(c *gin.Context, err *OpenAIFastBlocked
|
||||
if c == nil || err == nil {
|
||||
return
|
||||
}
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
c.JSON(http.StatusForbidden, gin.H{
|
||||
"error": gin.H{
|
||||
"type": "permission_error",
|
||||
|
||||
@ -2612,6 +2612,7 @@ func (s *OpenAIGatewayService) ProxyResponsesWebSocketFromClient(
|
||||
return openAIWSClientPayload{}, NewOpenAIWSClientCloseError(coderws.StatusPolicyViolation, "invalid websocket request payload", policyErr)
|
||||
}
|
||||
if blocked != nil {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
// Send a Realtime-style error event to the client first, then
|
||||
// signal the handler to close the connection with PolicyViolation.
|
||||
// We intentionally do NOT forward this frame upstream.
|
||||
|
||||
@ -280,6 +280,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
|
||||
return fmt.Errorf("apply openai fast policy on first ws frame: %w", policyErr)
|
||||
}
|
||||
if blocked != nil {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
// coder/websocket@v1.8.14 Conn.Write is synchronous: it acquires
|
||||
// writeFrameMu, writes the entire frame, and Flushes the underlying
|
||||
// bufio writer before returning (write.go:42 → write.go:307-311).
|
||||
@ -442,6 +443,7 @@ func (s *OpenAIGatewayService) proxyResponsesWebSocketV2Passthrough(
|
||||
return out, blocked, policyErr
|
||||
},
|
||||
onBlock: func(blocked *OpenAIFastBlockedError) {
|
||||
MarkOpsClientBusinessLimited(c, OpsClientBusinessLimitedReasonLocalPolicyDenied)
|
||||
// See note above on Conn.Write being synchronous w.r.t. flush;
|
||||
// no explicit flush is required to ensure the error event lands
|
||||
// before the close frame.
|
||||
|
||||
@ -538,7 +538,8 @@ SELECT
|
||||
COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 429), 0) AS upstream_429,
|
||||
COALESCE(COUNT(*) FILTER (WHERE error_owner = 'provider' AND NOT is_business_limited AND COALESCE(upstream_status_code, status_code, 0) = 529), 0) AS upstream_529
|
||||
FROM ops_error_logs
|
||||
WHERE created_at >= $1 AND created_at < $2`
|
||||
WHERE created_at >= $1 AND created_at < $2
|
||||
AND is_count_tokens = FALSE`
|
||||
|
||||
if err := c.db.QueryRowContext(ctx, q, start, end).Scan(
|
||||
&errorTotal,
|
||||
|
||||
60
backend/internal/service/ops_metrics_collector_test.go
Normal file
60
backend/internal/service/ops_metrics_collector_test.go
Normal file
@ -0,0 +1,60 @@
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/DATA-DOG/go-sqlmock"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestWriteOpenAIFastPolicyBlockedResponseMarksBusinessLimited(t *testing.T) {
|
||||
gin.SetMode(gin.TestMode)
|
||||
rec := httptest.NewRecorder()
|
||||
c, _ := gin.CreateTestContext(rec)
|
||||
|
||||
writeOpenAIFastPolicyBlockedResponse(c, &OpenAIFastBlockedError{Message: "custom fast policy block"})
|
||||
|
||||
require.Equal(t, http.StatusForbidden, rec.Code)
|
||||
require.True(t, HasOpsClientBusinessLimited(c))
|
||||
reason, ok := c.Get(OpsClientBusinessLimitedReasonKey)
|
||||
require.True(t, ok)
|
||||
require.Equal(t, OpsClientBusinessLimitedReasonLocalPolicyDenied, reason)
|
||||
}
|
||||
|
||||
func TestOpsMetricsCollectorQueryErrorCountsExcludesCountTokens(t *testing.T) {
|
||||
db, mock, err := sqlmock.New()
|
||||
require.NoError(t, err)
|
||||
|
||||
collector := &OpsMetricsCollector{db: db}
|
||||
start := time.Date(2026, 5, 26, 10, 0, 0, 0, time.UTC)
|
||||
end := start.Add(time.Hour)
|
||||
|
||||
mock.ExpectQuery(`(?s)FROM ops_error_logs\s+WHERE created_at >= \$1 AND created_at < \$2\s+AND is_count_tokens = FALSE`).
|
||||
WithArgs(start, end).
|
||||
WillReturnRows(sqlmock.NewRows([]string{
|
||||
"error_total",
|
||||
"business_limited",
|
||||
"error_sla",
|
||||
"upstream_excl",
|
||||
"upstream_429",
|
||||
"upstream_529",
|
||||
}).AddRow(int64(5), int64(2), int64(3), int64(1), int64(1), int64(1)))
|
||||
|
||||
errorTotal, businessLimited, errorSLA, upstreamExcl429529, upstream429, upstream529, err := collector.queryErrorCounts(context.Background(), start, end)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, int64(5), errorTotal)
|
||||
require.Equal(t, int64(2), businessLimited)
|
||||
require.Equal(t, int64(3), errorSLA)
|
||||
require.Equal(t, int64(1), upstreamExcl429529)
|
||||
require.Equal(t, int64(1), upstream429)
|
||||
require.Equal(t, int64(1), upstream529)
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
mock.ExpectClose()
|
||||
require.NoError(t, db.Close())
|
||||
require.NoError(t, mock.ExpectationsWereMet())
|
||||
}
|
||||
@ -34,9 +34,13 @@ const (
|
||||
|
||||
// Client-side configuration denials should remain visible in ops_error_logs,
|
||||
// but should be excluded from SLA/error-rate calculations.
|
||||
OpsClientBusinessLimitedKey = "ops_client_business_limited"
|
||||
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
|
||||
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
|
||||
OpsClientBusinessLimitedKey = "ops_client_business_limited"
|
||||
OpsClientBusinessLimitedReasonKey = "ops_client_business_limited_reason"
|
||||
OpsClientBusinessLimitedReasonIPRestriction = "api_key_ip_restriction"
|
||||
OpsClientBusinessLimitedReasonAPIKeyGroupUnavailable = "api_key_group_unavailable"
|
||||
OpsClientBusinessLimitedReasonAPIKeyGroupUnassigned = "api_key_group_unassigned"
|
||||
OpsClientBusinessLimitedReasonLocalFeatureGate = "local_feature_gate"
|
||||
OpsClientBusinessLimitedReasonLocalPolicyDenied = "local_policy_denied"
|
||||
)
|
||||
|
||||
func SetOpsLatencyMs(c *gin.Context, key string, value int64) {
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user