From 80ad82df52568992656b502ae7fbaaee63bb0bc9 Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 17:28:06 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E4=B8=8D=E5=86=8D=E9=A1=BA=E5=BB=B6=EF=BC=9A?= =?UTF-8?q?=E6=8D=A2=E5=8F=B7=E9=A2=84=E7=AE=97=E3=80=81=E5=90=8C=E5=AE=B6?= =?UTF-8?q?=E5=8E=9F=E5=9C=B0=E7=83=A7=E9=A2=84=E7=AE=97=E4=B8=8E=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E6=97=A0=E5=86=B7=E5=8D=B4=EF=BC=88#643=20#644?= =?UTF-8?q?=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 一条请求失败后下游直接拿到报错的三个真实原因,逐个修掉: 1. 换号预算会被候选队列截断 `switches` 默认 5、存盘里 10,而一条路由组请求的候选队列动辄 21 个。 试到第 10 个就收尾,排在第 12 位、不做工具模拟的那家其实一次就 200。 现在预算取 `max(用户值, 候选队列规模)`,并仍受转发层 `MAX_ROUTE_ATTEMPTS` (32)约束;配成 0 才是显式的「不换号」。 2. 本地构造期的错误绕过整套错误规则 `provider_loop` 的构造失败分支对非 556 错误一律 `return Err`,等于把 「这一次没发出去」当成「整份请求到此为止」,用户的错误处理设置完全不生效。 现在全部交给 `finish_failure` 判定。 3. 同一家的「能力性失败」把预算烧在同一个坑里 网页工具字符预算超限这类错误是本机按模型 profile 判定的( `web_tools_invalid_request`),同一家换账号拿到的是同一份 body、同一个预算。 现在按「跳过这一家」处理:本次请求排除该家全部候选,同时给失败的 账号×模型 落冷却;本家其他模型仍可用,不做整家拉黑。 配套的口径统一: - 默认任何上游错误都换号:`bad_request / content / unknown` 不再是 `return_error` 硬终止,403 与权限类也从 `hard_stop` 里移除;插件报的 裸 `stop` 只表示「别原地重发」,不再顺手掐掉换号。保留的硬终止只剩 「再发一次可能重复执行 / 重复计费」与「已取消」两类。 - 有错就落限制器:请求有误、内容被拒、能力不支持也补一段短冷却 (`fixed 60s`,模型维度),不再让同一个已知坏组合被下一条请求反复挑中。 - 存盘规则升级:`rulesVersion` 标记旧口径,加载时给缺冷却的规则补上出厂 冷却、把 `unknown` 的 `return_error` 恢复成换号;用户自己配的例外 (`overrides` 透传名单)原样保留,保存时统一盖上当前版本号。 - 「指定错误码」名单语义反转:命中 = 允许把错误透传给下游(`ReturnError` 例外),默认空 —— 任何错误都换号。 - 失败即解绑会话粘性(`StickySessions::forget`),避免同一会话的下一条 请求仍然先撞同一个失败账号。 - 路由组成员「空账号名单」= 该提供商全部可用账号,不再因为漏勾账号而 静默少候选(客户端模型列表同步同口径)。 - 设置面板「最多再换账号」上界 10 → 32,与后端默认值对齐。 - 单测覆盖:默认全换号、403 换号、5xx 不原地重发、400 换号且短冷却、 例外透传、网页工具预算跳过整家、本地构造失败跳过整家、`status 0` 收尾、 插件 `stop` 不阻断换号、旧存盘升级补冷却与例外保留。 --- .../server/src/server/api/retry_api.rs | 12 +- .../server/src/server/config/error_policy.rs | 107 ++++++++++- .../src/server/config/error_policy_types.rs | 109 ++++++++++- .../server/src/server/config/parse.rs | 13 +- .../server/src/server/config/types.rs | 51 +++--- .../server/src/server/core/routing_groups.rs | 46 +++-- .../src/server/core/upstream/error_policy.rs | 173 +++++++++++++++++- .../core/upstream/error_policy_evidence.rs | 25 ++- .../core/upstream/error_policy_finish.rs | 10 +- .../server/core/upstream/error_policy_send.rs | 7 +- .../src/server/core/upstream/provider_loop.rs | 53 ++++-- .../server/src/server/core/upstream/rotate.rs | 42 ++++- .../src/server/core/upstream/route_state.rs | 5 + .../src/server/core/upstream/route_sticky.rs | 15 ++ .../ui-shared/routing-errors-panel.tsx | 2 +- .../ui-shared/routing-errors-types.ts | 2 +- 16 files changed, 564 insertions(+), 108 deletions(-) diff --git a/desktop-tauri/src-tauri/server/src/server/api/retry_api.rs b/desktop-tauri/src-tauri/server/src/server/api/retry_api.rs index 2caed0a15..02ff3ed85 100644 --- a/desktop-tauri/src-tauri/server/src/server/api/retry_api.rs +++ b/desktop-tauri/src-tauri/server/src/server/api/retry_api.rs @@ -26,8 +26,8 @@ use serde_json::{json, Value}; use crate::server::config::{ self, RetryPatch, RetrySettings, KEY_RETRY_ACCOUNT_SWITCH_COUNT, KEY_RETRY_COUNT, KEY_RETRY_INTERVAL_SECONDS, KEY_RETRY_NO_RETRY_CODES, RETRY_CODE_MAX, RETRY_CODE_MIN, - RETRY_MAX_COUNT, RETRY_MAX_INTERVAL_SECONDS, RETRY_MAX_NO_RETRY_CODES, RETRY_MIN_COUNT, - RETRY_MIN_INTERVAL_SECONDS, + RETRY_MAX_COUNT, RETRY_MAX_INTERVAL_SECONDS, RETRY_MAX_NO_RETRY_CODES, RETRY_MAX_SWITCH_COUNT, + RETRY_MIN_COUNT, RETRY_MIN_INTERVAL_SECONDS, }; use crate::server::errors; use crate::server::http::{ok_json, parse_body}; @@ -67,7 +67,7 @@ pub async fn put_retry(State(_state): State, body: Bytes) -> Respon KEY_RETRY_ACCOUNT_SWITCH_COUNT, object.get(KEY_RETRY_ACCOUNT_SWITCH_COUNT), RETRY_MIN_COUNT, - RETRY_MAX_COUNT, + RETRY_MAX_SWITCH_COUNT, &mut patch.account_switch_count, ), ( @@ -96,9 +96,9 @@ pub async fn put_retry(State(_state): State, body: Bytes) -> Respon } } - // ── 第四项:指定错误码直接换号(整数数组)───────────────────── + // ── 第四项:例外错误码(整数数组,命中 = 允许把错误透传给下游)───────────────────── // 与三个数字项同一顺序:先整单校验,非法整体不落盘。允许 null / 缺省 - // (这一项不动);空数组是合法值 = 清空名单(任何错误都照常重试)。 + // (这一项不动);空数组是合法值 = 没有例外(任何错误都换号)。 if let Some(value) = object.get(KEY_RETRY_NO_RETRY_CODES) { if !value.is_null() { match parse_no_retry_codes(value) { @@ -147,7 +147,7 @@ fn retry_json(settings: RetrySettings) -> Value { }) } -/// 「指定错误码直接换号」的校验:整数数组、每项 100–599、最多 50 项。 +/// 「例外错误码」的校验:整数数组、每项 100–599、最多 50 项。 /// /// 与 `parse_bounded_int` 同一口径:只认 JSON 数字(含 `402.0` 这种整值浮点); /// 有一项非法就整单 400(不悄悄剔除),写盘前排序去重,落库形态规整。 diff --git a/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs b/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs index dff87dae0..6fa4c633a 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs @@ -21,6 +21,7 @@ impl Default for ErrorPolicySnapshot { pub(crate) fn from_raw(raw: &Map) -> Arc { if let Some(value) = raw.get(KEY_ERROR_POLICY) { if let Ok(settings) = serde_json::from_value::(value.clone()) { + let settings = upgrade_rules(settings); if settings.validate().is_ok() { return Arc::new(ErrorPolicySnapshot { settings, configured: true, legacy: false }); } } } @@ -46,12 +47,16 @@ pub(crate) fn from_raw(raw: &Map) -> Arc { if let Some(rule) = settings.rules.iter_mut().find(|rule| rule.kind == ErrorKind::RateLimit) { rule.fallback_seconds = 600; rule.exponential = true; } + // 旧「指定错误码直接换号」名单 → **例外**规则(允许把错误透传给下游)。 + // 语义在 #644 之后反转:默认所有上游错误都换号,名单表达的是「这几个码 + // 不换、直接返回客户端」。所以这里生成的是 `ReturnError` 例外,而不是 + // 原来的 `SwitchAccount`。 for status in retry.no_retry_codes.iter().copied().filter(|status| (400..600).contains(status)) { let kind = legacy_kind(status); - if matches!(kind, ErrorKind::BadRequest | ErrorKind::Content) { continue; } if let Some(mut rule) = settings.rules.iter().find(|rule| rule.kind == kind).cloned() { - rule.id = format!("legacy-{status}"); rule.name = format!("原直接换号设置 {status}"); rule.statuses = vec![status]; - rule.action = ErrorAction::SwitchAccount; + rule.id = format!("legacy-{status}"); rule.name = format!("原错误码名单例外 {status}"); + rule.statuses = vec![status]; rule.action = ErrorAction::ReturnError; + rule.cooldown = CooldownMode::None; if kind == ErrorKind::Unsupported { rule.scope = CooldownScope::Model; } settings.overrides.push(rule); } @@ -60,6 +65,47 @@ pub(crate) fn from_raw(raw: &Map) -> Arc { Arc::new(ErrorPolicySnapshot { settings, configured: false, legacy }) } +/// 把旧版本的存盘规则补到 [`RULES_VERSION`] 的口径。 +/// +/// 为什么必须做:规则表随版本出厂,用户在面板上真正关心的往往只是「要不要 +/// 换号」,冷却那几格常年停留在出厂值。#644 之前出厂值是「请求有误 / 内容被拒 / +/// 其他错误一律不冷却」,于是本地构造期的 400(例如网页工具字符预算超限) +/// 既白烧换号预算、又不会给失败的 账号×模型 留任何限制 —— 下一条请求继续 +/// 挑中它、继续 400。升级二进制却还带着旧存盘,表现就是「代码改了、线上原样」。 +/// +/// 升级只做**加法**,不动用户选的动作与数值: +/// 1. 冷却为空、而当前出厂值要求「固定冷却 / 失败退避」的规则,按出厂值补上 +/// (这就是「限制器」—— 有错就得挡一会儿,而不是任由下一条请求再撞); +/// 2. `未知错误` 的兜底动作若仍是「直接返回下游」,改回换号(旧出厂值就是这么 +/// 写的,而它和「路由组要保证下游不中断」直接冲突)。 +/// +/// 例外规则(`overrides`)是用户自己配的透传名单,任何时候都不动。 +fn upgrade_rules(mut settings: ErrorSettings) -> ErrorSettings { + if settings.rules_version >= RULES_VERSION { return settings; } + let stock = default_rules(); + let mut cooled = 0; + for rule in settings.rules.iter_mut() { + let Some(default) = stock.iter().find(|item| item.kind == rule.kind) else { continue }; + if rule.cooldown == CooldownMode::None + && matches!(default.cooldown, CooldownMode::Fixed | CooldownMode::Protection) + { + rule.cooldown = default.cooldown; + if rule.fallback_seconds == 0 { rule.fallback_seconds = default.fallback_seconds; } + rule.fallback_cap_seconds = rule.fallback_cap_seconds.max(rule.fallback_seconds); + cooled += 1; + } + if rule.kind == ErrorKind::Unknown && rule.action == ErrorAction::ReturnError { + rule.action = ErrorAction::SwitchAccount; + } + } + let versioned = settings.rules_version; + settings.rules_version = RULES_VERSION; + crate::server::logging::console_line("[Config]", &format!( + "错误处理规则已从旧版本(v{versioned})升级到 v{RULES_VERSION}:{cooled} 条规则补上冷却,未知错误恢复换号" + )); + settings +} + pub fn snapshot() -> Arc { super::CONFIG.read().ok().and_then(|guard| guard.as_ref().map(|config| config.error_policy.clone())).unwrap_or_default() } @@ -69,8 +115,12 @@ pub fn revision(snapshot: &ErrorPolicySnapshot) -> String { "legacy": snapshot.legacy }).to_string().as_bytes())) } -pub fn save(settings: ErrorSettings, expected: &str) -> Result, crate::server::errors::GatewayError> { +pub fn save(mut settings: ErrorSettings, expected: &str) -> Result, crate::server::errors::GatewayError> { use crate::server::errors::GatewayError; + // 前端提交的规则就是「当前版本的规则」:老前端不带 `rulesVersion`,被 + // serde 默认成 0 的话,保存一次就会被下一次加载当成旧配置又升级一遍, + // 用户刚改的动作会被默认值覆盖。所以保存时统一盖上当前版本号。 + settings.rules_version = RULES_VERSION; settings.validate().map_err(GatewayError::bad_request)?; super::try_update_raw_fields(|raw| { if revision(&from_raw(raw)) != expected { return Err(GatewayError::with_status(409, "错误处理设置已在其他窗口修改,请重新读取后保存")); } @@ -91,7 +141,7 @@ pub(crate) fn write_settings(raw: &mut Map, settings: &ErrorSetti for (key, value) in [(super::KEY_ROUTING_FAILOVER_BACKOFF, settings.backoff), (super::KEY_ROUTING_SINK, settings.sink), (super::KEY_ROUTING_BREAKER_ENABLED, settings.breaker)] { raw.insert(key.into(), value.into()); } raw.insert(super::KEY_RETRY_NO_RETRY_CODES.into(), settings.overrides.iter() - .filter(|rule| rule.id.starts_with("legacy-") && rule.enabled && rule.action == ErrorAction::SwitchAccount) + .filter(|rule| rule.id.starts_with("legacy-") && rule.enabled && rule.action == ErrorAction::ReturnError) .flat_map(|rule| rule.statuses.iter().copied()).map(Value::from).collect::>().into()); Ok(()) } @@ -111,10 +161,10 @@ pub(crate) fn sync_legacy(raw: &mut Map, retry_codes_changed: boo settings.overrides.retain(|rule| !rule.id.starts_with("legacy-")); for status in retry.no_retry_codes.iter().copied().filter(|status| (400..600).contains(status)) { let kind = legacy_kind(status); - if matches!(kind, ErrorKind::BadRequest | ErrorKind::Content) { continue; } if let Some(mut rule) = settings.rules.iter().find(|rule| rule.kind == kind).cloned() { - rule.id = format!("legacy-{status}"); rule.name = format!("原直接换号设置 {status}"); - rule.statuses = vec![status]; rule.action = ErrorAction::SwitchAccount; + rule.id = format!("legacy-{status}"); rule.name = format!("原错误码名单例外 {status}"); + rule.statuses = vec![status]; rule.action = ErrorAction::ReturnError; + rule.cooldown = CooldownMode::None; if kind == ErrorKind::Unsupported { rule.scope = CooldownScope::Model; } settings.overrides.insert(0, rule); } @@ -130,3 +180,44 @@ fn legacy_kind(status: u16) -> ErrorKind { 408 | 500 | 502 | 503 | 504 => ErrorKind::Transient, _ => ErrorKind::Unknown, } } + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::Map; + + /// 旧存盘里「所有规则都不冷却」时,升级要补上限制器,而不是放任旧口径。 + #[test] + fn stale_rules_get_cooldowns_on_upgrade() { + let mut settings = ErrorSettings::default(); + settings.rules_version = 0; + for rule in settings.rules.iter_mut() { rule.cooldown = CooldownMode::None; } + let mut raw = Map::new(); + raw.insert(KEY_ERROR_POLICY.into(), serde_json::to_value(&settings).expect("serialize")); + let snapshot = from_raw(&raw); + assert_eq!(snapshot.settings.rules_version, RULES_VERSION); + for rule in &snapshot.settings.rules { + if matches!(rule.kind, ErrorKind::BadRequest | ErrorKind::Content | ErrorKind::Unknown) { + assert_ne!(rule.cooldown, CooldownMode::None, "{} 升级后必须有冷却", rule.kind.id()); + } + } + assert!(snapshot.settings.validate().is_ok(), "升级后的设置必须自身合法"); + } + + /// 用户配的例外(透传名单)在升级时原样保留。 + #[test] + fn user_overrides_survive_upgrade() { + let mut settings = ErrorSettings::default(); + settings.rules_version = 0; + let mut rule = default_rules().into_iter().find(|rule| rule.kind == ErrorKind::BadRequest).expect("bad_request 规则"); + rule.id = "legacy-400".into(); rule.statuses = vec![400]; + rule.action = ErrorAction::ReturnError; rule.cooldown = CooldownMode::None; + settings.overrides.push(rule); + let mut raw = Map::new(); + raw.insert(KEY_ERROR_POLICY.into(), serde_json::to_value(&settings).expect("serialize")); + let snapshot = from_raw(&raw); + assert_eq!(snapshot.settings.overrides.len(), 1, "例外不能被升级逻辑丢掉"); + assert_eq!(snapshot.settings.overrides[0].action, ErrorAction::ReturnError); + assert_eq!(snapshot.settings.overrides[0].id, "legacy-400"); + } +} diff --git a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs index b79cbedd3..90258ec09 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs @@ -54,9 +54,31 @@ pub struct ErrorRule { fn all_providers() -> String { "*".into() } fn enabled() -> bool { true } +/// 换账号上限的上界:与转发层 `MAX_ROUTE_ATTEMPTS`(32)对齐。 +/// +/// 路由的意义是「试到有候选能答上来为止」,所以默认值直接取上界 —— 队列里 +/// 有几个可用账号就试几个,不再被一个 5 次的默认值提前截断。用户仍可调小。 +pub const MAX_SWITCHES: u64 = 32; + +/// 默认规则集的版本号。 +/// +/// 规则是**出厂默认**而不是用户手写的清单:升级到本版之后默认动作统一改成 +/// 「换号 / 跳过这一家」,并给请求有误类错误补上短冷却(避免同一个已知坏掉的 +/// 账号×模型被下一条请求反复挑中)。旧版本存下来的 `routingErrorPolicy` +/// 里那套 `return_error + 不冷却` 的组合必须跟着升级,否则用户升级了二进制 +/// 却还在跑旧口径 —— 见 `config::error_policy::from_raw` 的升级分支。 +pub const RULES_VERSION: u64 = 2; + #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase", deny_unknown_fields)] pub struct ErrorSettings { + /// 默认规则集的版本;0 = 本字段出现之前的旧配置。 + /// + /// 这里必须写字段级 `#[serde(default)]`:结构体上已有 `#[serde(default)]`, + /// 缺字段时会回落到 `ErrorSettings::default()` 的值(= 最新版本),那样旧 + /// 存盘就会被当成「已经是最新口径」而跳过升级。字段级默认取 0 才对。 + #[serde(default)] + pub rules_version: u64, pub resends: u64, pub switches: u64, pub interval_seconds: u64, @@ -78,7 +100,8 @@ pub struct ErrorSettings { impl Default for ErrorSettings { fn default() -> Self { Self { - resends: 3, switches: 5, interval_seconds: 5, wait_budget_seconds: 30, cooldown_floor_seconds: 30, + rules_version: RULES_VERSION, + resends: 0, switches: MAX_SWITCHES, interval_seconds: 5, wait_budget_seconds: 30, cooldown_floor_seconds: 30, backoff: true, backoff_seconds: 30, backoff_cap_seconds: 300, sink: true, sink_seconds: 1800, breaker: false, breaker_failures: 5, breaker_seconds: 60, breaker_successes: 2, rules: default_rules(), overrides: Vec::new(), @@ -94,23 +117,38 @@ pub fn default_rules() -> Vec { id: kind.id().into(), name: name.into(), kind, statuses: statuses.to_vec(), code: String::new(), provider: "*".into(), model: String::new(), enabled: true, action, cooldown, fallback_seconds, exponential, fallback_cap_seconds: 1800, scope, }; + // ── 默认动作统一为「换下一个候选」───────────────────────── + // 路由的职责是让下游不中断:上游任何一次失败都说明「这个账号 × 这个模型 + // 这一次没答上来」,换一个候选(同家另一个账号、另一家、路由组里的另一个 + // 成员模型)仍可能成功。所以除登录失效(先续期再换)与能力不支持(换家) + // 之外,全部直接顺延,**不做同一账号的原地重发** —— 那只会让客户端干等, + // 并且把「上游已经明确拒绝」当成「再试一次也许就好了」。 + // + // 冷却只给真正属于账号/额度状态的类型:限流、额度、余额、连接或服务异常 + // (退避 + 熔断计数)、登录失效(账号级)。请求本身有误(400/413/422)、 + // 内容被拒(内容策略)与能力不支持也**留一段短冷却**:不是为了惩罚账号, + // 而是为了让「刚在同一个 账号×模型 上失败过的组合」在接下来的几十秒里别 + // 再被同一条坏请求抢着试一遍 —— 这就是用户要的「限制器」。 + // + // 要「允许直接把错误返回下游」的码,走「提供商例外」(`overrides`)配置, + // 见 `config::error_policy` 对旧「指定错误码」名单的映射。 vec![ rule(RateLimit, "请求限流", &[429], SwitchAccount, Upstream, 60, false, CooldownScope::Auto), rule(Quota, "额度耗尽", &[429], SwitchAccount, Upstream, 600, true, CooldownScope::Auto), rule(Auth, "登录失效", &[401], RefreshThenSwitch, None, 60, false, CooldownScope::Account), rule(Balance, "余额不足", &[402], SwitchAccount, Upstream, 600, false, CooldownScope::Account), - rule(Transient, "连接或服务异常", &[408, 500, 502, 503, 504], RetryThenSwitch, Protection, 60, false, CooldownScope::Auto), - rule(Unsupported, "能力不支持", &[501], SwitchProvider, None, 60, false, CooldownScope::Auto), - rule(BadRequest, "请求有误", &[400, 413, 422], ReturnError, None, 60, false, CooldownScope::Model), - rule(Content, "内容被拒绝", &[], ReturnError, None, 60, false, CooldownScope::Model), - rule(Unknown, "其他错误", &[], ReturnError, None, 60, false, CooldownScope::Auto), + rule(Transient, "连接或服务异常", &[408, 500, 502, 503, 504], SwitchAccount, Protection, 60, false, CooldownScope::Auto), + rule(Unsupported, "能力不支持", &[501], SwitchProvider, Fixed, 60, false, CooldownScope::Model), + rule(BadRequest, "请求有误", &[400, 413, 422], SwitchAccount, Fixed, 60, false, CooldownScope::Model), + rule(Content, "内容被拒绝", &[], SwitchAccount, Fixed, 60, false, CooldownScope::Model), + rule(Unknown, "其他错误", &[], SwitchAccount, Protection, 60, false, CooldownScope::Auto), ] } impl ErrorSettings { pub fn validate(&self) -> Result<(), String> { let bounds = [ - (self.resends, 0, 10, "原账号重发次数"), (self.switches, 0, 10, "换账号上限"), + (self.resends, 0, 10, "原账号重发次数"), (self.switches, 0, MAX_SWITCHES, "换账号上限"), (self.interval_seconds, 0, 300, "重发间隔"), (self.wait_budget_seconds, 0, 300, "累计等待上限"), (self.cooldown_floor_seconds, 1, 60, "最短冷却"), (self.backoff_seconds, 1, 3600, "首次失败休息"), (self.backoff_cap_seconds, self.backoff_seconds, 3600, "最长失败休息"), (self.sink_seconds, 30, 3600, "降低优先级时长"), @@ -121,6 +159,7 @@ impl ErrorSettings { if !(min..=max).contains(&value) { return Err(format!("{name}必须是 {min}–{max} 的整数")); } } if self.rules.len() != 9 || self.overrides.len() > 100 { return Err("全局错误规则必须完整,提供商例外最多 100 条".into()); } + if self.rules_version > RULES_VERSION { return Err("这份设置来自更新的版本,请升级后再保存".into()); } let mut ids = std::collections::HashSet::new(); let mut kinds = std::collections::HashSet::new(); for (rule, global) in self.rules.iter().map(|rule| (rule, true)).chain(self.overrides.iter().map(|rule| (rule, false))) { @@ -136,8 +175,12 @@ impl ErrorSettings { if rule.kind != ErrorKind::Auth && rule.action == ErrorAction::RefreshThenSwitch { return Err("仅登录失效规则可以使用凭证续期".into()); } - if matches!(rule.kind, ErrorKind::BadRequest | ErrorKind::Content) && (rule.action != ErrorAction::ReturnError || rule.cooldown != CooldownMode::None) { - return Err("已识别的请求错误和内容拒绝不能设置为重复发送、换号或账号冷却".into()); + // 请求有误 / 内容被拒:既可以换号(默认)、跳过这一家,也可以透传下游; + // 冷却只能是「固定时长 / 失败退避 / 不冷却」—— 上游这类响应通常不带 + // `resetAt`,按上游时间冷却等于把账号无限期挂住。 + if matches!(rule.kind, ErrorKind::BadRequest | ErrorKind::Content) && (rule.cooldown == CooldownMode::Upstream + || !matches!(rule.action, ErrorAction::ReturnError | ErrorAction::SwitchAccount | ErrorAction::SwitchProvider)) { + return Err("已识别的请求错误和内容拒绝只能选择换账号、跳过该提供商或返回下游,且不能按上游时间冷却".into()); } if rule.kind == ErrorKind::Unknown && !rule.enabled { return Err("其他错误的兜底规则不能关闭".into()); } } @@ -153,3 +196,51 @@ impl ErrorSettings { .unwrap_or(&self.rules[0]) } } + +#[cfg(test)] +mod tests { + use super::*; + + /// 默认口径:除登录失效(先续期)与能力不支持(换家)之外,全部换号; + /// 一次原地重发都不做,换号上限覆盖整条候选队列。 + #[test] + fn defaults_fail_over_on_every_upstream_error() { + let settings = ErrorSettings::default(); + assert_eq!(settings.resends, 0, "默认不做同一账号原地重发"); + assert_eq!(settings.switches, MAX_SWITCHES, "默认换号上限要能走完候选队列"); + assert!(settings.validate().is_ok(), "默认设置必须自身合法"); + for rule in &settings.rules { + match rule.kind { + ErrorKind::Auth => assert_eq!(rule.action, ErrorAction::RefreshThenSwitch), + ErrorKind::Unsupported => assert_eq!(rule.action, ErrorAction::SwitchProvider), + _ => assert_eq!(rule.action, ErrorAction::SwitchAccount, "{} 默认应换号", rule.kind.id()), + } + } + } + + /// 请求有误 / 内容被拒默认也换号(并留短冷却);显式配成 ReturnError 依然合法。 + #[test] + fn bad_request_and_content_may_switch_or_passthrough() { + let settings = ErrorSettings::default(); + assert!(settings.validate().is_ok()); + let mut passthrough = ErrorSettings::default(); + for rule in &mut passthrough.rules { + if matches!(rule.kind, ErrorKind::BadRequest | ErrorKind::Content) { + rule.action = ErrorAction::ReturnError; + rule.cooldown = CooldownMode::None; + } + } + assert!(passthrough.validate().is_ok()); + } + + /// 换号上限的新上界必须容得下默认值。 + #[test] + fn switch_bounds_cover_default() { + assert!(MAX_SWITCHES >= 10); + let mut settings = ErrorSettings::default(); + settings.switches = MAX_SWITCHES; + assert!(settings.validate().is_ok()); + settings.switches = MAX_SWITCHES + 1; + assert!(settings.validate().is_err()); + } +} diff --git a/desktop-tauri/src-tauri/server/src/server/config/parse.rs b/desktop-tauri/src-tauri/server/src/server/config/parse.rs index 3487b5687..987321931 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/parse.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/parse.rs @@ -323,7 +323,7 @@ pub(super) fn retry_from(map: &Map) -> RetrySettings { KEY_RETRY_ACCOUNT_SWITCH_COUNT, defaults.account_switch_count, RETRY_MIN_COUNT, - RETRY_MAX_COUNT, + RETRY_MAX_SWITCH_COUNT, ), interval_seconds: bounded_int_field( map, @@ -336,14 +336,13 @@ pub(super) fn retry_from(map: &Map) -> RetrySettings { } } -/// 「指定错误码直接换号」名单的解析。 +/// **例外**错误码名单的解析(命中 = 允许把错误透传给下游、不换号)。 /// /// ── 各形态的落点 ──────────────────────────────────────────── -/// - 键缺失 / 不是数组 → **默认名单**([402]):配置从没写过这一项是常态, -/// 默认值就该在这时出现;整体写坏(手改成了字符串)与「没配过」无法区分, -/// 按同一口径处理(回落默认,见模块头)。 -/// - 空数组 → **空名单**:这是用户的明确选择(任何错误都照常重试), -/// 必须与「没配过」区分开。 +/// - 键缺失 / 不是数组 → **空名单**:默认就是「任何上游错误都换号」, +/// 没有例外。整体写坏(手改成了字符串)与「没配过」无法区分,按同一 +/// 口径处理(回落默认,见模块头)。 +/// - 空数组 → **空名单**:与默认一致,语义是「没有例外」。 /// - 数组里的垃圾项(非整数 / 越出 100–599)→ **丢弃该项**:接口写入侧 /// 已整单校验(400),能到这里的坏项只来自手改库,跳过比整单回落默认 /// 更接近手改者的本意。排序去重:判定只问 contains,顺序无意义, diff --git a/desktop-tauri/src-tauri/server/src/server/config/types.rs b/desktop-tauri/src-tauri/server/src/server/config/types.rs index 399fa5cb2..72275508e 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/types.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/types.rs @@ -634,27 +634,30 @@ pub const KEY_RETRY_ACCOUNT_SWITCH_COUNT: &str = "retryCrossProviderCount"; /// 两次重试之间的等待秒数 pub const KEY_RETRY_INTERVAL_SECONDS: &str = "retryIntervalSeconds"; -/// 同一账号的原地重发次数默认值:失败后再试 3 次(连同首发共 4 次发送) -pub const DEFAULT_RETRY_COUNT: i64 = 3; -/// 换账号的次数默认值:失败后最多换 5 个账号。 -/// -/// 比原地重发那档更宽是刻意的:换号的成本主要在「等到下一个可用账号」, -/// 而一个账号失败往往说明它这份额度确实不通,多换几个比快速失败更符合预期。 -pub const DEFAULT_RETRY_SWITCH_COUNT: i64 = 5; +/// 同一账号的原地重发次数默认值:0 —— 上游明确报错后不再原地重发。 +/// +/// 原地重发的成本是「客户端干等一个间隔」,收益只是「上游也许抖了一下」。 +/// 有别的候选时,换一个账号/模型的成功率高得多,所以默认直接顺延;需要 +/// 原地重发的人可以在设置里调大。 +pub const DEFAULT_RETRY_COUNT: i64 = 0; +/// 换账号的次数默认值:32 —— 与转发层 `MAX_ROUTE_ATTEMPTS` 对齐,等于「试完 +/// 队列里的全部候选」,不再用一个 5 次的默认值提前截断。 +pub const DEFAULT_RETRY_SWITCH_COUNT: i64 = 32; /// 重试间隔默认值:5 秒 pub const DEFAULT_RETRY_INTERVAL_SECONDS: i64 = 5; -/// 「指定错误码直接换号」的配置键(值是 HTTP 状态码数组,如 `[402, 429]`)。 +/// **例外**错误码名单的配置键(值是 HTTP 状态码数组,如 `[400]`)。 +/// +/// ── 语义在 #644 之后反转 ──────────────────────────────── +/// 现在默认是「任何上游错误都换号」,名单表达的是**例外**:命中的状态码 +/// **允许直接返回给下游、不换号**(例如调用方希望自己处理 400 的请求校验错误)。 /// -/// 键名里的 `NoRetry` 是历史措辞(最早的语义是「命中即报错」),行为后来 -/// 改成了「不在同一账号重发、直接换下一个账号」—— 键名是配置契约,改名 -/// 会让老配置读不到,保留至今。命中名单的上游失败跳过本账号:原地重发与 -/// 同账号补救(内容拦截换提示词 / 401 刷新)都不做,按队列换下一个账号 -/// 继续试,换满仍失败才把错误给客户端。 +/// 键名里的 `NoRetry` 是历史措辞(最早的语义是「命中即报错」,后来一度改成 +/// 「命中就直换」),键名是配置契约,改名会让老配置读不到,所以保留至今 —— +/// 语义以本节为准:**命中 = 允许把错误透传给下游(因此不再换号)**。 pub const KEY_RETRY_NO_RETRY_CODES: &str = "noRetryStatusCodes"; -/// 默认名单:402(WorkBuddy 积分不足)。余额问题重发结论不变, -/// 客户端拿到 402 才能如实体感「这个账号没钱了」。 -pub const DEFAULT_NO_RETRY_CODES: &[u16] = &[402]; +/// 默认名单:空 —— 任何上游错误都换号,不把错误透传给下游。 +pub const DEFAULT_NO_RETRY_CODES: &[u16] = &[]; /// 名单里状态码的合法范围:HTTP 状态码本身就定义在 100–599 pub const RETRY_CODE_MIN: u16 = 100; pub const RETRY_CODE_MAX: u16 = 599; @@ -669,6 +672,11 @@ pub const RETRY_MAX_NO_RETRY_CODES: usize = 50; /// 该档(不重发 / 不换号),间隔 0 = 立即重发。 pub const RETRY_MIN_COUNT: i64 = 0; pub const RETRY_MAX_COUNT: i64 = 10; +/// 「换账号次数」的上界:与转发层 `MAX_ROUTE_ATTEMPTS`(32)对齐。 +/// +/// 换账号不是「重发」:一份请求最多试几个候选,本身受循环上限约束,所以 +/// 这里放宽到 32,让默认值等于「试完队列里的全部候选」。 +pub const RETRY_MAX_SWITCH_COUNT: i64 = 32; pub const RETRY_MIN_INTERVAL_SECONDS: i64 = 0; pub const RETRY_MAX_INTERVAL_SECONDS: i64 = 300; @@ -691,11 +699,12 @@ pub struct RetrySettings { pub account_switch_count: i64, /// 两次重试之间的间隔(秒) pub interval_seconds: i64, - /// **指定错误码直接换号**名单([`KEY_RETRY_NO_RETRY_CODES`])。 + /// **例外**错误码名单([`KEY_RETRY_NO_RETRY_CODES`]):命中的上游状态码允许 + /// 直接返回给下游、不再换号。 /// - /// 为什么是 `Arc<[u16]>` 而不是 `Vec`:快照被逐失败请求取用, - /// `Arc` 让克隆只付一次指针自增;判定(`no_retry`)读的是共享切片, - /// 不需要任何锁。 + /// 默认空 = 任何上游错误都换号。为什么是 `Arc<[u16]>` 而不是 `Vec`: + /// 快照被逐失败请求取用,`Arc` 让克隆只付一次指针自增;判定(`no_retry`) + /// 读的是共享切片,不需要任何锁。 pub no_retry_codes: Arc<[u16]>, } @@ -718,7 +727,7 @@ impl RetrySettings { self.account_switch_count.max(0) as usize } - /// 这个上游状态码是否命中「直接换号」名单。 + /// 这个上游状态码是否在**例外**名单里(命中 = 允许直接把错误返回下游)。 pub fn no_retry(&self, status: u16) -> bool { self.no_retry_codes.contains(&status) } diff --git a/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs b/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs index a8310de33..777716590 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/routing_groups.rs @@ -138,18 +138,32 @@ pub fn project_with_vision(group: &RoutingGroup, accounts: &[Value], needs: &Req "providerLabel": providers::label_of(&member.provider), "model": actual, "name": actual, "state": "unavailable", "reason": reason })); continue; } + // 成员的参与账号:**空名单 = 该提供商的全部可用账号**(#644「全加入路由」)。 + // 少勾一个账号就等于少一条退路;路由组不该因为漏选而在失败时静默少候选。 + let mut candidates: Vec<(String, Value)> = Vec::new(); if member.accounts.is_empty() { - result.blocked.push(json!({ "id": member.id, "memberId": member.id, "provider": member.provider, "model": actual, - "name": actual, "state": "excluded", "reason": "该成员没有参与账号" })); - } - for id in &member.accounts { - let Some(account) = by_id.get(id.as_str()).filter(|account| routing::provider_of(account) == member.provider) else { - result.blocked.push(json!({ "id": id, "memberId": member.id, "provider": member.provider, "model": actual, - "name": id, "state": "unavailable", "reason": "参与账号已移除或不属于此提供商" })); + for account in accounts.iter().filter(|account| routing::provider_of(account) == member.provider + && account.get("enabled") != Some(&Value::Bool(false))) { + if let Some(id) = routing::account_id(account) { + candidates.push((id.to_string(), (*account).clone())); + } + } + if candidates.is_empty() { + result.blocked.push(json!({ "id": member.id, "memberId": member.id, "provider": member.provider, "model": actual, + "name": actual, "state": "excluded", "reason": "该提供商没有可用账号" })); continue; - }; - let mut account = (*account).clone(); - account["routingCandidateId"] = candidate_key(&member.id, id).into(); + } + } else { + for id in &member.accounts { + match by_id.get(id.as_str()).filter(|account| routing::provider_of(account) == member.provider) { + Some(account) => candidates.push((id.clone(), (*account).clone())), + None => result.blocked.push(json!({ "id": id, "memberId": member.id, "provider": member.provider, "model": actual, + "name": id, "state": "unavailable", "reason": "参与账号已移除或不属于此提供商" })), + } + } + } + for (id, mut account) in candidates { + account["routingCandidateId"] = candidate_key(&member.id, &id).into(); account["routingMemberId"] = member.id.clone().into(); account["routingModel"] = actual.clone().into(); account["routingPriority"] = result.accounts.len().into(); @@ -181,9 +195,15 @@ pub fn append_public(store: &AccountStore, scope: Option<&KeyScope>, data: &mut for group in config::routing_profiles().groups.values() { if !key_scope::allows_model(scope, &group.model) || data.iter().any(|item| model_id(item).eq_ignore_ascii_case(&group.model)) { continue; } let members: Vec<_> = group.members.iter().filter(|member| member.included && key_scope::allows_provider(scope, &member.provider) - && provider_active(&member.provider) - && accounts.available_providers.contains(&member.provider) && member.accounts.iter().any(|id| - accounts.accounts.iter().any(|account| routing::account_id(account) == Some(id) && routing::provider_of(account) == member.provider && account.get("enabled") != Some(&Value::Bool(false))))) + && provider_active(&member.provider) && accounts.available_providers.contains(&member.provider) + && if member.accounts.is_empty() { + // 空名单 = 该提供商的全部账号(与 `project` 同口径) + accounts.accounts.iter().any(|account| routing::provider_of(account) == member.provider + && account.get("enabled") != Some(&Value::Bool(false))) + } else { + member.accounts.iter().any(|id| accounts.accounts.iter().any(|account| routing::account_id(account) == Some(id) + && routing::provider_of(account) == member.provider && account.get("enabled") != Some(&Value::Bool(false)))) + }) .filter_map(|member| catalog_model(&member.provider, &member.model)).filter(|item| !providers::images::model_is_image(item)).collect(); if members.is_empty() { continue; } let mut item = json!({ "id": group.model, "name": group.name, "object": "model", "owned_by": "routing-group", "kind": "chat", "routing_group": true }); diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs index 66ee2c54f..7331023a3 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs @@ -56,7 +56,7 @@ pub(crate) fn decide_with(snapshot: &config::ErrorPolicySnapshot, provider: &str let settings = &snapshot.settings; let code = evidence::machine_code(body); let kind = evidence::kind(provider, class, status, &code); - let rule = settings.resolve(kind, status, &code, provider, model).clone(); + let mut rule = settings.resolve(kind, status, &code, provider, model).clone(); let upstream_scope = evidence::scope(body); let mut scope = match (rule.scope, upstream_scope) { (_, CooldownScope::Provider) => CooldownScope::Provider, @@ -65,11 +65,32 @@ pub(crate) fn decide_with(snapshot: &config::ErrorPolicySnapshot, provider: &str (scope, _) => scope, }; let account_model = matches!(code.as_str(), "account_model_unavailable" | "account_model_permissions_unknown" | "model_not_available_for_account"); - let hard_stop = explicit_stop || evidence::hard_stop(status, &code) - || matches!(kind, ErrorKind::BadRequest | ErrorKind::Content) - || local && status == 501 && !account_model && !matches!(code.as_str(), "unsupported" | "no_method" | "not_implemented" - | "route_not_supported" | "subscription_sharing_route_not_supported" | "subscription_sharing_unsupported_capability"); + // 「请求有误 / 内容被拒」不再写死成硬终止(#644):默认它们也换号, + // 因为换一家/换一个账号的审核与参数兼容性仍可能不同。想让它们直接回到 + // 下游,用例外名单(`overrides` 里的 `ReturnError`)显式表达。 + let hard_stop = explicit_stop || evidence::hard_stop(status, &code); let mut action = if hard_stop { ErrorAction::ReturnError } else { rule.action }; + // 没有拿到上游响应(status 0)= 链路/本地问题:换账号解决不了,避免把 + // 整个候选队列当成「试错池」白打一遍。 + if status == 0 { action = ErrorAction::ReturnError; } + // ── 本地就判定「这家吃不下这份请求」─────────────────────── + // 典型是网页工具(`toolEmulation`)按模型预算在**本机**判定的 400: + // `网页工具请求超过 16000 字符预算`。这类结论与账号无关 —— 同一家的 + // 另一个账号拿到的是同一份 body、同一个预算,撞的是同一堵墙。逐个账号 + // 试过去只会把换号预算烧在同一家身上,让真正能答上来的候选(例如不做 + // 工具模拟的模型)永远轮不到。 + // + // 所以这里升级成「跳过这家」:本次请求里把这家其余账号一并排除(见 + // `ErrorDecision::skips_provider`),同时给失败的 账号×模型 记一段冷却, + // 让下一条请求也不会先撞一次。冷却只落在模型维度,不是整家拉黑 —— + // 这家别的模型仍可能正常。 + let provider_local = matches!(kind, ErrorKind::BadRequest | ErrorKind::Unsupported) + && (evidence::provider_local(&code) || (local && matches!(status, 400 | 413 | 415 | 422 | 501))); + if provider_local && action != ErrorAction::ReturnError { + action = ErrorAction::SwitchProvider; + if scope == CooldownScope::Provider { scope = CooldownScope::Model; } + if rule.cooldown == CooldownMode::None { rule.cooldown = CooldownMode::Fixed; } + } if account_model { scope = CooldownScope::Model; if action == ErrorAction::SwitchProvider { action = ErrorAction::SwitchAccount; } @@ -119,13 +140,21 @@ pub(super) fn classified_error(class: &UpstreamErrorClass, body: &Value) -> Gate error } -pub(super) fn plugin_stops(provider: &str, policy: &crate::server::core::providers::policy::ErrorPolicy, - class: &UpstreamErrorClass, status: u16, body: &Value) -> bool { +/// 插件报的 `stop` 是否等于「整条请求到此为止」。 +/// +/// ── `stop` 不再顺手掐掉换号(#644)─────────────────────── +/// 插件声明 `stop` 的本义是「这份 body 不要再原地重发」,不是「换一个候选 +/// 也没救」。把它当成硬终止的后果是:账号级失败(被封、限额、WAF)静默收尾, +/// 队列里明明还有健康账号却一个都不试。现在只有两种情况算硬终止: +/// `stop_switching`(插件明确要求不要换号,例如提交状态不确定)与 +/// [`evidence::hard_stop`] 的证据类判据(重复执行 / 重复计费 / 已取消)。 +/// +/// `suspend_account` 不进这里 —— 停用当前账号之后,正该换下一个账号继续。 +pub(super) fn plugin_stops(_provider: &str, policy: &crate::server::core::providers::policy::ErrorPolicy, + _class: &UpstreamErrorClass, status: u16, body: &Value) -> bool { if !policy.stop { return false; } let code = evidence::machine_code(body); - let unsupported = evidence::kind(provider, class, status, &code) == ErrorKind::Unsupported; - policy.suspend_account || policy.stop_switching - || evidence::hard_stop(status, &code) || !unsupported + policy.stop_switching || evidence::hard_stop(status, &code) } pub(super) fn attach(error: &mut GatewayError, decision: &ErrorDecision) { @@ -165,3 +194,127 @@ pub(super) fn exclude_provider(service: &super::UpstreamService, providers: &[&s } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::server::config::error_policy::{ + default_rules, CooldownMode, ErrorAction, ErrorKind, ErrorPolicySnapshot, ErrorSettings, + }; + + fn snapshot() -> ErrorPolicySnapshot { + ErrorPolicySnapshot { settings: ErrorSettings::default(), configured: true, legacy: false } + } + + fn fatal(status: u16, message: &str) -> UpstreamErrorClass { + UpstreamErrorClass::Fatal { status, message: message.into(), upstream_code: None } + } + + fn decide(status: u16, body: serde_json::Value) -> ErrorDecision { + decide_with(&snapshot(), "demo", "demo-model", &fatal(status, "upstream said no"), status, &body, false, false) + } + + /// 429 默认就是「冷却这个账号 × 模型,然后换下一个候选」。 + #[test] + fn rate_limit_switches_and_cools_down() { + let decision = decide(429, serde_json::json!({ "message": "too many requests" })); + assert_eq!(decision.kind, ErrorKind::RateLimit); + assert_eq!(decision.action, ErrorAction::SwitchAccount); + assert!(decision.switches() && decision.cooldown()); + } + + /// 上游 403(被封 / 无权)属于账号级结论,必须顺延而不是收尾。 + #[test] + fn forbidden_switches_instead_of_stopping() { + let decision = decide(403, serde_json::json!({ "code": "forbidden" })); + assert!(!decision.hard_stop, "403 不再算硬终止"); + assert_eq!(decision.action, ErrorAction::SwitchAccount); + assert!(decision.switches()); + } + + /// 5xx 不再原地重发,第一次失败就换候选。 + #[test] + fn transient_does_not_resend_in_place() { + let decision = decide(502, serde_json::json!({ "message": "bad gateway" })); + assert_eq!(decision.kind, ErrorKind::Transient); + assert!(!decision.retries(), "默认不做同一账号原地重发"); + assert_eq!(decision.action, ErrorAction::SwitchAccount); + } + + /// 请求本身有误也换号(有别的候选仍可能成功),并留一段短冷却 —— + /// 「限制器」的意义:刚在这个 账号×模型 上失败过的组合,别让下一条请求 + /// 又抢着试一遍。 + #[test] + fn bad_request_switches_and_cools_briefly() { + let decision = decide(400, serde_json::json!({ "code": "invalid_request" })); + assert_eq!(decision.kind, ErrorKind::BadRequest); + assert_eq!(decision.action, ErrorAction::SwitchAccount); + assert!(decision.switches()); + assert!(decision.cooldown(), "请求有误也要落一段冷却,避免反复挑中同一个坏组合"); + assert_eq!(decision.scope, CooldownScope::Model, "冷却只落在模型维度,不整家拉黑"); + } + + /// 例外名单(旧「指定错误码」键,语义已反转为「允许透传下游」)命中时 + /// 不再换号,直接把错误交回客户端。 + #[test] + fn passthrough_exception_returns_error() { + let mut settings = ErrorSettings::default(); + let mut rule = default_rules().into_iter().find(|rule| rule.kind == ErrorKind::BadRequest).unwrap(); + rule.id = "legacy-400".into(); + rule.name = "原错误码名单例外 400".into(); + rule.statuses = vec![400]; + rule.action = ErrorAction::ReturnError; + rule.cooldown = CooldownMode::None; + settings.overrides.push(rule); + let snapshot = ErrorPolicySnapshot { settings, configured: true, legacy: false }; + let decision = decide_with(&snapshot, "demo", "demo-model", + &fatal(400, "upstream said no"), 400, &serde_json::json!({ "code": "invalid_request" }), false, false); + assert_eq!(decision.action, ErrorAction::ReturnError); + assert!(!decision.switches(), "命中例外就不该再换号"); + } + + /// 网页工具预算超限是**本地**按模型 profile 判定的 400:同一家的另一个 + /// 账号拿到的是同一份 body、同一个预算,撞的是同一堵墙。所以要直接跳过 + /// 这一家(而不是把预算逐个账号烧下去),并给失败的 账号×模型 记冷却。 + #[test] + fn web_tools_budget_error_skips_provider() { + let decision = decide(400, serde_json::json!({ "code": "web_tools_invalid_request" })); + assert_eq!(decision.action, ErrorAction::SwitchProvider); + assert!(decision.switches(), "跳过这一家也要继续顺延"); + assert!(decision.skips_provider(), "本次请求不该再试这一家的其他账号"); + assert!(decision.cooldown(), "失败组合要落冷却,别让下一条请求再撞一次"); + assert_eq!(decision.scope, CooldownScope::Model); + } + + /// 真正「本地构造就失败」的 400(`local = true`)同样按跳过这家处理。 + #[test] + fn local_build_failure_skips_provider() { + let decision = decide_with(&snapshot(), "demo", "demo-model", + &fatal(400, "网页工具请求超过 16000 字符预算"), 400, &serde_json::json!({}), true, false); + assert_eq!(decision.action, ErrorAction::SwitchProvider); + assert!(decision.skips_provider()); + } + + /// 没有拿到上游响应(status 0)是链路问题:换账号解决不了,保持收尾。 + #[test] + fn transport_failure_does_not_switch() { + let decision = decide(0, serde_json::json!({ "message": "connect error" })); + assert_eq!(decision.action, ErrorAction::ReturnError); + assert!(!decision.switches()); + } + + /// 插件报 `stop` 只表示「别再原地重发」,不再顺手掐掉换号。 + #[test] + fn plugin_stop_does_not_block_switching() { + let policy = crate::server::core::providers::policy::ErrorPolicy { + stop: true, stop_switching: false, account_cooldown_until: None, suspend_account: false, + reason: String::new(), + }; + assert!(!plugin_stops("demo", &policy, &fatal(503, "busy"), 503, &serde_json::json!({}))); + let explicit = crate::server::core::providers::policy::ErrorPolicy { + stop: true, stop_switching: true, account_cooldown_until: None, suspend_account: false, + reason: String::new(), + }; + assert!(plugin_stops("demo", &explicit, &fatal(503, "busy"), 503, &serde_json::json!({}))); + } +} diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs index 94305ffa6..60fdbd8bb 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs @@ -48,6 +48,16 @@ pub(super) fn kind(provider: &str, class: &UpstreamErrorClass, status: u16, code ErrorKind::Unknown } +/// 「这份 body 本身就发不出去」的本地业务码 —— 换账号无用,得换一家。 +/// +/// 这些码由本机在**发送前**生成(网页工具的字符预算、视觉兼容的前置校验), +/// 它们的结论是「提供商 × 模型能力吃不下这份请求」,与具体账号无关。 +/// 判据按码前缀匹配:新增同类能力检查时只要沿用同一前缀,就自动获得 +/// 「跳过这一家 + 落模型冷却」的处置。 +pub(super) fn provider_local(code: &str) -> bool { + ["web_tools_", "vision_", "native_payload_"].iter().any(|prefix| code.starts_with(prefix)) +} + pub(super) fn scope(body: &Value) -> CooldownScope { match body.get("credentialScope").or_else(|| body.get("credential_scope")).or_else(|| body.get("scope")).and_then(Value::as_str) { Some("provider") => CooldownScope::Provider, @@ -100,15 +110,22 @@ pub(in crate::server::core::upstream) fn gateway_class(error: &GatewayError) -> }) } +/// 「不要再换号、直接把错误交回下游」的硬终止判据。 +/// +/// ── 为什么 403 / 权限类不再在这里 ───────────────────────── +/// 上游 403(账号被风控、凭证对该模型无权、订阅级禁止)是**账号级**结论: +/// 换下一个账号或另一家仍可能成功,属于「应该顺延」而不是「整条请求没救」。 +/// #644 之后它们走默认的换号规则(并落账号级冷却/熔断)。这里只保留 +/// 「再发一次可能造成重复执行/重复计费/状态错乱」与「用户已取消」两类。 pub(super) fn hard_stop(status: u16, code: &str) -> bool { - matches!(status, 403 | 409 | 428 | 499) + matches!(status, 409 | 428 | 499) || code.ends_with("cleanup_unconfirmed") || code.starts_with("claude_session_") - || matches!(code, "permission_denied" | "access_denied" | "permission_error" | "validation_required" + || matches!(code, "validation_required" | "tabbit_submission_unconfirmed" | "tabbit_account_changed" | "tabbit_state_conflict" | "claude_response_started" | "account_origin_authorization_required" | "plugin_busy" | "gateway_busy" | "account_busy" | "plugin_response_started" | "plugin_resource_limit" - | "plugin_protocol_error" | "plugin_trap" | "forbidden" | "native_helper_not_authorized" | "native_helper_unavailable" - | "request_replay_blocked" | "web_tools_invalid_request" | "gateway_capability_missing" | "aborted" | "cancelled" + | "plugin_protocol_error" | "plugin_trap" | "native_helper_not_authorized" | "native_helper_unavailable" + | "request_replay_blocked" | "gateway_capability_missing" | "aborted" | "cancelled" | "upstream_submission_unconfirmed" | "runtime_invalid_error_metadata" | "runtime_protocol_error" | "runtime_stream_interrupted" | "runtime_classification_unavailable" | "runtime_transport_unavailable" | "network_cleanup_unconfirmed" | "runtime_upstream_error" | "builtin_runtime_error" | "runtime_sdk_content") diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs index d9d98b012..86a257f37 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs @@ -8,7 +8,15 @@ use super::{error_policy, rotate, RouteTarget, UpstreamService}; use super::payload::ProviderContext; pub(super) fn finish_failure(service: &UpstreamService, ctx: &ProviderContext<'_>, providers: &[&str], keys: &rotate::CooldownKeys<'_>, - target: &RouteTarget, error: &mut GatewayError, tried: &mut Vec, switches_left: &mut usize, switch_total: usize) -> bool { + target: &RouteTarget, error: &mut GatewayError, tried: &mut Vec, switches_left: &mut usize, switch_total: usize, + conv_key: &str, sticky_model: &str, sticky_generation: u64) -> bool { + // 失败即解除会话绑定:粘性只在成功时建立,绑着一个刚失败的候选只会让 + // 同一会话的下一条请求先撞一次同样的错误(429 时表现为「每次都先吃一个 + // 429」)。解绑放在最前面 —— 哪怕这一次最终收尾(hard_stop / 无候选), + // 也不该把失败账号继续粘在这个会话上。 + if let Some(candidate) = target.attempt_key() { + service.route.sticky_forget(conv_key, sticky_model, &candidate, sticky_generation); + } let model = error.routing_error.as_ref().map(|decision| decision.model.clone()).unwrap_or_else(|| { target.account.as_ref().map(|account| keys.for_account(account)).unwrap_or_else(|| keys.for_provider(&target.provider)) }); diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_send.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_send.rs index c86736629..201f778d7 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_send.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_send.rs @@ -46,7 +46,12 @@ pub(super) async fn send_with_retry( if telemetry.replay_guard().blocked() || telemetry.is_cancelled() { return Err(stopped(if telemetry.is_cancelled() { super::cancellation::cancelled_error() } else { gateway })); } - let decision = error_policy::for_error_with(&budget.policy, adapter.provider_id(), &budget.model, &gateway, !error.safe_to_retry); + let mut decision = error_policy::for_error_with(&budget.policy, adapter.provider_id(), &budget.model, &gateway, !error.safe_to_retry); + // 传输层失败(连不上 / 写不出去 / 读不回来)换成别的账号也一样: + // 这类问题出在本机链路或出口上,换号只会把整个队列白打一遍。 + // 保持「原地处理 + 原样收尾」,与 #644 的口径不冲突 —— 那条规则的 + // 主语是「上游返回的错误」。 + if error.network_failure { decision.action = crate::server::config::error_policy::ErrorAction::ReturnError; } error_policy::attach(&mut gateway, &decision); error_policy::note(telemetry, &decision, None); if decision.hard_stop { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs index 2453e1601..eeececd05 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs @@ -674,7 +674,20 @@ async fn attempt_queue( // 结果是「能换几个账号」根本没人管 —— 某家囤了 9 个账号时一次请求会一路 // 试到第 10 个,而用户填的那个数字要等跨家才生效。 let mut budget = RetryBudget::from_snapshot(ctx.error_policy.clone()); - let switch_total = ctx.error_policy.settings.switches as usize; + // ── 换号预算的**下限**是整条候选队列 ───────────────────────── + // 用户填的 `switches` 表达「最多换几次」,但它不能比候选队列还短: + // 队列里有 N 个没试过的候选时提前收尾,就是把「换一个本来能答上来的 + // 账号/提供商」变成「直接 400 给下游」。生产上的形状就是 21 个候选、 + // 默认 10 次预算 —— 试到第 10 个(全是同一家的工具预算 400)就收尾, + // 而排在第 12 位、不做工具模拟的那家其实一条就 200。 + // + // 所以取 `max(用户值, 队列规模)`,再受转发层的 `MAX_ROUTE_ATTEMPTS` + // 约束(那才是防死循环的硬闸)。 + let queue_cap = rotate::scoped_account_cap(service, provider_ids, cooldown_keys.request_model(), + ctx.pinned_account, Some(&ctx.routing)); + // 配成 0 是显式的「不换号」,尊重它;其余情况一律按队列规模兜底。 + let configured = ctx.error_policy.settings.switches as usize; + let switch_total = if configured == 0 { 0 } else { configured.max(queue_cap).min(MAX_ROUTE_ATTEMPTS) }; let mut switches_left = switch_total; let mut decision = ctx.telemetry.route_decision().unwrap_or_else(|| json!({})); decision["errorPolicyConfig"] = json!({ "revision": config::error_policy::revision(&ctx.error_policy), @@ -798,7 +811,8 @@ async fn attempt_queue( } let mut error = error; if super::error_policy_finish::finish_failure(service, ctx, provider_ids, &cooldown_keys, - &failed_target, &mut error, &mut tried_ids, &mut switches_left, switch_total) { + &failed_target, &mut error, &mut tried_ids, &mut switches_left, switch_total, + &conv_key, &model, sticky_generation) { last_error = Some(error); continue 'accounts; } return Err(error); @@ -874,7 +888,7 @@ async fn attempt_queue( if ctx.telemetry.replay_guard().blocked() { return Err(error); } if super::vision_compat::handles_error(ctx, &error) { return Err(error); } if matches!(error.code.as_deref(), Some("plugin_busy" | "gateway_busy" | "account_busy" | "plugin_resource_limit" - | "plugin_protocol_error" | "plugin_trap" | "plugin_response_started" | "web_tools_invalid_request")) { return Err(error); } + | "plugin_protocol_error" | "plugin_trap" | "plugin_response_started")) { return Err(error); } let account_unavailable = matches!(error.code.as_deref(), Some("account_model_unavailable" | "account_model_permissions_unknown")); if ctx.images.is_some() && !account_unavailable { return Err(error); } @@ -895,7 +909,8 @@ async fn attempt_queue( } let mut error = error; if super::error_policy_finish::finish_failure(service, ctx, provider_ids, &cooldown_keys, - &failed_target, &mut error, &mut tried_ids, &mut switches_left, switch_total) { + &failed_target, &mut error, &mut tried_ids, &mut switches_left, switch_total, + &conv_key, &model, sticky_generation) { last_error = Some(error); continue 'accounts; } return Err(error); @@ -1146,21 +1161,17 @@ async fn attempt_queue( // 个模型名在别几家都能答。换号顺延本来就不该只看「发出去之后 // 上游怎么回」:既然本地就判定不可用,队列里的下一家有机会。 // - // 其余构造期错误(400 缺模型名 / 转换失败、401 账号缺凭证、 - // 500 内部坏了)继续**直接终止**:那些是「这份请求本身坏了」, - // 换个账号发的是同一份坏请求,换满预算只会把一次 400 变成四条 - // 400,并把真实原因埋进顺延链里。 - let account_unavailable = error.code.as_deref().is_some_and(|code| { - matches!(code, "account_model_unavailable" | "account_model_permissions_unknown") - }); + // 其余构造期错误(网页工具字符预算超限这类 400、账号对模型无 + // 权限、转换失败)同样交给**统一错误规则**判定(#644):规则会 + // 区分「换一个账号仍可能好」(换号)、「同一家换账号也没用」(跳过 + // 这一家)与「再发一次可能重复执行」(硬终止)。在这里写死 + // `return Err` 等于绕过用户的整套错误处理设置。 if i32::from(readiness::LOCAL_PRECONDITION_STATUS) != error.status_code { - if account_unavailable || error.status_code == 501 { - let mut error = error; - if super::error_policy_finish::finish_failure(service, ctx, provider_ids, &cooldown_keys, - &target, &mut error, &mut tried_ids, &mut switches_left, switch_total) { - last_error = Some(error); continue 'accounts; - } - return Err(error); + let mut error = error; + if super::error_policy_finish::finish_failure(service, ctx, provider_ids, &cooldown_keys, + &target, &mut error, &mut tried_ids, &mut switches_left, switch_total, + &conv_key, &model, sticky_generation) { + last_error = Some(error); continue 'accounts; } return Err(error); } @@ -1559,7 +1570,8 @@ async fn attempt_queue( if ctx.telemetry.replay_guard().blocked() { return Err(failure.error); } let mut error = failure.error; if super::error_policy_finish::finish_failure(service, ctx, provider_ids, &cooldown_keys, - &target, &mut error, &mut tried_ids, &mut switches_left, switch_total) { + &target, &mut error, &mut tried_ids, &mut switches_left, switch_total, + &conv_key, &model, sticky_generation) { last_error = Some(error); continue 'accounts; } return Err(error); @@ -1664,7 +1676,8 @@ async fn attempt_queue( super::error_policy::attach(&mut error, &decision); ctx.telemetry.finish_last_attempt(Some(i64::from(error.status_code)), Some(&error.message)); if super::error_policy_finish::finish_failure(service, ctx, provider_ids, &cooldown_keys, - &target, &mut error, &mut tried_ids, &mut switches_left, switch_total) { + &target, &mut error, &mut tried_ids, &mut switches_left, switch_total, + &conv_key, &model, sticky_generation) { last_error = Some(error); continue 'accounts; } return Err(error); diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs index c9aeabff3..805f02e04 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs @@ -502,12 +502,7 @@ pub(super) fn scoped_accounts( pinned: Option<&str>, request_routing: Option<&super::route_context::RequestRouting>, ) -> Vec { - let accounts = accounts_in_providers(service, providers, pinned); - let accounts = if pinned.is_some() && !request_routing.is_some_and(|routing| routing.strict_vision) { accounts } else if let Some(request_routing) = request_routing { - request_routing.accounts(accounts) - } else { - crate::server::core::routing_scope::apply_accounts(&accounts, model) - }; + let accounts = scoped_in_providers(service, providers, model, pinned, request_routing); let keys = routing::CooldownKeys::new(model); let accounts: Vec = accounts.into_iter().map(|mut account| { let provider = provider_of(&account).to_string(); @@ -525,6 +520,41 @@ pub(super) fn scoped_accounts( if routable.is_empty() { accounts } else { routable } } +/// 候选账号的**作用域过滤**(路由组 / 指定账号 / 路由作用范围),不含冷却判定。 +fn scoped_in_providers( + service: &UpstreamService, + providers: &[&str], + model: &str, + pinned: Option<&str>, + request_routing: Option<&super::route_context::RequestRouting>, +) -> Vec { + let accounts = accounts_in_providers(service, providers, pinned); + if pinned.is_some() && !request_routing.is_some_and(|routing| routing.strict_vision) { + accounts + } else if let Some(request_routing) = request_routing { + request_routing.accounts(accounts) + } else { + crate::server::core::routing_scope::apply_accounts(&accounts, model) + } +} + +/// 这条请求的候选池**规模**(不含冷却 / 限额 / 熔断判定)。 +/// +/// 用途只有一个:给「最多换几个账号」定一条**下限**。队列里有多少候选, +/// 这条请求至少就该换几次 —— 否则一条 10 个候选的队列配上默认 5 次的预算, +/// 就会出现「第 6 个候选本来能答上来,却因为预算用完直接 400 给下游」。 +/// 这里刻意不逐账号查冷却(那是 `scoped_accounts` 的活):拿到的是上界, +/// 宁可多给几个额度(真失败时队列自己会把名额用光),也不要提前截断。 +pub(super) fn scoped_account_cap( + service: &UpstreamService, + providers: &[&str], + model: &str, + pinned: Option<&str>, + request_routing: Option<&super::route_context::RequestRouting>, +) -> usize { + scoped_in_providers(service, providers, model, pinned, request_routing).len() +} + /// 组装选路结果;代理解析失败时把提示**带进选路结果**(本次回退直连)。 /// /// ── 提示为什么是返回值的一部分而不是当场打日志 ──────────────── diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs index a2af2719a..0a84cc34d 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/route_state.rs @@ -335,6 +335,11 @@ impl RouteState { self.sticky.bind(key, model, account_id, source, now, ttl_ms, generation); } + /// 失败时解除该会话对某个候选的绑定(见 `StickySessions::forget`)。 + pub(super) fn sticky_forget(&self, key: &str, model: &str, candidate: &str, generation: u64) { + self.sticky.forget(key, model, candidate, generation); + } + pub(super) fn sticky_list(&self, model: Option<&str>, now: i64) -> Vec { let (profiles, base) = crate::server::config::routing_snapshot(); self.sticky.list(model, now, |name| profiles.settings_for(name, base).sticky.ttl_ms().unwrap_or(0)) diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/route_sticky.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/route_sticky.rs index 0d762c5bd..5f02dd363 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/route_sticky.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/route_sticky.rs @@ -109,6 +109,21 @@ impl StickySessions { })).collect() } + /// 某个候选失败后解除该会话的绑定(仅当绑定的正是这个候选)。 + /// + /// 粘性只在成功时建立;失败不解除的后果是「同一会话的下一条请求仍然先撞 + /// 同一个失败账号」—— 429 时表现为「这个会话每次先吃一个 429」。 + /// `generation` 与 `bind` 同款守卫:期间有人手动清空过,就不动新状态。 + pub(super) fn forget(&self, key: &str, model: &str, candidate: &str, generation: u64) { + if key.is_empty() || candidate.is_empty() { return; } + let Ok(mut table) = self.table.lock() else { return }; + if table.generation != generation { return; } + let id = entry_id(key, model); + if table.entries.get(&id).is_some_and(|entry| entry.account_id == candidate) { + table.entries.remove(&id); + } + } + pub(super) fn clear(&self, id: Option<&str>, model: Option<&str>) -> usize { let Ok(mut table) = self.table.lock() else { return 0 }; let before = table.entries.len(); diff --git a/desktop-tauri/ui-shared/routing-errors-panel.tsx b/desktop-tauri/ui-shared/routing-errors-panel.tsx index 17f13c45b..3258733ca 100644 --- a/desktop-tauri/ui-shared/routing-errors-panel.tsx +++ b/desktop-tauri/ui-shared/routing-errors-panel.tsx @@ -85,7 +85,7 @@ export function RoutingErrorsPanel({ api, t, active = true, identity = '', reloa }}>{t('恢复建议值')}
patch({ resends })} /> - patch({ switches })} /> + patch({ switches })} /> patch({ intervalSeconds })} /> patch({ waitBudgetSeconds })} />
}> diff --git a/desktop-tauri/ui-shared/routing-errors-types.ts b/desktop-tauri/ui-shared/routing-errors-types.ts index 614af4d22..d4f221474 100644 --- a/desktop-tauri/ui-shared/routing-errors-types.ts +++ b/desktop-tauri/ui-shared/routing-errors-types.ts @@ -50,7 +50,7 @@ export function cooldownSummary(rule: ErrorRule, settings: ErrorSettings, t: Err } export function errorPolicyValidation(settings: ErrorSettings, t: ErrorTranslate): string | null { const fields: [number, number, number, string][] = [ - [settings.resends, 0, 10, t('原账号重发')], [settings.switches, 0, 10, t('最多再换账号')], + [settings.resends, 0, 10, t('原账号重发')], [settings.switches, 0, 32, t('最多再换账号')], [settings.intervalSeconds, 0, 300, t('重发间隔')], [settings.waitBudgetSeconds, 0, 300, t('累计等待上限')], [settings.cooldownFloorSeconds, 1, 60, t('最短限流冷却')], [settings.backoffSeconds, 1, 3600, t('首次休息时间')], [settings.backoffCapSeconds, settings.backoffSeconds, 3600, t('连续失败最长休息')], [settings.sinkSeconds, 30, 3600, t('最长保留时间')], From b69f123215df164bca8da888a97429c0ec69bfbc Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 17:28:21 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E8=A1=A5=E9=BD=90=20tauri=20dev=20?= =?UTF-8?q?=E7=BC=BA=E7=9C=81=E4=BA=8C=E8=BF=9B=E5=88=B6=EF=BC=9Asrc-tauri?= =?UTF-8?q?=20=E4=B8=8E=20server=20=E5=90=84=E6=9C=89=E4=B8=80=E4=B8=AA=20?= =?UTF-8?q?bin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit workspace 拆出 `server/` 之后,`desktop-tauri` 与它各有一个 bin,`tauri dev` 调的是不带 `--bin` 的裸 `cargo run`,会报 `could not determine which binary to run`。桌面端是默认入口,补 `default-run = "workbuddy-proxy-desktop"`。 --- desktop-tauri/src-tauri/Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/desktop-tauri/src-tauri/Cargo.toml b/desktop-tauri/src-tauri/Cargo.toml index 9e92ed4bb..fa40e622d 100644 --- a/desktop-tauri/src-tauri/Cargo.toml +++ b/desktop-tauri/src-tauri/Cargo.toml @@ -9,6 +9,10 @@ edition = "2021" # 要求 1.88,会让「照着 rust-version 装旧工具链」的构建直接失败。 rust-version = "1.88" license = "AGPL-3.0-only" +# workspace 拆出 `server/` 之后本 crate 与它各有一个 bin,`tauri dev` 调的是 +# 裸 `cargo run`(不带 --bin):不指定 default-run 就会报 +# 「could not determine which binary to run」。桌面端是默认入口。 +default-run = "workbuddy-proxy-desktop" repository = "https://github.com/aimod-cc/Agent2API" homepage = "https://github.com/aimod-cc/Agent2API" From 64ef458fc9cfbadcaa29cb0d10f1b6ac07a31a43 Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 18:47:40 +0800 Subject: [PATCH 3/7] =?UTF-8?q?429=20=E9=87=8C=E7=9A=84=E3=80=8C=E7=A7=AF?= =?UTF-8?q?=E5=88=86/=E5=A5=97=E9=A4=90=E3=80=8D=E6=8C=89=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E7=BA=A7=E5=A4=84=E7=90=86=EF=BC=9A=E5=90=8C=E4=B8=80?= =?UTF-8?q?=E4=B8=AA=E8=B4=A6=E5=8F=B7=E4=B8=8D=E5=86=8D=E6=8D=A2=E7=9D=80?= =?UTF-8?q?=E6=A8=A1=E5=9E=8B=E6=92=9E=E4=B8=A4=E6=AC=A1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 生产日志里看到的形状(`attempts=9`): ``` 7. loomy/2793/spark-x2.6 429 rate_limit scope=model 积分余额不足 8. loomy/2793/deepseek-v4-flash 429 rate_limit scope=model 积分余额不足 9. zcode/8800…-2793/glm-5.3 429 quota scope=model 套餐池没有有效套餐 ``` 同一个 Loommy 账号连着出现两次,看起来像「429 没换模型」,实际是路由组把它 当成两个候选(成员模型不同):第一次失败后冷却只落在 `spark-x2.6` 这个模型 键上,另一个成员模型照旧可选,同一个账号再撞一次同一堵墙。 改动: - 新增 `evidence::resource_kind`:429/402 的文案里出现「积分余额不足 / 欠费 / 没有有效套餐 / 额度已用尽 / quota / usage limit」时,把它还原成 `balance` / `quota`,而不是一律算「请求限流」;文案明确写着 `rate limit / too many requests / 限流` 时保持限流,不误判。 - 限流 / 额度 / 余额这类消耗的是**账号**的池子,冷却与本次请求的排除都升到 账号级(上游声明整家共享时仍保留 `provider`)。账号级冷却走 `routingCooldowns["@account"]`,同时 `finish_failure` 会把这个账号的其余 候选一起从本次队列里排掉 —— 一个账号只撞一次。 - 单测:429 包 402 积分余额不足 → `balance` + 账号级;套餐池无有效套餐 → `quota` + 账号级;真限流 → 账号级;上游声明 provider 时保留 provider。 --- .../src/server/core/upstream/error_policy.rs | 51 ++++++++++++++++++- .../core/upstream/error_policy_evidence.rs | 28 ++++++++-- 2 files changed, 75 insertions(+), 4 deletions(-) diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs index 7331023a3..d3d7d8ded 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy.rs @@ -55,7 +55,13 @@ pub(crate) fn decide_with(snapshot: &config::ErrorPolicySnapshot, provider: &str status: u16, body: &Value, local: bool, explicit_stop: bool) -> ErrorDecision { let settings = &snapshot.settings; let code = evidence::machine_code(body); - let kind = evidence::kind(provider, class, status, &code); + let mut kind = evidence::kind(provider, class, status, &code); + // 429/402 里包着的「积分/余额/套餐」按文案还原(见 `evidence::resource_kind`): + // 这类是账号级资源耗尽,方向是「排掉这个账号、换下一家」,而不是「等一会儿 + // 在同一账号重试」。不还原的话,路由组会拿同一个账号换着成员模型再撞一次。 + if matches!(kind, ErrorKind::RateLimit | ErrorKind::Quota | ErrorKind::Balance | ErrorKind::Unknown) { + if let Some(actual) = evidence::resource_kind(body.get("message").and_then(Value::as_str).unwrap_or("")) { kind = actual; } + } let mut rule = settings.resolve(kind, status, &code, provider, model).clone(); let upstream_scope = evidence::scope(body); let mut scope = match (rule.scope, upstream_scope) { @@ -98,6 +104,13 @@ pub(crate) fn decide_with(snapshot: &config::ErrorPolicySnapshot, provider: &str if matches!(kind, ErrorKind::Auth | ErrorKind::Balance) && rule.scope == CooldownScope::Auto && scope != CooldownScope::Provider { scope = CooldownScope::Account; } + // 限流 / 额度 / 余额消耗的是**账号**的池子,不是某个模型的独立额度:冷却和 + // 本次请求的排除都按账号维度走。上游若声成整家共享(Provider)则保留, + // 其余(含上游写 `scope: model` 的)一律升到账号级 —— 否则同一个账号会 + // 被路由组的不同成员模型各撞一次,日志里就是「连着两个 429 没换模型」。 + if matches!(kind, ErrorKind::RateLimit | ErrorKind::Quota) && scope != CooldownScope::Provider && !account_model { + scope = CooldownScope::Account; + } if kind != ErrorKind::Auth && action == ErrorAction::RefreshThenSwitch { action = ErrorAction::ReturnError; } let now = logging::now_ms(); let mut reset_at = evidence::reset(body, now); @@ -295,6 +308,42 @@ mod tests { assert!(decision.skips_provider()); } + /// 429 里包着的「积分余额不足」(聚合上游常把它套在 429 上)是账号级资源 + /// 耗尽:按账号冷却,本次请求把这个账号的其余候选一起排掉 —— 否则路由组会 + /// 拿同一个账号换着成员模型再撞一次,日志里就是连着两条 429。 + #[test] + fn balance_wrapped_in_429_cools_account() { + let decision = decide(429, serde_json::json!({ "message": "上游返回 429: 上游返回 402: 积分余额不足,请获取积分后再试" })); + assert_eq!(decision.kind, ErrorKind::Balance); + assert_eq!(decision.action, ErrorAction::SwitchAccount); + assert_eq!(decision.scope, CooldownScope::Account, "账号级资源耗尽要按账号冷却"); + assert!(decision.switches() && decision.cooldown()); + } + + /// 「套餐池没有有效套餐」同理:账号级额度,不是某一个模型的额度。 + #[test] + fn subscription_quota_cools_account() { + let decision = decide(429, serde_json::json!({ "message": "ZCode 套餐池当前没有有效套餐(1005)" })); + assert_eq!(decision.kind, ErrorKind::Quota); + assert_eq!(decision.scope, CooldownScope::Account); + assert!(decision.cooldown()); + } + + /// 真正的频率限制也按账号冷却(限流消耗的是账号的池子)。 + #[test] + fn rate_limit_cools_account() { + let decision = decide(429, serde_json::json!({ "message": "too many requests, rate limit exceeded" })); + assert_eq!(decision.kind, ErrorKind::RateLimit); + assert_eq!(decision.scope, CooldownScope::Account); + } + + /// 但上游明确声明整家共享冷却时保留 Provider,不缩到账号级。 + #[test] + fn provider_scope_is_preserved_for_rate_limit() { + let decision = decide(429, serde_json::json!({ "message": "rate limit exceeded", "credentialScope": "provider" })); + assert_eq!(decision.scope, CooldownScope::Provider); + } + /// 没有拿到上游响应(status 0)是链路问题:换账号解决不了,保持收尾。 #[test] fn transport_failure_does_not_switch() { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs index 60fdbd8bb..414a36366 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_evidence.rs @@ -49,7 +49,6 @@ pub(super) fn kind(provider: &str, class: &UpstreamErrorClass, status: u16, code } /// 「这份 body 本身就发不出去」的本地业务码 —— 换账号无用,得换一家。 -/// /// 这些码由本机在**发送前**生成(网页工具的字符预算、视觉兼容的前置校验), /// 它们的结论是「提供商 × 模型能力吃不下这份请求」,与具体账号无关。 /// 判据按码前缀匹配:新增同类能力检查时只要沿用同一前缀,就自动获得 @@ -58,8 +57,31 @@ pub(super) fn provider_local(code: &str) -> bool { ["web_tools_", "vision_", "native_payload_"].iter().any(|prefix| code.starts_with(prefix)) } -pub(super) fn scope(body: &Value) -> CooldownScope { - match body.get("credentialScope").or_else(|| body.get("credential_scope")).or_else(|| body.get("scope")).and_then(Value::as_str) { +/// 429/402 里包着的「账号资源已经用完」语义,按**文案**还原成真正的错误类型。 +/// +/// 聚合类上游常把余额/套餐问题套在 429 里返回,例如: +/// `上游返回 429: 上游返回 402: 积分余额不足,请获取积分后再试`、 +/// `ZCode 套餐池当前没有有效套餐(1005)`。只按状态码归成「请求限流」会让 +/// 处置方向错掉 —— 限流是「等一会儿还能用」,余额/套餐是「这个账号暂时 +/// 不可能答」,后者必须按**账号级**冷却并把该账号的其余候选一起排掉, +/// 否则路由组会拿同一个账号换着模型再撞一次(429 连着两条就是这么来的)。 +/// +/// 返回 `None` 表示文案看不出资源问题(例如真正的频率限制),保持原分类。 +pub(super) fn resource_kind(message: &str) -> Option { + let text = message.to_lowercase(); + // 明确的频率限制优先排除:文案里常同时出现「额度」「quota」这类词。 + const RATE: [&str; 5] = ["rate limit", "rate_limit", "too many requests", "限流", "频率限制"]; + if RATE.iter().any(|word| text.contains(word)) { return None; } + const BALANCE: [&str; 8] = ["积分余额不足", "积分不足", "余额不足", "欠费", "insufficient balance", "insufficient credit", + "credit balance", "payment required"]; + if BALANCE.iter().any(|word| text.contains(word)) { return Some(ErrorKind::Balance); } + const QUOTA: [&str; 9] = ["套餐池", "没有有效套餐", "套餐到期", "套餐已到期", "额度已用尽", "quota", "usage limit", + "配额不足", "quota exceeded"]; + if QUOTA.iter().any(|word| text.contains(word)) { return Some(ErrorKind::Quota); } + None +} + +pub(super) fn scope(body: &Value) -> CooldownScope { match body.get("credentialScope").or_else(|| body.get("credential_scope")).or_else(|| body.get("scope")).and_then(Value::as_str) { Some("provider") => CooldownScope::Provider, Some("account" | "credential" | "subscription" | "oauth" | "token") => CooldownScope::Account, _ => CooldownScope::Model, From 2f69e3a0564d20ccf57d45374ba517e2ed27a3f5 Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 19:00:19 +0800 Subject: [PATCH 4/7] =?UTF-8?q?fix(routing):=20=E6=8D=A2=E5=8F=B7=E4=B8=8A?= =?UTF-8?q?=E9=99=90=E6=8A=AC=E5=88=B0=20100=E3=80=81=E5=85=A8=E5=86=B7?= =?UTF-8?q?=E5=8D=B4=E5=85=9C=E5=BA=95=E3=80=81provider=5Flocal=20?= =?UTF-8?q?=E5=86=B7=E5=8D=B4=E9=93=BA=E6=BB=A1=E8=AF=A5=E5=AE=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 换账号上限统一抬到 100:`MAX_SWITCHES`、`RETRY_MAX_SWITCH_COUNT` 与转发层 `MAX_ROUTE_ATTEMPTS` 三处对齐(默认值仍 32,账号多的部署可自行上调),设置页 输入与校验上界、五语文案同步改成 0–100 - 首轮选路:候选全部处于冷却时不再直接回 429 `routing_cooldown`。新增 `last_resort`:只越过「我们自己估算」的冷却(`guessed = true`),挑最早恢复 且本轮未试过的候选先试一次;上游明确声明过的冷却(`guessed = false`)照旧尊重 - 顺延路径:`pick_next_account` 在整条队列被冷却/并发/退避挡空时同样走 `last_resort`,不再以「当前路由没有其他可用对象」直接收尾 - provider_local 的 400(网页工具预算超限、该家不接受的参数):冷却从「只罚撞墙 的那个账号」改为铺到该家全部账号(键仍是模型维度,这家别的模型不受影响), 后续请求不会派同家另一个账号再吃一次一模一样的 400 --- desktop-tauri/i18n/i18n-src/en/settings.json | 2 +- desktop-tauri/i18n/i18n-src/ja/settings.json | 2 +- desktop-tauri/i18n/i18n-src/ko/settings.json | 2 +- .../i18n/i18n-src/pt-BR/settings.json | 2 +- .../i18n/i18n-src/zh-Hant/settings.json | 2 +- .../src/server/config/error_policy_types.rs | 9 ++- .../server/src/server/config/types.rs | 11 +-- .../core/upstream/error_policy_finish.rs | 15 ++++ .../server/src/server/core/upstream/mod.rs | 6 +- .../server/src/server/core/upstream/rotate.rs | 79 ++++++++++++++++++- .../ui-islands/src/islands/settings-model.ts | 4 +- .../ui-shared/routing-errors-panel.tsx | 2 +- .../ui-shared/routing-errors-preview.tsx | 2 +- .../ui-shared/routing-errors-types.ts | 2 +- 14 files changed, 119 insertions(+), 21 deletions(-) diff --git a/desktop-tauri/i18n/i18n-src/en/settings.json b/desktop-tauri/i18n/i18n-src/en/settings.json index 7dc76c228..0d10cd574 100644 --- a/desktop-tauri/i18n/i18n-src/en/settings.json +++ b/desktop-tauri/i18n/i18n-src/en/settings.json @@ -464,7 +464,7 @@ "天数": "Days", "失败 {n} 个": "failed: {n}", "失败 {n} 个:{detail}": "Failed: {n}; {detail}", - "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–10,0 表示失败立即报错。": "How many more accounts to try on failure (N = switch to at most N accounts; the account that made the first attempt does not count). Switching to another account under the same provider and switching to a different provider each count as one; once the limit is reached and the request still fails, an error is returned. Accepts 0–10; 0 means report the error immediately on failure.", + "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–100,0 表示失败立即报错。": "How many more accounts to try on failure (N = switch to at most N accounts; the account that made the first attempt does not count). Switching to another account under the same provider and switching to a different provider each count as one; once the limit is reached and the request still fails, an error is returned. Accepts 0–100; 0 means report the error immediately on failure.", "如果主库已被替换到一半,替换前的数据在 backups/ 目录里仍可整文件拷回。": "", "存储桶(Bucket)": "Bucket", "存储概况已刷新": "Storage overview refreshed", diff --git a/desktop-tauri/i18n/i18n-src/ja/settings.json b/desktop-tauri/i18n/i18n-src/ja/settings.json index 90f4c3e6c..0271dcf1e 100644 --- a/desktop-tauri/i18n/i18n-src/ja/settings.json +++ b/desktop-tauri/i18n/i18n-src/ja/settings.json @@ -464,7 +464,7 @@ "天数": "日数", "失败 {n} 个": "失敗 {n} 件", "失败 {n} 个:{detail}": "失敗 {n} 件:{detail}", - "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–10,0 表示失败立即报错。": "失敗後に最大であと何個のアカウントに切り替えて試すか(N を入力 = 最大 N 個まで切替。初回のアカウントは含みません)。同じプロバイダー配下の別のアカウントへの切替も、別のプロバイダーへの切替もそれぞれ 1 回と数えます。使い切ってもなお失敗する場合はエラーを返します。0–10 を入力でき、0 は失敗時に即座にエラーを返すことを意味します。", + "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–100,0 表示失败立即报错。": "失敗後に最大であと何個のアカウントに切り替えて試すか(N を入力 = 最大 N 個まで切替。初回のアカウントは含みません)。同じプロバイダー配下の別のアカウントへの切替も、別のプロバイダーへの切替もそれぞれ 1 回と数えます。使い切ってもなお失敗する場合はエラーを返します。0–100 を入力でき、0 は失敗時に即座にエラーを返すことを意味します。", "如果主库已被替换到一半,替换前的数据在 backups/ 目录里仍可整文件拷回。": "", "存储桶(Bucket)": "バケット(Bucket)", "存储概况已刷新": "ストレージ概要を更新しました", diff --git a/desktop-tauri/i18n/i18n-src/ko/settings.json b/desktop-tauri/i18n/i18n-src/ko/settings.json index 19ae189b8..e3c837970 100644 --- a/desktop-tauri/i18n/i18n-src/ko/settings.json +++ b/desktop-tauri/i18n/i18n-src/ko/settings.json @@ -464,7 +464,7 @@ "天数": "일수", "失败 {n} 个": "실패 {n}개", "失败 {n} 个:{detail}": "실패 {n}개: {detail}", - "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–10,0 表示失败立即报错。": "실패 후 최대 몇 개의 계정으로 더 시도할지 정합니다(N 입력 = 최대 N개 전환, 최초 계정은 제외). 같은 제공자의 다른 계정으로 바꾸거나 다른 제공자로 바꾸는 것 모두 1회로 계산하며, 횟수를 다 써도 실패하면 오류를 반환합니다. 0–10 입력 가능하며, 0이면 실패 시 즉시 오류를 반환합니다.", + "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–100,0 表示失败立即报错。": "실패 후 최대 몇 개의 계정으로 더 시도할지 정합니다(N 입력 = 최대 N개 전환, 최초 계정은 제외). 같은 제공자의 다른 계정으로 바꾸거나 다른 제공자로 바꾸는 것 모두 1회로 계산하며, 횟수를 다 써도 실패하면 오류를 반환합니다. 0–100 입력 가능하며, 0이면 실패 시 즉시 오류를 반환합니다.", "如果主库已被替换到一半,替换前的数据在 backups/ 目录里仍可整文件拷回。": "", "存储桶(Bucket)": "버킷(Bucket)", "存储概况已刷新": "저장소 개요를 새로 고쳤습니다", diff --git a/desktop-tauri/i18n/i18n-src/pt-BR/settings.json b/desktop-tauri/i18n/i18n-src/pt-BR/settings.json index 0110dc778..a3731a26c 100644 --- a/desktop-tauri/i18n/i18n-src/pt-BR/settings.json +++ b/desktop-tauri/i18n/i18n-src/pt-BR/settings.json @@ -464,7 +464,7 @@ "天数": "Dias", "失败 {n} 个": "Falhas: {n}", "失败 {n} 个:{detail}": "Falhas: {n} — {detail}", - "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–10,0 表示失败立即报错。": "Quantas contas a mais tentar em caso de falha (preencha N = trocar por até N contas; a que fez o primeiro envio não conta). Trocar para outra conta do mesmo provedor ou para outro provedor conta igualmente como uma troca; se o limite se esgotar e a falha persistir, um erro é retornado. Valores de 0–10; 0 significa retornar o erro imediatamente após a falha.", + "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–100,0 表示失败立即报错。": "Quantas contas a mais tentar em caso de falha (preencha N = trocar por até N contas; a que fez o primeiro envio não conta). Trocar para outra conta do mesmo provedor ou para outro provedor conta igualmente como uma troca; se o limite se esgotar e a falha persistir, um erro é retornado. Valores de 0–100; 0 significa retornar o erro imediatamente após a falha.", "如果主库已被替换到一半,替换前的数据在 backups/ 目录里仍可整文件拷回。": "", "存储桶(Bucket)": "Bucket", "存储概况已刷新": "Visão geral do armazenamento atualizada", diff --git a/desktop-tauri/i18n/i18n-src/zh-Hant/settings.json b/desktop-tauri/i18n/i18n-src/zh-Hant/settings.json index 2c72815aa..33d0fe1f7 100644 --- a/desktop-tauri/i18n/i18n-src/zh-Hant/settings.json +++ b/desktop-tauri/i18n/i18n-src/zh-Hant/settings.json @@ -449,7 +449,7 @@ "天数": "", "失败 {n} 个": "", "失败 {n} 个:{detail}": "", - "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–10,0 表示失败立即报错。": "", + "失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–100,0 表示失败立即报错。": "", "如果主库已被替换到一半,替换前的数据在 backups/ 目录里仍可整文件拷回。": "", "存储桶(Bucket)": "", "存储概况已刷新": "", diff --git a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs index 90258ec09..cdb65eb51 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs @@ -54,11 +54,12 @@ pub struct ErrorRule { fn all_providers() -> String { "*".into() } fn enabled() -> bool { true } -/// 换账号上限的上界:与转发层 `MAX_ROUTE_ATTEMPTS`(32)对齐。 +/// 换账号上限的上界:与转发层 `MAX_ROUTE_ATTEMPTS` 对齐。 /// -/// 路由的意义是「试到有候选能答上来为止」,所以默认值直接取上界 —— 队列里 -/// 有几个可用账号就试几个,不再被一个 5 次的默认值提前截断。用户仍可调小。 -pub const MAX_SWITCHES: u64 = 32; +/// 路由的意义是「试到有候选能答上来为止」,所以默认值取一个足够大的数 —— +/// 队列里有几个可用账号就试几个。上限放到 100(而不是 32):账号多的部署 +/// 一次排队就能排到几十个候选,「配了 100 却只换 32 次」等于又给队列截了尾。 +pub const MAX_SWITCHES: u64 = 100; /// 默认规则集的版本号。 /// diff --git a/desktop-tauri/src-tauri/server/src/server/config/types.rs b/desktop-tauri/src-tauri/server/src/server/config/types.rs index 72275508e..67bc74cc9 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/types.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/types.rs @@ -640,8 +640,8 @@ pub const KEY_RETRY_INTERVAL_SECONDS: &str = "retryIntervalSeconds"; /// 有别的候选时,换一个账号/模型的成功率高得多,所以默认直接顺延;需要 /// 原地重发的人可以在设置里调大。 pub const DEFAULT_RETRY_COUNT: i64 = 0; -/// 换账号的次数默认值:32 —— 与转发层 `MAX_ROUTE_ATTEMPTS` 对齐,等于「试完 -/// 队列里的全部候选」,不再用一个 5 次的默认值提前截断。 +/// 换账号的次数默认值:32 —— 覆盖绝大多数部署的候选队列;上限另见 +/// [`RETRY_MAX_SWITCH_COUNT`](100),账号多的人可以自己往上调。 pub const DEFAULT_RETRY_SWITCH_COUNT: i64 = 32; /// 重试间隔默认值:5 秒 pub const DEFAULT_RETRY_INTERVAL_SECONDS: i64 = 5; @@ -672,11 +672,12 @@ pub const RETRY_MAX_NO_RETRY_CODES: usize = 50; /// 该档(不重发 / 不换号),间隔 0 = 立即重发。 pub const RETRY_MIN_COUNT: i64 = 0; pub const RETRY_MAX_COUNT: i64 = 10; -/// 「换账号次数」的上界:与转发层 `MAX_ROUTE_ATTEMPTS`(32)对齐。 +/// 「换账号次数」的上界:与转发层 `MAX_ROUTE_ATTEMPTS`(100)对齐。 /// /// 换账号不是「重发」:一份请求最多试几个候选,本身受循环上限约束,所以 -/// 这里放宽到 32,让默认值等于「试完队列里的全部候选」。 -pub const RETRY_MAX_SWITCH_COUNT: i64 = 32; +/// 这里放到 100,让账号多的部署能把整条队列试完,而不是被一个 10 次的上界 +/// 提前收尾。 +pub const RETRY_MAX_SWITCH_COUNT: i64 = 100; pub const RETRY_MIN_INTERVAL_SECONDS: i64 = 0; pub const RETRY_MAX_INTERVAL_SECONDS: i64 = 300; diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs index 86a257f37..8d4fe9911 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs @@ -24,6 +24,21 @@ pub(super) fn finish_failure(service: &UpstreamService, ctx: &ProviderContext<'_ let decision = error.routing_error.clone().unwrap_or_else(|| error_policy::for_error_with(&ctx.error_policy, &target.provider, &model, error, false)); error_policy::attach(error, &decision); error_policy::note(ctx.telemetry, &decision, None); + // ── 「这家吃不下这份请求」要把整家铺上冷却,而不是只罚撞墙的那个账号 ── + // provider_local 的 400(网页工具预算超限、该家不接受的参数)与账号无关: + // 同一家的另一个账号拿到的是同一份 body、同一个预算,撞的是同一堵墙。只给 + // 失败账号记冷却的话,下一条请求会派同家的下一个账号再吃一次一模一样的 + // 400 —— 用户看到的就是「前面报过错的没有被熔断,后续请求继续往上撞」。 + // 冷却键仍是模型维度(这家别的模型照常可用),但铺到该家的全部账号。 + if decision.skips_provider() && decision.scope == crate::server::config::error_policy::CooldownScope::Model + && ctx.pinned_account.is_none() { + for account in rotate::scoped_accounts(service, providers, keys.request_model(), None, Some(&ctx.routing)) { + if crate::server::core::routing::provider_of(&account) != target.provider { continue; } + let Some(id) = crate::server::core::routing::account_id(&account) else { continue }; + if Some(id) == target.account_id.as_deref() { continue; } + error_policy::remember_cooldown(service, settings, id, &model, error, &decision); + } + } if decision.hard_stop || ctx.telemetry.replay_guard().blocked() || ctx.telemetry.is_cancelled() || ctx.telemetry.response_started() { return false; } if let Some(account) = target.account_id.as_deref() { if let Some(entry) = error_policy::remember_cooldown(service, settings, account, &model, error, &decision) { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs index 99837e3a9..7096c24bd 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/mod.rs @@ -114,7 +114,11 @@ const INFLIGHT_WAIT_MS: u64 = 45_000; /// Node 版靠 `pickNextAccount` 返回 null 收尾 —— 每次轮换都会往 triedIds 里 /// 加一个账号,而账号上限是 20,所以必然收敛。这里额外加一个轮数上限兜底: /// 万一账号数据被手工改出「同一个 id 出现两次」之类的怪状,宁可报错也不要空转。 -pub(super) const MAX_ROUTE_ATTEMPTS: usize = 32; +/// +/// 取 100 而不是 32:`换账号上限` 允许配到 100(见 `config::MAX_SWITCHES`), +/// 硬闸若仍是 32,「配了 100 却只换 32 次」又变回同一个老问题。这里只防回环, +/// 真正的收敛条件仍是「队列里没有没试过的候选」。 +pub(super) const MAX_ROUTE_ATTEMPTS: usize = 100; /// 在途请求的完成信号(去重队列用)。 /// diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs index 805f02e04..4e1f02f93 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs @@ -379,6 +379,25 @@ fn select_target_account_sync( if pinned.is_none() { let earliest = enabled.iter().map(|account| routing::rate_limit_reset_at(account, keys, now)).filter(|at| *at > now).min(); if let Some(at) = earliest { + // ── 兜底:全队列都在冷却,但冷却未必是上游说的 ──────────── + // `routing_cooldown` 这条 429 是本请求「一个候选都没试就失败」的 + // 唯一来源,也是用户看到的「又是直接报错、没换号没换家」。 + // + // 冷却有两类来源:上游明确给了 `resetAt`(`guessed = false`)—— + // 它说「现在别来」,必须尊重,照旧回 429 让客户端等;另一类是我们 + // 自己按退避规则**估算**出来的时长(`guessed = true`),用它把整条 + // 请求判死属于「自己的记账把自己的活掐了」。所以这里只越过估算 + // 冷却:挑最早恢复的那个先试一次。成了就解开,不成也照常记一次 + // 失败(下一次估算会更长)。 + if let Some(account) = last_resort(&enabled, keys, tried_ids, now) { + let id = routing::account_id(&account).unwrap_or_default().to_string(); + if let Some(picked) = service.store.get_session_by_id(&id) { + logging::console_line("[Upstream]", &format!( + "候选全部处于冷却(最早 {} 恢复),按兜底尝试估算冷却中的账号 {id}", errors::format_reset_at_text(at))); + decision["lastResort"] = Value::Bool(true); + return Ok(with_proxy_notice(account, picked.proxy, picked.proxy_error, id, Some(decision))); + } + } return Err(GatewayError::with_status(429, format!("当前路由对象处于冷却期,最早 {} 恢复;未提前重发", errors::format_reset_at_text(at))) .with_code("routing_cooldown").with_retry_after_seconds(at.saturating_sub(now).max(0) as u64 / 1000 + 1)); } @@ -789,7 +808,65 @@ pub(super) fn pick_next_account( sticky_account: None, }; let plan = routing::plan_route(&candidates, keys, &counts, tried_ids, now, &meta); - plan.ordered.into_iter().next().map(|entry| entry.account) + if let Some(entry) = plan.ordered.into_iter().next() { + return Some(entry.account); + } + // 队列被冷却 / 并发 / 退避挡得一个不剩 —— 顺延路径同样不能就此收尾: + // 与首轮选路共用 `last_resort` 的口径(只越过我们自己的估算冷却)。 + last_resort(&candidates, keys, tried_ids, now) +} + +/// 账号此刻的冷却如果**全是我们自己估算**的,返回最晚的恢复时间。 +/// +/// 任何一个上游明确声明过的冷却(`guessed != true`)都直接返回 `None` —— +/// 那是上游在说「现在别来」,兜底不得越过它。 +fn guessed_reset(account: &Value, wire: &str, now: i64) -> Option { + let mut entries: Vec<&Value> = Vec::new(); + if let Some(entry) = account.get("rateLimits").and_then(|limits| limits.get(wire)) { + entries.push(entry); + } + if let Some(entry) = account.get("routingCooldowns").and_then(|limits| limits.get("@account")) { + entries.push(entry); + } + if let Some(entry) = account.get("providerCooldown") { + entries.push(entry); + } + let mut latest = 0i64; + for entry in entries { + let Some(at) = entry["resetAt"].as_i64().filter(|at| *at > now) else { continue }; + if entry["guessed"].as_bool() != Some(true) { + return None; + } + latest = latest.max(at); + } + (latest > 0).then_some(latest) +} + +/// 全队列被冷却挡住时的兜底候选:估算冷却中、最早恢复、本请求还没试过的那个。 +/// +/// 只看估算冷却(见 [`guessed_reset`]),所以「上游明确让我们等」的队列不会 +/// 被这条兜底绕过;反复失败时估算会一次比一次长,不会把同一个账号无休止地 +/// 重试(且每次都会进 `tried_ids`)。 +fn last_resort( + accounts: &[Value], + keys: &routing::CooldownKeys<'_>, + tried_ids: &[String], + now: i64, +) -> Option { + accounts + .iter() + .filter(|account| { + if matches!(account.get("enabled"), Some(Value::Bool(false))) { + return false; + } + let key = crate::server::core::routing_groups::attempt_key(account); + !key.is_empty() && !tried_ids.iter().any(|tried| tried == &key) + }) + .filter_map(|account| { + guessed_reset(account, &keys.for_account(account), now).map(|at| (at, account)) + }) + .min_by_key(|(at, account)| (*at, account.get("priority").and_then(Value::as_i64).unwrap_or(i64::MAX))) + .map(|(_, account)| Value::clone(account)) } /// 选路决策快照(请求日志「路由回放」的数据源)。 diff --git a/desktop-tauri/ui-islands/src/islands/settings-model.ts b/desktop-tauri/ui-islands/src/islands/settings-model.ts index 68a7b68fe..2200ad275 100644 --- a/desktop-tauri/ui-islands/src/islands/settings-model.ts +++ b/desktop-tauri/ui-islands/src/islands/settings-model.ts @@ -641,8 +641,8 @@ export const RETRY_FIELDS: NumberField[] = [ label: t('切换账号重试次数'), unit: t('次'), min: 0, - max: 10, - hint: t('失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–10,0 表示失败立即报错。'), + max: 100, + hint: t('失败后最多再换几个账号试(填 N = 最多换 N 个,首发那个不算)。换到同一家名下另一个账号、或换到另一家,都各算一次;换满仍失败就返回错误。可填 0–100,0 表示失败立即报错。'), }, { key: 'retryIntervalSeconds', diff --git a/desktop-tauri/ui-shared/routing-errors-panel.tsx b/desktop-tauri/ui-shared/routing-errors-panel.tsx index 3258733ca..89f29d10f 100644 --- a/desktop-tauri/ui-shared/routing-errors-panel.tsx +++ b/desktop-tauri/ui-shared/routing-errors-panel.tsx @@ -85,7 +85,7 @@ export function RoutingErrorsPanel({ api, t, active = true, identity = '', reloa }}>{t('恢复建议值')}
patch({ resends })} /> - patch({ switches })} /> + patch({ switches })} /> patch({ intervalSeconds })} /> patch({ waitBudgetSeconds })} />
}> diff --git a/desktop-tauri/ui-shared/routing-errors-preview.tsx b/desktop-tauri/ui-shared/routing-errors-preview.tsx index d577bdb81..1c3c3d0c4 100644 --- a/desktop-tauri/ui-shared/routing-errors-preview.tsx +++ b/desktop-tauri/ui-shared/routing-errors-preview.tsx @@ -24,7 +24,7 @@ export function ErrorPreviewPanel({ settings, providers, t, input, onChange }: { ({ value: item.id, label: item.label }))} onChange={provider => patch({ provider })} /> patch({ model })} /> patch({ pinned })} /> -
patch({ switchesUsed })} /> +
patch({ switchesUsed })} /> patch({ resendsUsed })} /> patch({ strikes })} />
diff --git a/desktop-tauri/ui-shared/routing-errors-types.ts b/desktop-tauri/ui-shared/routing-errors-types.ts index d4f221474..c903609d8 100644 --- a/desktop-tauri/ui-shared/routing-errors-types.ts +++ b/desktop-tauri/ui-shared/routing-errors-types.ts @@ -50,7 +50,7 @@ export function cooldownSummary(rule: ErrorRule, settings: ErrorSettings, t: Err } export function errorPolicyValidation(settings: ErrorSettings, t: ErrorTranslate): string | null { const fields: [number, number, number, string][] = [ - [settings.resends, 0, 10, t('原账号重发')], [settings.switches, 0, 32, t('最多再换账号')], + [settings.resends, 0, 10, t('原账号重发')], [settings.switches, 0, 100, t('最多再换账号')], [settings.intervalSeconds, 0, 300, t('重发间隔')], [settings.waitBudgetSeconds, 0, 300, t('累计等待上限')], [settings.cooldownFloorSeconds, 1, 60, t('最短限流冷却')], [settings.backoffSeconds, 1, 3600, t('首次休息时间')], [settings.backoffCapSeconds, settings.backoffSeconds, 3600, t('连续失败最长休息')], [settings.sinkSeconds, 30, 3600, t('最长保留时间')], From 7f08c2f8c377ee45684d6576f11d786a43f73184 Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 19:14:26 +0800 Subject: [PATCH 5/7] =?UTF-8?q?fix(routing):=20401=20=E7=BB=AD=E6=9C=9F?= =?UTF-8?q?=E5=A4=B1=E8=B4=A5=E5=90=8E=E5=86=B7=E5=8D=B4=E8=B4=A6=E5=8F=B7?= =?UTF-8?q?=EF=BC=8C=E7=AD=89=E5=BE=85=E9=A2=84=E7=AE=97=E8=80=97=E5=B0=BD?= =?UTF-8?q?=E4=B8=8D=E5=86=8D=E7=9B=B4=E6=8E=A5=E6=8A=9B=E9=94=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 登录失效(401)从「不冷却」改成账号级固定 60 秒冷却:续期没救回来的账号 此前既不冷却、也不退出轮转,下一条请求照样轮到它再吃一次 401(日志上就是 每隔几秒一条 CodeArts 401)。现在先退出轮转 60 秒,让队列里别的候选去答 - `RULES_VERSION` 抬到 3,老存盘的 Auth 规则(冷却为空)随升级补上这段冷却 (`upgrade_rules` 只做加法,不动用户选的动作与数值) - 401 续期成功但等待预算已用完时,不再 `return Err` 直接收尾 —— 落下去走统一 分类(换号 / 换家),把剩下的候选试完 --- .../src/server/config/error_policy_types.rs | 23 +++++++++++++++++-- .../src/server/core/upstream/provider_loop.rs | 21 ++++++++++++----- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs index cdb65eb51..687f2fdc8 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs @@ -68,7 +68,11 @@ pub const MAX_SWITCHES: u64 = 100; /// 账号×模型被下一条请求反复挑中)。旧版本存下来的 `routingErrorPolicy` /// 里那套 `return_error + 不冷却` 的组合必须跟着升级,否则用户升级了二进制 /// 却还在跑旧口径 —— 见 `config::error_policy::from_raw` 的升级分支。 -pub const RULES_VERSION: u64 = 2; +/// +/// v3:登录失效(401)也带账号级短冷却。续期没救回来的账号如果既不冷却、 +/// 也不退出轮转,下一条请求还会轮到它再吃一次 401 —— 日志上就是每隔几秒 +/// 一条 401,队列里明明有别的家能答,却一直排在一个已经失效的账号后面。 +pub const RULES_VERSION: u64 = 3; #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(default, rename_all = "camelCase", deny_unknown_fields)] @@ -131,12 +135,16 @@ pub fn default_rules() -> Vec { // 而是为了让「刚在同一个 账号×模型 上失败过的组合」在接下来的几十秒里别 // 再被同一条坏请求抢着试一遍 —— 这就是用户要的「限制器」。 // + // 登录失效(401)原本没有冷却:续期失败后账号既不冷却、也不会被移出队列, + // 于是**下一条请求**仍然轮到它、再吃一次 401、再续期一次 —— 日志里的 + // 「每隔几秒一条 CodeArts 401」就是这么来的。现在给它一段账号级短冷却: + // 续期没救回来的账号先退出轮转 60 秒,让队列里别的候选先去答。 // 要「允许直接把错误返回下游」的码,走「提供商例外」(`overrides`)配置, // 见 `config::error_policy` 对旧「指定错误码」名单的映射。 vec![ rule(RateLimit, "请求限流", &[429], SwitchAccount, Upstream, 60, false, CooldownScope::Auto), rule(Quota, "额度耗尽", &[429], SwitchAccount, Upstream, 600, true, CooldownScope::Auto), - rule(Auth, "登录失效", &[401], RefreshThenSwitch, None, 60, false, CooldownScope::Account), + rule(Auth, "登录失效", &[401], RefreshThenSwitch, Fixed, 60, false, CooldownScope::Account), rule(Balance, "余额不足", &[402], SwitchAccount, Upstream, 600, false, CooldownScope::Account), rule(Transient, "连接或服务异常", &[408, 500, 502, 503, 504], SwitchAccount, Protection, 60, false, CooldownScope::Auto), rule(Unsupported, "能力不支持", &[501], SwitchProvider, Fixed, 60, false, CooldownScope::Model), @@ -219,6 +227,17 @@ mod tests { } } + /// 登录失效也要留冷却:续期没救回来的账号必须先退出轮转,否则下一条请求 + /// 还会轮到它、再吃一次 401(日志上就是每隔几秒一条 401)。 + #[test] + fn auth_rule_cools_the_account_after_401() { + let settings = ErrorSettings::default(); + let rule = settings.rules.iter().find(|rule| rule.kind == ErrorKind::Auth).expect("auth rule"); + assert_eq!(rule.action, ErrorAction::RefreshThenSwitch, "401 仍然先续期"); + assert_ne!(rule.cooldown, CooldownMode::None, "401 续期失败后要冷却账号"); + assert_eq!(rule.scope, CooldownScope::Account, "401 是账号级的失效"); + } + /// 请求有误 / 内容被拒默认也换号(并留短冷却);显式配成 ReturnError 依然合法。 #[test] fn bad_request_and_content_may_switch_or_passthrough() { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs index eeececd05..1b42e1a0f 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs @@ -1495,12 +1495,21 @@ async fn attempt_queue( service, provider_id, target.account_id.as_deref(), ).await { let delay = host_rule.retry_delay(0); - if !budget.take_retry(delay) { return Err(failure.error); } - ctx.telemetry.note_attempt_retry("登录凭证已刷新,在原账号重试一次", Some(401), delay); - super::error_policy_send::sleep_or_cancel(ctx.telemetry, delay).await; - session = fresh; - last_error = Some(failure.error); - continue; + // 等待预算用完只说明「这一刻不适合原地再等」,**不是**整条 + // 请求的结论:以前这里直接 `return Err`,用户看到的就是 + // 「401 之后什么都没换就抛出异常」。现在落下去走统一分类 + // (换号 / 换家),把剩下的候选试完。 + if budget.take_retry(delay) { + ctx.telemetry.note_attempt_retry("登录凭证已刷新,在原账号重试一次", Some(401), delay); + super::error_policy_send::sleep_or_cancel(ctx.telemetry, delay).await; + session = fresh; + last_error = Some(failure.error); + continue; + } + logging::console_line( + "[Upstream]", + &format!("账号 {account_id} 的凭证已刷新,但等待预算已用完,改为换号继续"), + ); } } else { // 刷新**失败**:这是「401 了、也试过续期、但没救回来」的 From 260346ed59f1e1a3790ad4318c4f6a80c6f1b0b6 Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 19:27:31 +0800 Subject: [PATCH 6/7] =?UTF-8?q?fix(routing):=20=E5=A4=B1=E8=B4=A5=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E9=BB=98=E8=AE=A4=E7=86=94=E6=96=AD=E5=B9=B6=E9=80=80?= =?UTF-8?q?=E5=87=BA=E8=BD=AE=E8=BD=AC=EF=BC=8C401=20=E4=B8=80=E6=AC=A1?= =?UTF-8?q?=E5=8D=B3=E7=86=94=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 熔断计数从「只对连接或服务异常」扩到**任何一次失败**:账号连续失败到阈值 就进 `plan_route` 的 `open` 集合,先被摘出轮转一段时间,队列里其他家才有 机会接活;冷却只是「过一会儿再试」,熔断才是「先别用它」 - 登录失效(401)一次就熔断(续期没救回来 = 上游不认这个登录态,再试只会 再 401 一次),阈值取 1;其余错误仍按「连续失败阈值」 - `ErrorSettings` 默认改成 `breaker: true, breaker_failures: 3, breaker_seconds: 120`; 旧存盘(`rules_version < 3`)随升级打开熔断并把阈值收到 3 - 成功恢复改读错误规则那份设置(`error_policy.settings`):失败计数与清零 必须同一把尺子,否则升级后两边不同步,历史失败过的账号每失败一次都被当成 「刚攒够阈值」反复熔断 - `last_resort` 不越过熔断中的账号:熔断的含义就是「先别用它」 --- .../server/src/server/config/error_policy.rs | 12 ++++++++- .../src/server/config/error_policy_types.rs | 13 +++++++++- .../core/upstream/error_policy_finish.rs | 25 ++++++++++++++++--- .../src/server/core/upstream/provider_loop.rs | 12 ++++++--- .../server/src/server/core/upstream/rotate.rs | 13 +++++++--- 5 files changed, 61 insertions(+), 14 deletions(-) diff --git a/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs b/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs index 6fa4c633a..9696eed6a 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/error_policy.rs @@ -98,10 +98,20 @@ fn upgrade_rules(mut settings: ErrorSettings) -> ErrorSettings { rule.action = ErrorAction::SwitchAccount; } } + // 熔断从 v3 起是默认口径(而不是只对「连接或服务异常」生效的那一档): + // 存盘里仍是关闭的,说明当年跑的就是旧口径,跟着升级打开 —— 否则升级了 + // 二进制,失败的账号照样冷却一到期就回到队首。 + let mut breaker_turned_on = false; + if !settings.breaker { + settings.breaker = true; + settings.breaker_failures = settings.breaker_failures.clamp(1, 3); + breaker_turned_on = true; + } let versioned = settings.rules_version; settings.rules_version = RULES_VERSION; crate::server::logging::console_line("[Config]", &format!( - "错误处理规则已从旧版本(v{versioned})升级到 v{RULES_VERSION}:{cooled} 条规则补上冷却,未知错误恢复换号" + "错误处理规则已从旧版本(v{versioned})升级到 v{RULES_VERSION}:{cooled} 条规则补上冷却,未知错误恢复换号{}", + if breaker_turned_on { ",熔断打开(连续失败阈值 3)" } else { "" } )); settings } diff --git a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs index 687f2fdc8..0e20d3b5b 100644 --- a/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs +++ b/desktop-tauri/src-tauri/server/src/server/config/error_policy_types.rs @@ -108,7 +108,9 @@ impl Default for ErrorSettings { rules_version: RULES_VERSION, resends: 0, switches: MAX_SWITCHES, interval_seconds: 5, wait_budget_seconds: 30, cooldown_floor_seconds: 30, backoff: true, backoff_seconds: 30, backoff_cap_seconds: 300, sink: true, sink_seconds: 1800, - breaker: false, breaker_failures: 5, breaker_seconds: 60, breaker_successes: 2, + // 熔断默认打开:失败的账号连续失败到阈值后先退出轮转,让队列里 + // 其他家先接活 —— 冷却只是「过一会儿再试」,熔断才是「先别用它」。 + breaker: true, breaker_failures: 3, breaker_seconds: 120, breaker_successes: 2, rules: default_rules(), overrides: Vec::new(), } } @@ -238,6 +240,15 @@ mod tests { assert_eq!(rule.scope, CooldownScope::Account, "401 是账号级的失效"); } + /// 熔断默认打开、阈值要短:失败的账号先退出轮转,而不是冷却一到期就回到队首。 + #[test] + fn breaker_is_on_by_default() { + let settings = ErrorSettings::default(); + assert!(settings.breaker, "失败的账号要能被熔断摘出轮转"); + assert!(settings.breaker_failures <= 3, "阈值要短,才拦得住反复撞墙的账号"); + assert!(settings.validate().is_ok(), "默认设置必须自身合法"); + } + /// 请求有误 / 内容被拒默认也换号(并留短冷却);显式配成 ReturnError 依然合法。 #[test] fn bad_request_and_content_may_switch_or_passthrough() { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs index 8d4fe9911..151eca878 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs @@ -49,10 +49,27 @@ pub(super) fn finish_failure(service: &UpstreamService, ctx: &ProviderContext<'_ if matches!(decision.kind, ErrorKind::RateLimit | ErrorKind::Quota) && settings.sink && ctx.pinned_account.is_none() { service.route.sink_mark_for(account, logging::now_ms(), settings.sink_seconds); } - if decision.kind == ErrorKind::Transient && decision.rule.cooldown == crate::server::config::error_policy::CooldownMode::Protection { - if settings.backoff { service.route.backoff_mark_for(account, logging::now_ms(), settings.backoff_seconds, settings.backoff_cap_seconds); } - if settings.breaker { - service.route.breaker_failure(account, settings.breaker_failures as i64, settings.breaker_seconds as i64, logging::now_ms()); + if decision.kind == ErrorKind::Transient && decision.rule.cooldown == crate::server::config::error_policy::CooldownMode::Protection + && settings.backoff { + service.route.backoff_mark_for(account, logging::now_ms(), settings.backoff_seconds, settings.backoff_cap_seconds); + } + // ── 失败计数与熔断:任何一次失败都算,不分错误类型 ────────────── + // 以前只在「连接或服务异常」上计数,于是 401 / 400 / 429 的账号永远不会 + // 真正「熔断」——冷却一到期又回到队首,下一次请求继续撞同一堵墙。用户 + // 的口径是「失败的账号就该熔断,然后轮换到其他供应商」:所以这里把每次 + // 失败都记进去,连续失败到阈值就把这个账号从轮转里摘出去一段时间 + // (`plan_route` 的 `open` 集合),别的家才有机会接活。 + if settings.breaker { + // 登录失效一次就熔断:401 是「上游不认这个登录态」,续期又没救回来, + // 再试一次只会再 401 一次 —— 立刻摘出轮转,别等攒够阈值。 + let threshold = if decision.kind == ErrorKind::Auth { 1 } else { settings.breaker_failures as i64 }; + let opened = service.route.breaker_failure( + account, threshold, settings.breaker_seconds as i64, logging::now_ms()); + if opened { + let label = target.account.as_ref().map(super::account_display).unwrap_or_default(); + logging::console_line("[Upstream]", &format!( + "账号「{label}」失败达 {threshold} 次,熔断 {} 秒后才会重新进入轮转", settings.breaker_seconds)); + ctx.telemetry.note_error_policy(json!({ "reason": "账号连续失败,已熔断" })); } } if decision.kind == ErrorKind::Auth && target.account.as_ref().is_some_and(|account| account["needsRelogin"] == true) { diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs index 1b42e1a0f..8cdaa6de8 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/provider_loop.rs @@ -1349,10 +1349,14 @@ async fn attempt_queue( // 时,粘的必须是现在这个能干活的。 let route_settings = ctx.routing.settings; if let Some(account_id) = target.account_id.as_deref() { - if route_settings.breaker_enabled { - service - .route - .breaker_success(account_id, route_settings.breaker_success_threshold); + // 熔断的「开关与阈值」以错误规则那份设置为准:失败计数在 + // `error_policy_finish` 里读的是 `error_policy.settings`, + // 成功恢复若还读 `routing.settings`,升级后两边一旦不同步, + // 失败计数就永远清零不了 —— 于是一个历史失败的账号每失败 + // 一次都被当成「刚攒够阈值」,反复熔断。 + let policy = &ctx.error_policy.settings; + if policy.breaker { + service.route.breaker_success(account_id, policy.breaker_successes as i64); } service.route.backoff_clear(account_id); service.route.sink_clear(account_id); diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs index 4e1f02f93..146c0e85d 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/rotate.rs @@ -389,7 +389,7 @@ fn select_target_account_sync( // 请求判死属于「自己的记账把自己的活掐了」。所以这里只越过估算 // 冷却:挑最早恢复的那个先试一次。成了就解开,不成也照常记一次 // 失败(下一次估算会更长)。 - if let Some(account) = last_resort(&enabled, keys, tried_ids, now) { + if let Some(account) = last_resort(&enabled, keys, tried_ids, now, &snap.open) { let id = routing::account_id(&account).unwrap_or_default().to_string(); if let Some(picked) = service.store.get_session_by_id(&id) { logging::console_line("[Upstream]", &format!( @@ -813,7 +813,7 @@ pub(super) fn pick_next_account( } // 队列被冷却 / 并发 / 退避挡得一个不剩 —— 顺延路径同样不能就此收尾: // 与首轮选路共用 `last_resort` 的口径(只越过我们自己的估算冷却)。 - last_resort(&candidates, keys, tried_ids, now) + last_resort(&candidates, keys, tried_ids, now, &snap.open) } /// 账号此刻的冷却如果**全是我们自己估算**的,返回最晚的恢复时间。 @@ -845,13 +845,15 @@ fn guessed_reset(account: &Value, wire: &str, now: i64) -> Option { /// 全队列被冷却挡住时的兜底候选:估算冷却中、最早恢复、本请求还没试过的那个。 /// /// 只看估算冷却(见 [`guessed_reset`]),所以「上游明确让我们等」的队列不会 -/// 被这条兜底绕过;反复失败时估算会一次比一次长,不会把同一个账号无休止地 -/// 重试(且每次都会进 `tried_ids`)。 +/// 被这条兜底绕过;**熔断中的账号也不碰** —— 熔断的含义就是「先别用它」,兜底 +/// 越过它等于把熔断作废。反复失败时估算会一次比一次长,不会把同一个账号 +/// 无休止地重试(且每次都会进 `tried_ids`)。 fn last_resort( accounts: &[Value], keys: &routing::CooldownKeys<'_>, tried_ids: &[String], now: i64, + open: &std::collections::HashSet, ) -> Option { accounts .iter() @@ -859,6 +861,9 @@ fn last_resort( if matches!(account.get("enabled"), Some(Value::Bool(false))) { return false; } + if routing::account_id(account).is_some_and(|id| open.contains(id)) { + return false; + } let key = crate::server::core::routing_groups::attempt_key(account); !key.is_empty() && !tried_ids.iter().any(|tried| tried == &key) }) From 4a40dfcf4530d4f8808950f7c1331cf9b4211507 Mon Sep 17 00:00:00 2001 From: isee <121077313@qq.com> Date: Sat, 21 Nov 2026 19:35:47 +0800 Subject: [PATCH 7/7] =?UTF-8?q?fix(routing):=20=E4=B8=8A=E6=B8=B8=E6=8A=A5?= =?UTF-8?q?=E4=BD=99=E9=A2=9D=E4=B8=8D=E8=B6=B3=E6=97=B6=E6=8A=8A=E8=B4=A6?= =?UTF-8?q?=E5=8F=B7=E7=A7=AF=E5=88=86=E8=AF=BB=E6=95=B0=E6=8C=89=200=20?= =?UTF-8?q?=E8=90=BD=E5=BA=93=EF=BC=88#643=20#644=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 日志里常见「上游返回 429: 上游返回 402: 积分余额不足,请获取积分后再试」, 但账号页积分显示并不为 0:余额是查上游接口读回来的,字段形状一变就会读出 一个明显偏高的数字。限制器与低余额跳过都按这个读数判断,账号一直显得「有钱」, 每条请求都被派上去再稳定吃一次 402,冷却/熔断之外缺少一层「它确实没钱」的 事实落点。 - usage_records 新增 force_zero_balance:把该账号的 usage 记录(totalLeft / available)覆写成 0 并落库、刷新内存余额事实,余额列随后显示 0,低余额 跳过与限制器立刻能拦住它;不动查询配置,下一次成功查询读到真余额会自己 把 0 顶掉(充值后无需人工改数)。 - error_policy_finish:decision.kind == Balance(402 或 429 包 402「积分余额 不足」经 resource_kind 还原)时调用 force_zero_balance。 --- .../core/upstream/error_policy_finish.rs | 10 ++++++ .../server/src/server/core/usage_records.rs | 31 +++++++++++++++++++ 2 files changed, 41 insertions(+) diff --git a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs index 151eca878..16c21ec04 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/upstream/error_policy_finish.rs @@ -49,6 +49,16 @@ pub(super) fn finish_failure(service: &UpstreamService, ctx: &ProviderContext<'_ if matches!(decision.kind, ErrorKind::RateLimit | ErrorKind::Quota) && settings.sink && ctx.pinned_account.is_none() { service.route.sink_mark_for(account, logging::now_ms(), settings.sink_seconds); } + // ── 上游说「余额/积分不足」→ 把本地余额读数按 0 落一次 ────────── + // 余额是查上游接口读回来的,字段形状一变就会读出一个明显偏高的数字; + // 限制器与低余额跳过都按这个数字判断,于是账号一直显得「有钱」,每条 + // 请求都被派上去再吃一次 402「积分余额不足」。上游的 402 比本地读数更 + // 权威:这里直接落 0,余额列显示 0、低余额规则立刻拦下它,充值后下一 + // 次成功查询会自己把 0 顶掉。 + if decision.kind == ErrorKind::Balance { + crate::server::core::usage_records::force_zero_balance( + account, "上游报余额不足,余额读数已按 0 处理;充值后自动恢复"); + } if decision.kind == ErrorKind::Transient && decision.rule.cooldown == crate::server::config::error_policy::CooldownMode::Protection && settings.backoff { service.route.backoff_mark_for(account, logging::now_ms(), settings.backoff_seconds, settings.backoff_cap_seconds); diff --git a/desktop-tauri/src-tauri/server/src/server/core/usage_records.rs b/desktop-tauri/src-tauri/server/src/server/core/usage_records.rs index 925283aec..70112ed6f 100644 --- a/desktop-tauri/src-tauri/server/src/server/core/usage_records.rs +++ b/desktop-tauri/src-tauri/server/src/server/core/usage_records.rs @@ -188,6 +188,37 @@ pub fn balance_facts() -> HashMap { // ─── 写 ────────────────────────────────────────────────────── +/// 上游明确说「余额/积分不足」时,把账号余额读数**按 0 落一次**。 +/// +/// 为什么不信本地的余额查询:积分靠上游接口读,字段形状一变、或者把「钱包」 +/// 当成积分池,就会读出一个明显偏高的数字。限制器和低余额跳过都按这个数字 +/// 判断,于是账号一直显得「有钱」——每条请求都被派上去,再稳定吃一次 402 +/// `积分余额不足`。上游的 402 是比本地读数**更权威的事实**:这里把它直接落成 +/// 0,余额列跟着显示 0、低余额规则立刻拦下它,等下一次成功查询(或用户充值 +/// 后重新读到真余额)再自动恢复 —— 不需要人去手动改数。 +/// +/// 只覆盖余额(`totalLeft` / `available`),不动 `query_config` 与查询配置, +/// 所以下一次查询照常进行,读数一旦对得上就把 0 顶掉。 +pub fn force_zero_balance(account_id: &str, reason: &str) { + if account_id.is_empty() { return; } + let current = load(account_id); + let unit = current.usage.as_ref().and_then(|usage| usage.get("unit")).and_then(Value::as_str) + .filter(|unit| !unit.is_empty()).unwrap_or("积分"); + let usage = serde_json::json!({ + "unit": unit, + "available": 0.0, + "remaining": 0.0, + "totalLeft": 0.0, + "note": reason, + "forcedZeroAt": logging::now_ms(), + }); + let outcome = UsageOutcome { usage: Some(usage), error: None, code: None, diagnostics: None }; + // 失败不当错误:写不进去只是这次没落上 0,冷却与熔断仍然照常生效。 + if !write_result(account_id, &outcome, ¤t.query_config, "upstream-balance-zero") { + logging::verbose("[Usage]", &format!("账号 {account_id} 的余额按 0 落库失败(上游已报余额不足)")); + } +} + /// 记一笔「尝试开始」(心跳路径在**发请求之前**调用,见模块头)。 /// 只推进时间戳,不动上一次的结果 —— 查询还没回来,旧读数仍是最新事实。 pub fn mark_attempt(account_id: &str) {