forked from BasedHardware/omi
-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathfailures.ts
More file actions
297 lines (271 loc) · 10.8 KB
/
Copy pathfailures.ts
File metadata and controls
297 lines (271 loc) · 10.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
import { AcpError, isAcpProviderAuthFailure } from "../adapters/acp.js";
import type { ProductionAdapterId } from "../adapters/interface.js";
export type RuntimeFailureSource = "adapter_process" | "adapter_execution" | "runtime";
/** Closed cross-surface taxonomy; detailed legacy codes remain diagnostic-only. */
export const RUNTIME_FAILURE_CODES = [
"authentication",
"quota_exceeded",
"invalid_request",
"timeout",
"transport_interruption",
"adapter_unavailable",
"adapter_incompatible",
"bridge_start_failed",
"provider_setup_needed",
"malformed_or_oversized_tool_result",
"cancelled",
"stale_owner",
"policy_denied",
"unknown",
] as const;
export type RuntimeFailureCode = (typeof RUNTIME_FAILURE_CODES)[number];
export function isRuntimeFailureCode(value: unknown): value is RuntimeFailureCode {
return typeof value === "string" && (RUNTIME_FAILURE_CODES as readonly string[]).includes(value);
}
export interface RuntimeFailure {
/** Detailed local code retained for logs and backward-compatible UI copy. */
code: string;
/** Bounded code carried across adapters and checked by the shared fixture. */
failureCode?: RuntimeFailureCode;
userMessage: string;
technicalMessage?: string;
source?: RuntimeFailureSource;
adapterId?: string;
provider?: string;
retryable?: boolean;
recoveryAction?: "worker_recycled";
recoveryOutcome?: "recovered" | "stop_failed" | "binding_stale_failed";
retryDisposition?: "next_send";
/** Qualification-only gateway attribution; null is explicit unknown. */
jitCostStatus?: "estimated" | "unknown";
jitEstimatedCostUsd?: number | null;
}
export class AdapterRuntimeError extends Error {
readonly failure: RuntimeFailure;
constructor(failure: RuntimeFailure) {
super(failure.userMessage);
this.name = "AdapterRuntimeError";
this.failure = normalizeRuntimeFailure(failure);
}
}
export function messageFrom(error: unknown): string {
if (error instanceof AdapterRuntimeError) {
return error.failure.userMessage;
}
return error instanceof Error ? error.message : String(error);
}
const CONTEXT_SNAPSHOT_PROJECTION_MISMATCH = "context_snapshot_projection_mismatch";
export function unexpectedQueryErrorDiagnostic(error: unknown): string | null {
const message = messageFrom(error);
if (message === CONTEXT_SNAPSHOT_PROJECTION_MISMATCH) return null;
return `Unhandled query error: ${String(error)}`;
}
export const WORKER_RECYCLED_NEXT_SEND_MESSAGE =
"The local agent reset its session after an error. Send your message again.";
const PROVIDER_BILLING_HTTP_402 = /\bhttp[\s/]*402\b/i;
const PROVIDER_BILLING_402_STATUS = /\b(?:402\s+status|status(?:\s+code)?\s*[:=]?\s*402)\b/i;
/** A 402 is a provider billing rejection, not a poisoned worker. Recycle can
* still clean the process; retrying the same turn cannot clear Payment Required.
*/
export function isProviderBillingFailure(text: string): boolean {
const lower = text.toLowerCase();
return (
lower.includes("payment required")
|| PROVIDER_BILLING_HTTP_402.test(lower)
|| PROVIDER_BILLING_402_STATUS.test(lower)
|| lower.includes("credit balance is too low")
);
}
export function applyProviderBillingClassification(failure: RuntimeFailure): RuntimeFailure {
const haystack = `${failure.technicalMessage ?? ""}\n${failure.userMessage}`;
if (!isProviderBillingFailure(haystack)) return failure;
return {
...failure,
failureCode: "quota_exceeded",
retryable: false,
};
}
/** Recycle metadata stays on every pi-mono execution throw so the next send
* can mint a fresh worker. Non-retryable causes keep their own copy — wrapping
* them as "send again" hid HTTP 402 behind the unclassified transcript marker.
*/
export function attachWorkerRecycle(
failure: RuntimeFailure,
outcome: { stopSucceeded: boolean; bindingInvalidationSucceeded: boolean },
): RuntimeFailure {
const recoveryOutcome: NonNullable<RuntimeFailure["recoveryOutcome"]> = !outcome.stopSucceeded
? "stop_failed"
: outcome.bindingInvalidationSucceeded
? "recovered"
: "binding_stale_failed";
const recycled: RuntimeFailure = {
...failure,
recoveryAction: "worker_recycled",
recoveryOutcome,
};
if (failure.retryable === false) return recycled;
return {
...recycled,
userMessage: WORKER_RECYCLED_NEXT_SEND_MESSAGE,
retryable: true,
retryDisposition: "next_send",
};
}
export function failureFromError(
error: unknown,
fallback: Omit<RuntimeFailure, "userMessage"> & { userMessage?: string }
): RuntimeFailure {
if (error instanceof AdapterRuntimeError) {
return applyProviderBillingClassification(error.failure);
}
if (error instanceof AcpError && isAcpProviderAuthFailure(error)) {
return normalizeRuntimeFailure({
code: "provider_auth_required",
failureCode: "authentication",
userMessage: "Claude sign-in is required to continue this chat.",
technicalMessage: messageFrom(error),
source: fallback.source ?? "adapter_execution",
adapterId: fallback.adapterId ?? "acp",
retryable: false,
});
}
return applyProviderBillingClassification(normalizeRuntimeFailure({
...fallback,
userMessage: fallback.userMessage ?? messageFrom(error),
technicalMessage: fallback.technicalMessage ?? messageFrom(error),
}));
}
export function normalizeRuntimeFailure(failure: RuntimeFailure): RuntimeFailure {
const userMessage = compactWhitespace(failure.userMessage) || "Agent run failed";
const technicalMessage = compactWhitespace(failure.technicalMessage ?? "");
return {
...failure,
failureCode: failure.failureCode ?? normalizeRuntimeFailureCode(failure.code),
userMessage,
technicalMessage: technicalMessage || undefined,
};
}
/** Maps every adapter/runtime detail code onto the shared bounded vocabulary. */
export function normalizeRuntimeFailureCode(value: string): RuntimeFailureCode {
const code = value.toLowerCase();
if (code.includes("auth")) return "authentication";
if (code.includes("quota") || code.includes("rate_limit") || code.includes("429")) return "quota_exceeded";
if (code.includes("invalid") || code.includes("malformed") || code.includes("400")) return "invalid_request";
if (code.includes("timeout") || code.includes("timed_out")) return "timeout";
if (code.includes("transport") || code.includes("process") || code.includes("connection")) return "transport_interruption";
if (code.includes("not_registered") || code.includes("unavailable")) return "adapter_unavailable";
if (code.includes("config") || code.includes("incompatible") || code.includes("stale_binding")) return "adapter_incompatible";
if (code.includes("bridge_start")) return "bridge_start_failed";
if (code.includes("provider_setup")) return "provider_setup_needed";
if (code.includes("oversized") || code.includes("tool_result")) return "malformed_or_oversized_tool_result";
if (code.includes("cancel")) return "cancelled";
if (code.includes("stale_owner") || code.includes("owner_changed")) return "stale_owner";
if (code.includes("policy") || code.includes("permission") || code.includes("authority")) return "policy_denied";
return "unknown";
}
export function sanitizeProcessDiagnostic(text: string): string {
return text
.replace(/Bearer\s+[A-Za-z0-9._~+/=-]+/gi, "Bearer [redacted]")
.replace(/sk-[A-Za-z0-9_-]{12,}/g, "sk-[redacted]")
.replace(/(api[_-]?key["'\s:=]+)[A-Za-z0-9._~+/=-]+/gi, "$1[redacted]")
.replace(/\s+/g, " ")
.trim()
.slice(0, 1_000);
}
export function failureFromProcessExit(input: {
adapterId: ProductionAdapterId;
exitCode: number | null;
recentStderr: string;
}): RuntimeFailure {
const diagnostic = sanitizeProcessDiagnostic(input.recentStderr);
const technicalMessage = diagnostic || `${input.adapterId} ACP process exited with code ${input.exitCode}`;
const classified = classifyAdapterProcessFailure(input.adapterId, diagnostic);
if (classified) {
return normalizeRuntimeFailure({
source: "adapter_process",
adapterId: input.adapterId,
technicalMessage,
...classified,
});
}
const provider = providerFromDiagnostic(diagnostic);
const label = adapterFailureLabel(input.adapterId, provider);
return normalizeRuntimeFailure({
code: "adapter_process_exited",
source: "adapter_process",
adapterId: input.adapterId,
provider,
retryable: true,
userMessage: `${label} failed: ${technicalMessage}`,
technicalMessage,
});
}
function classifyAdapterProcessFailure(
adapterId: ProductionAdapterId,
diagnostic: string
): (Pick<RuntimeFailure, "code" | "userMessage"> & Partial<RuntimeFailure>) | undefined {
if (adapterId === "openclaw" && isOpenClawInvalidConfig(diagnostic)) {
return {
code: "adapter_config_invalid",
retryable: false,
userMessage:
"OpenClaw needs a config migration. Run `openclaw doctor --fix`, then retry. Inspect with `openclaw config validate`.",
};
}
return undefined;
}
// Adapter stderr is unstructured; this is the sanctioned adapter-boundary sniffing site
// (Phase 6 item 7 exception). Prefer typed RuntimeFailure codes when the adapter can classify.
function isOpenClawInvalidConfig(diagnostic: string): boolean {
const lower = diagnostic.toLowerCase();
return (
lower.includes("openclaw config is invalid") ||
lower.includes("invalid config at") ||
lower.includes("legacy config keys detected") ||
lower.includes("openclaw doctor --fix") ||
lower.includes("openclaw config validate") ||
lower.includes("channels.telegram.streaming: invalid config")
);
}
export function failureFromProcessError(input: {
adapterId: ProductionAdapterId;
message: string;
}): RuntimeFailure {
const diagnostic = sanitizeProcessDiagnostic(input.message);
const provider = providerFromDiagnostic(diagnostic);
const label = adapterFailureLabel(input.adapterId, provider);
return normalizeRuntimeFailure({
code: "adapter_process_error",
source: "adapter_process",
adapterId: input.adapterId,
provider,
retryable: true,
userMessage: `${label} failed: ${diagnostic || `${input.adapterId} ACP process error`}`,
technicalMessage: diagnostic,
});
}
function providerFromDiagnostic(diagnostic: string): string | undefined {
const lower = diagnostic.toLowerCase();
if (lower.includes("openai")) return "openai";
if (lower.includes("anthropic") || lower.includes("claude")) return "anthropic";
if (lower.includes("gemini") || lower.includes("google")) return "google";
return undefined;
}
function adapterFailureLabel(adapterId: ProductionAdapterId, provider?: string): string {
switch (adapterId) {
case "openclaw":
return "OpenClaw";
case "hermes":
return "Hermes";
case "pi-mono":
return "pi-mono";
case "acp":
if (provider === "openai") {
return "OpenAI";
}
return "ACP";
}
}
function compactWhitespace(text: string): string {
return text.replace(/\s+/g, " ").trim();
}