106 lines
8.8 KiB
TypeScript
106 lines
8.8 KiB
TypeScript
/** World-loaded watchdog retries, message reports, and recovery with a virtual clock. */
|
|
import {BACKGROUND_CONTEXT as context} from '@earendil-works/chord/context';
|
|
import {createModels} from '@earendil-works/pi-ai/models';
|
|
import {fauxProvider, fauxAssistantMessage} from '@earendil-works/pi-ai/providers/faux';
|
|
import {Harness, createRegistry, configure, LiveDoc} from '@earendil-works/pi-durable';
|
|
import {loadExtensions, fs, Metadata} from './extensions';
|
|
import {openStorage} from './sqlite';
|
|
const assert = (value: any, message: string) => {if (!value) throw new Error(message)};
|
|
const until = async (test: () => any) => {const deadline = Date.now() + 15000; while (!await test()) {if (Date.now() > deadline) throw new Error('Watchdog deadline'); await new Promise(r => setTimeout(r, 0))}};
|
|
let now = globalThis.watchdogMode === 'resume' ? 1_060_000 : 1_000_000;
|
|
const timers = new Set<any>();
|
|
const models = createModels(), faux = fauxProvider({tokensPerSecond: 100000}); models.setProvider(faux.provider);
|
|
const registry = createRegistry(), storage = await openStorage(globalThis.storagePath);
|
|
const harness = await Harness.open(storage, {models, registry, now: () => now, setTimer: (callback, delay) => {const timer = {callback, at: now + delay}; timers.add(timer); return () => timers.delete(timer)}, settings: {retry: {maxRetries: 0}, get extensions() {return registry.snapshot().installed()}}}, context);
|
|
const root = await harness.root(context, {agent: {model: {provider: 'faux', modelId: 'faux-1'}, cwd: '/src/sausage'}});
|
|
const route = (transcript: any): any => {
|
|
faux.appendResponses([route]);
|
|
const last = transcript.messages.findLast((m: any) => m.role === 'user');
|
|
const text = typeof last?.content === 'string' ? last.content : '';
|
|
if (text === 'fail' || text.startsWith('Your previous run')) return fauxAssistantMessage('', {stopReason: 'error', errorMessage: 'Our servers are currently overloaded. Please try again later.'});
|
|
if (text === 'permanent') return fauxAssistantMessage('', {stopReason: 'error', errorMessage: 'invalid model configuration'});
|
|
return fauxAssistantMessage('ok');
|
|
};
|
|
faux.setResponses([route]);
|
|
const loader = await loadExtensions(harness, registry, storage, models, context);
|
|
assert(!loader.state().error, `Extension load: ${loader.state().error}`);
|
|
const sha = await fs.read('/src/sausage/.git/sausage-last-good');
|
|
const base = `file:///run/sausage/extensions/${sha}/extensions`;
|
|
const {MainAgent} = await import(`${base}/subagents/api.ts`);
|
|
const {Messages} = await import(`${base}/messages/api.ts`);
|
|
const {diagnose} = await import(`${base}/watchdog/server.ts`);
|
|
const service = globalThis.watchdogService;
|
|
const tasks = async () => (await harness.commit((tx: any) => tx.scanTasks({kind: 'watchdog.failure'}, 100), context)).items;
|
|
const records = async () => (await harness.commit((tx: any) => tx.scanConversations({}, 100), context)).items;
|
|
const create = async (title: string) => harness.commit(async (tx: any) => {
|
|
const record = await tx.createConversation({ownership: {kind: 'ownerless'}});
|
|
await configure(tx, record.id, {model: {provider: 'faux', modelId: 'faux-1'}, cwd: '/src/sausage'});
|
|
Object.assign(await tx.doc(Metadata, record.id), {title});
|
|
return record.id;
|
|
}, context);
|
|
let primary = (await harness.snapshot(MainAgent, root.id, context))['$main-agent'];
|
|
if (!primary) {primary = await create('@mainAgent'); await harness.commit(async (tx: any) => {(await tx.doc(MainAgent, root.id))['$main-agent'] = primary}, context)}
|
|
const submit = async (id: number, text: string, requestId?: string) => (await (await harness.conversation(id, context))!.submit({type: 'input', content: text, requestId}, context)).wait(context);
|
|
const requests = async (id: number) => (await storage.scanSubmissions({conversationId: id}, 100, undefined, context)).items;
|
|
const retries = async (id: number) => (await requests(id)).filter((r: any) => r.requestId?.startsWith('watchdog-retry:'));
|
|
const reports = async (id: number) => (await harness.snapshot(Messages, root.id, context)).messages.filter((m: any) => m.from === id && m.text.startsWith('[run failed]'));
|
|
const advance = (ms: number) => {now += ms; for (const timer of [...timers]) if (timer.at <= now) {timers.delete(timer); timer.callback()} harness.resume()};
|
|
harness.resume();
|
|
try {
|
|
if (globalThis.watchdogMode === 'resume') {
|
|
const ids = await records();
|
|
const branch = (await Promise.all(ids.map(async (r: any) => ({id: r.id, title: (await harness.snapshot(Metadata, r.id, context))?.title})))).find(r => r.title === 'Crash gap')!.id;
|
|
await until(async () => (await reports(branch)).length === 1);
|
|
await harness.waitForIdle(context);
|
|
assert((await retries(branch)).length === 1, 'Restart duplicated the retry');
|
|
assert((await reports(branch)).length === 1, 'Restart duplicated the report');
|
|
await until(async () => (await service.failures(0)).filter((f: any) => f.conversation === branch).length === 1);
|
|
assert((await service.failures(0)).filter((f: any) => f.conversation === branch).length === 1, 'Restart duplicated failure history');
|
|
} else {
|
|
const branch = await create(globalThis.watchdogMode === 'kill' ? 'Crash gap' : 'Provider blip');
|
|
await submit(branch, 'fail');
|
|
await until(async () => (await tasks()).some((t: any) => t.state.status === 'sleeping'));
|
|
const sleeper = (await tasks()).find((t: any) => t.state.status === 'sleeping');
|
|
assert(sleeper.state.until === 1_060_000, 'Retry deadline must be durable');
|
|
assert((await retries(branch)).length === 0 && (await reports(branch)).length === 0, 'Premature retry or report');
|
|
if (globalThis.watchdogMode === 'kill') {await fs.write('/watchdog-ready', 'yes'); await new Promise(() => {})}
|
|
advance(59_999); await harness.waitForIdle(context);
|
|
assert((await retries(branch)).length === 0, 'Retry happened before its deadline');
|
|
advance(1);
|
|
await until(async () => (await reports(branch)).length === 1);
|
|
await harness.waitForIdle(context);
|
|
assert((await retries(branch)).length === 1, 'Expected one retry');
|
|
assert((await reports(branch))[0].to === primary, 'The message must reach @mainAgent');
|
|
assert((await reports(branch))[0].text.includes('After one automatic retry'), 'Report must describe its retry');
|
|
await until(async () => (await service.failures(0)).length === 1);
|
|
const history = await service.failures(0);
|
|
assert(history.length === 1 && history[0].retried && history[0].title === 'Provider blip', 'Failure history');
|
|
advance(120_000); await harness.waitForIdle(context);
|
|
assert((await retries(branch)).length === 1 && (await reports(branch)).length === 1, 'A failed retry must not retry again');
|
|
const permanent = await create('Bad config'); await submit(permanent, 'permanent');
|
|
await until(async () => (await reports(permanent)).length === 1);
|
|
assert((await retries(permanent)).length === 0, 'Permanent errors must report without retry');
|
|
const excluded = await create('Board owns failure');
|
|
await globalThis.messageService.send(primary, {to: `#${excluded}`, text: 'healthy', reply: false}, 'setup', context);
|
|
await harness.waitForIdle(context);
|
|
await harness.commit(async (tx: any) => {(await tx.doc(Messages, root.id)).messages.push({id: 999, from: primary, to: excluded, text: 'fail', at: now, reply: true, state: 'delivered'})}, context);
|
|
await submit(excluded, 'fail', 'board:999'); await harness.waitForIdle(context);
|
|
assert(!(await tasks()).some((t: any) => t.input.conversation === excluded), 'Reply-expected messages handle their own failures');
|
|
for (const [text, temporary] of [['Overloaded (529)', true], ['429 Too Many Requests', false], ['invalid model configuration', false], ['Incomplete Claude Code upstream response (status unknown, capture incomplete, relay failure Error)', true]] as const) {
|
|
assert(diagnose(text, fauxAssistantMessage('', {stopReason: 'error', errorMessage: text})).temporary === temporary, `Diagnosis: ${text}`);
|
|
}
|
|
assert(!(await harness.snapshot(LiveDoc, branch, context)).run, 'Failed generation must stop');
|
|
const {idleNote} = await import(`${base}/watchdog/idle.ts`);
|
|
const {Branch} = await import(`${base}/lineage/api.ts`);
|
|
await harness.commit(async (tx: any) => {
|
|
(await tx.doc(Branch, branch)).branch = {from: root.id, at: 0, intent: 'Idle work'};
|
|
Object.assign(await tx.doc(Metadata, branch), {inputAt: 1, answerAt: 1});
|
|
}, context);
|
|
const idle = () => harness.commit((tx: any) => idleNote(tx, globalThis.testHost, primary, 45 * 60_000), context);
|
|
assert((await idle()).entries[0].data.ids.includes(branch), 'Idle branch reminder');
|
|
assert(await idle() === undefined, 'Unchanged idle IDs must not repeat');
|
|
await harness.commit(async (tx: any) => {delete (await tx.doc(Branch, branch)).branch}, context);
|
|
assert((await idle()).entries[0].model[0].content === 'No branches are idle.', 'Idle reminder must clear');
|
|
}
|
|
} finally {await loader.close(); await harness.close(context)}
|