/** World-loaded watchdog retries, message reports, and recovery with a virtual clock. */
import {BACKGROUND_CONTEXT as context} from '@earendil-works/chord/context';
import {createModels} from '@earendil-works/pi-ai/models';
import {fauxProvider, fauxAssistantMessage} from '@earendil-works/pi-ai/providers/faux';
import {Harness, createRegistry, configure, LiveDoc} from '@earendil-works/pi-durable';
import {loadExtensions, fs, Metadata} from './extensions';
import {openStorage} from './sqlite';
const assert = (value: any, message: string) => {if (!value) throw new Error(message)};
const until = async (test: () => any) => {const deadline = Date.now() + 15000; while (!await test()) {if (Date.now() > deadline) throw new Error('Watchdog deadline'); await new Promise(r => setTimeout(r, 0))}};
let now = globalThis.watchdogMode === 'resume' ? 1_060_000 : 1_000_000;
const timers = new Set<any>();
const models = createModels(), faux = fauxProvider({tokensPerSecond: 100000}); models.setProvider(faux.provider);
const registry = createRegistry(), storage = await openStorage(globalThis.storagePath);
const harness = await Harness.open(storage, {models, registry, now: () => now, setTimer: (callback, delay) => {const timer = {callback, at: now + delay}; timers.add(timer); return () => timers.delete(timer)}, settings: {retry: {maxRetries: 0}, get extensions() {return registry.snapshot().installed()}}}, context);
const root = await harness.root(context, {agent: {model: {provider: 'faux', modelId: 'faux-1'}, cwd: '/src/sausage'}});
const route = (transcript: any): any => {
  faux.appendResponses([route]);
  const last = transcript.messages.findLast((m: any) => m.role === 'user');
  const text = typeof last?.content === 'string' ? last.content : '';
  if (text === 'fail' || text.startsWith('Your previous run')) return fauxAssistantMessage('', {stopReason: 'error', errorMessage: 'Our servers are currently overloaded. Please try again later.'});
  if (text === 'permanent') return fauxAssistantMessage('', {stopReason: 'error', errorMessage: 'invalid model configuration'});
  return fauxAssistantMessage('ok');
};
faux.setResponses([route]);
const loader = await loadExtensions(harness, registry, storage, models, context);
assert(!loader.state().error, `Extension load: ${loader.state().error}`);
const sha = await fs.read('/src/sausage/.git/sausage-last-good');
const base = `file:///run/sausage/extensions/${sha}/extensions`;
const {MainAgent} = await import(`${base}/subagents/api.ts`);
const {Messages} = await import(`${base}/messages/api.ts`);
const {diagnose} = await import(`${base}/watchdog/server.ts`);
const service = globalThis.watchdogService;
const tasks = async () => (await harness.commit((tx: any) => tx.scanTasks({kind: 'watchdog.failure'}, 100), context)).items;
const records = async () => (await harness.commit((tx: any) => tx.scanConversations({}, 100), context)).items;
const create = async (title: string) => harness.commit(async (tx: any) => {
  const record = await tx.createConversation({ownership: {kind: 'ownerless'}});
  await configure(tx, record.id, {model: {provider: 'faux', modelId: 'faux-1'}, cwd: '/src/sausage'});
  Object.assign(await tx.doc(Metadata, record.id), {title});
  return record.id;
}, context);
let primary = (await harness.snapshot(MainAgent, root.id, context))['$main-agent'];
if (!primary) {primary = await create('@mainAgent'); await harness.commit(async (tx: any) => {(await tx.doc(MainAgent, root.id))['$main-agent'] = primary}, context)}
const submit = async (id: number, text: string, requestId?: string) => (await (await harness.conversation(id, context))!.submit({type: 'input', content: text, requestId}, context)).wait(context);
const requests = async (id: number) => (await storage.scanSubmissions({conversationId: id}, 100, undefined, context)).items;
const retries = async (id: number) => (await requests(id)).filter((r: any) => r.requestId?.startsWith('watchdog-retry:'));
const reports = async (id: number) => (await harness.snapshot(Messages, root.id, context)).messages.filter((m: any) => m.from === id && m.text.startsWith('[run failed]'));
const advance = (ms: number) => {now += ms; for (const timer of [...timers]) if (timer.at <= now) {timers.delete(timer); timer.callback()} harness.resume()};
harness.resume();
try {
  if (globalThis.watchdogMode === 'resume') {
    const ids = await records();
    const branch = (await Promise.all(ids.map(async (r: any) => ({id: r.id, title: (await harness.snapshot(Metadata, r.id, context))?.title})))).find(r => r.title === 'Crash gap')!.id;
    await until(async () => (await reports(branch)).length === 1);
    await harness.waitForIdle(context);
    assert((await retries(branch)).length === 1, 'Restart duplicated the retry');
    assert((await reports(branch)).length === 1, 'Restart duplicated the report');
    await until(async () => (await service.failures(0)).filter((f: any) => f.conversation === branch).length === 1);
    assert((await service.failures(0)).filter((f: any) => f.conversation === branch).length === 1, 'Restart duplicated failure history');
  } else {
    const branch = await create(globalThis.watchdogMode === 'kill' ? 'Crash gap' : 'Provider blip');
    await submit(branch, 'fail');
    await until(async () => (await tasks()).some((t: any) => t.state.status === 'sleeping'));
    const sleeper = (await tasks()).find((t: any) => t.state.status === 'sleeping');
    assert(sleeper.state.until === 1_060_000, 'Retry deadline must be durable');
    assert((await retries(branch)).length === 0 && (await reports(branch)).length === 0, 'Premature retry or report');
    if (globalThis.watchdogMode === 'kill') {await fs.write('/watchdog-ready', 'yes'); await new Promise(() => {})}
    advance(59_999); await harness.waitForIdle(context);
    assert((await retries(branch)).length === 0, 'Retry happened before its deadline');
    advance(1);
    await until(async () => (await reports(branch)).length === 1);
    await harness.waitForIdle(context);
    assert((await retries(branch)).length === 1, 'Expected one retry');
    assert((await reports(branch))[0].to === primary, 'The message must reach @mainAgent');
    assert((await reports(branch))[0].text.includes('After one automatic retry'), 'Report must describe its retry');
    await until(async () => (await service.failures(0)).length === 1);
    const history = await service.failures(0);
    assert(history.length === 1 && history[0].retried && history[0].title === 'Provider blip', 'Failure history');
    advance(120_000); await harness.waitForIdle(context);
    assert((await retries(branch)).length === 1 && (await reports(branch)).length === 1, 'A failed retry must not retry again');
    const permanent = await create('Bad config'); await submit(permanent, 'permanent');
    await until(async () => (await reports(permanent)).length === 1);
    assert((await retries(permanent)).length === 0, 'Permanent errors must report without retry');
    const excluded = await create('Board owns failure');
    await globalThis.messageService.send(primary, {to: `#${excluded}`, text: 'healthy', reply: false}, 'setup', context);
    await harness.waitForIdle(context);
    await harness.commit(async (tx: any) => {(await tx.doc(Messages, root.id)).messages.push({id: 999, from: primary, to: excluded, text: 'fail', at: now, reply: true, state: 'delivered'})}, context);
    await submit(excluded, 'fail', 'board:999'); await harness.waitForIdle(context);
    assert(!(await tasks()).some((t: any) => t.input.conversation === excluded), 'Reply-expected messages handle their own failures');
    for (const [text, temporary] of [['Overloaded (529)', true], ['429 Too Many Requests', false], ['invalid model configuration', false], ['Incomplete Claude Code upstream response (status unknown, capture incomplete, relay failure Error)', true]] as const) {
      assert(diagnose(text, fauxAssistantMessage('', {stopReason: 'error', errorMessage: text})).temporary === temporary, `Diagnosis: ${text}`);
    }
    assert(!(await harness.snapshot(LiveDoc, branch, context)).run, 'Failed generation must stop');
    const {idleNote} = await import(`${base}/watchdog/idle.ts`);
    const {Branch} = await import(`${base}/lineage/api.ts`);
    await harness.commit(async (tx: any) => {
      (await tx.doc(Branch, branch)).branch = {from: root.id, at: 0, intent: 'Idle work'};
      Object.assign(await tx.doc(Metadata, branch), {inputAt: 1, answerAt: 1});
    }, context);
    const idle = () => harness.commit((tx: any) => idleNote(tx, globalThis.testHost, primary, 45 * 60_000), context);
    assert((await idle()).entries[0].data.ids.includes(branch), 'Idle branch reminder');
    assert(await idle() === undefined, 'Unchanged idle IDs must not repeat');
    await harness.commit(async (tx: any) => {delete (await tx.doc(Branch, branch)).branch}, context);
    assert((await idle()).entries[0].model[0].content === 'No branches are idle.', 'Idle reminder must clear');
  }
} finally {await loader.close(); await harness.close(context)}