128 lines
7.2 KiB
TypeScript
128 lines
7.2 KiB
TypeScript
// Ultimate's input-placement cases, scheduler hold, and the headless channel contract.
|
|
import { BACKGROUND_CONTEXT as context } from '@earendil-works/chord/context';
|
|
import { createModels } from '@earendil-works/pi-ai/models';
|
|
import { fauxProvider, fauxAssistantMessage, fauxToolCall } from '@earendil-works/pi-ai/providers/faux';
|
|
import { Harness, createRegistry, defineDoc, defineExtension, defineTask, GenerationTask, hook, InboxDoc } from '@earendil-works/pi-durable';
|
|
import { openStorage } from './sqlite';
|
|
import { serve } from './input';
|
|
const assert = (value: any, message: string) => { if (!value) throw new Error(message); };
|
|
const equal = (a: any, b: any) => assert(JSON.stringify(a) === JSON.stringify(b), `Expected ${JSON.stringify(b)}, got ${JSON.stringify(a)}`);
|
|
const until = async (condition: () => any) => {
|
|
const deadline = Date.now() + 10000;
|
|
while (!await condition()) {
|
|
if (Date.now() > deadline) throw new Error('Condition deadline');
|
|
await new Promise(resolve => setTimeout(resolve, 0));
|
|
}
|
|
};
|
|
const Placed = defineDoc<{ count: number }>({ kind: 'input-test.placed', version: 1,
|
|
checkpointWhen: () => false, scope: 'conversation', history: 'latest', fork: 'initial', initial: () => ({ count: 0 }) });
|
|
const gate = Promise.withResolvers<void>();
|
|
let held = false, reject = false, calls = 0, second = false;
|
|
const Notes = defineExtension({ name: 'input-test', hooks: [hook(GenerationTask, {
|
|
beforeInput: async (_input, { tx, conversationId }) => {
|
|
calls++;
|
|
const state = await tx.doc(Placed, conversationId);
|
|
state.count++;
|
|
if (reject) throw new Error('reject placement');
|
|
return { entries: [{ kind: 'input-test.note', data: { count: state.count },
|
|
model: [{ role: 'user', content: `before ${state.count}`, timestamp: 1 }] }] };
|
|
},
|
|
})] });
|
|
const Park = defineTask<null, { phase: 'first' | 'second' }, null>({ name: 'hold-test', version: 1,
|
|
initial: () => ({ phase: 'first' }), phases: {
|
|
first: async (_task, runtime, ctx) => {
|
|
held = true;
|
|
await gate.promise;
|
|
await runtime.commit(() => ({ status: 'pending', checkpoint: { phase: 'second' } }), ctx);
|
|
},
|
|
second: async (_task, runtime, ctx) => {
|
|
second = true;
|
|
await runtime.commit(() => ({ status: 'terminal', outcome: { status: 'completed', result: null } }), ctx);
|
|
},
|
|
}, abort: (_task, runtime, ctx) => runtime.commit(() => ({ status: 'terminal', outcome: { status: 'aborted' } }), ctx),
|
|
});
|
|
const models = createModels();
|
|
const faux = fauxProvider({ tokensPerSecond: 100000 });
|
|
models.setProvider(faux.provider);
|
|
const registry = createRegistry();
|
|
registry.install(Notes);
|
|
registry.install(defineExtension({ name: 'hold-test', tasks: [Park] }));
|
|
const storage = await openStorage(globalThis.storagePath);
|
|
const harness = await Harness.open(storage, { models, registry, settings: { followUpMode: 'all' } }, context);
|
|
const scenario = globalThis.foundationCase;
|
|
const root = await harness.root(context, { agent: { model: { provider: 'faux', modelId: 'faux-1' },
|
|
extensions: scenario === 'placement' ? [Notes] : [] } });
|
|
try {
|
|
if (scenario === 'placement' || scenario === 'followups') {
|
|
faux.setResponses([async () => {
|
|
held = true;
|
|
await gate.promise;
|
|
return fauxAssistantMessage([fauxToolCall('unknown', {})], { stopReason: 'toolUse' });
|
|
}, fauxAssistantMessage('first answer'), fauxAssistantMessage('follow-ups answer'),
|
|
fauxAssistantMessage('extra'), fauxAssistantMessage('extra')]);
|
|
const first = await root.submit({ type: 'input', content: 'first', requestId: 'one' }, context);
|
|
await until(() => held);
|
|
const queued = [];
|
|
const followups = scenario === 'placement' ? ['next'] : ['one', 'two', 'three'];
|
|
for (const content of followups) queued.push(await root.submit({ type: 'input', content, whenBusy: 'followUp' }, context));
|
|
const steer = await root.submit({ type: 'input', content: 'steer', whenBusy: 'steer' }, context);
|
|
if (scenario === 'placement') equal(calls, 1);
|
|
gate.resolve();
|
|
await Promise.all([first, steer, ...queued].map(submission => submission.wait(context)));
|
|
const view = await root.context(context);
|
|
if (scenario === 'placement') {
|
|
equal(view.messages.filter(m => m.role === 'user').map(m => m.content), ['before 1', 'first', 'steer', 'before 2', 'next']);
|
|
const notes = view.entries.filter(e => e.kind === 'input-test.note');
|
|
equal(notes.length, 2);
|
|
assert(notes.every(e => e.edits === undefined), 'Hook notes must be placed, not edits');
|
|
equal(calls, 2);
|
|
assert(view.entries.filter(e => e.kind === 'pi.user').every(e => e.input === true), 'Inputs need their marker');
|
|
await root.submit({ type: 'input', content: 'first', requestId: 'one' }, context);
|
|
equal(calls, 2);
|
|
reject = true;
|
|
let rejected = false;
|
|
try { await root.submit({ type: 'input', content: 'rejected' }, context); }
|
|
catch (error) { rejected = String(error).includes('reject placement'); }
|
|
assert(rejected, 'Placement must reject');
|
|
equal((await harness.snapshot(Placed, root.id, context))?.count, 2);
|
|
equal((await root.context(context)).entries.filter(e => e.kind === 'input-test.note').length, 2);
|
|
} else {
|
|
equal(view.messages.flatMap(m => m.role === 'user' ? [m.content] : m.role === 'assistant'
|
|
? m.content.flatMap(p => p.type === 'text' ? [`> ${p.text}`] : []) : []),
|
|
['first', 'steer', '> first answer', 'one', 'two', 'three', '> follow-ups answer']);
|
|
equal(await harness.snapshot(InboxDoc, root.id, context), { items: [] });
|
|
}
|
|
} else if (scenario === 'thinking') {
|
|
faux.setResponses([fauxAssistantMessage('unset'), fauxAssistantMessage('high')]);
|
|
await (await root.submit({ type: 'input', content: 'one' }, context)).wait(context);
|
|
await root.configure({ thinkingLevel: 'high' }, context);
|
|
await (await root.submit({ type: 'input', content: 'two' }, context)).wait(context);
|
|
equal((await root.context(context)).messages.flatMap(m => m.role === 'assistant' ? [m.thinkingLevel] : []), ['off', 'high']);
|
|
} else if (scenario === 'hold') {
|
|
const task = await root.commit(tx => tx.createTask(Park, null, { ownership: { kind: 'conversation' } }), context);
|
|
harness.resume();
|
|
await until(() => held);
|
|
harness.hold();
|
|
gate.resolve();
|
|
await until(async () => (await harness.getTask(task, context))?.state.status === 'pending');
|
|
equal((await harness.getTask(task, context))?.state.checkpoint.phase, 'second');
|
|
harness.resume();
|
|
equal((await harness.inspect(context)).scheduling, 'paused');
|
|
assert(!second, 'Held scheduler started another phase');
|
|
harness.release();
|
|
await until(() => second);
|
|
await harness.waitForIdle(context);
|
|
equal((await harness.getTask(task, context))?.state.status, 'terminal');
|
|
} else if (scenario === 'headless') {
|
|
faux.setResponses([fauxAssistantMessage('channel answer')]);
|
|
await serve(harness, context);
|
|
const submissions = await storage.scanSubmissions({ conversationId: root.id }, 100, undefined, context);
|
|
equal(submissions.items.length, 1);
|
|
equal(submissions.items[0].requestId, 'channel-request');
|
|
assert(JSON.stringify(await root.context(context)).includes('channel answer'), 'Channel answer missing');
|
|
} else throw new Error('Unknown foundation case');
|
|
} finally {
|
|
gate.resolve();
|
|
await harness.close(context);
|
|
}
|