242 lines
14 KiB
TypeScript
242 lines
14 KiB
TypeScript
// World extension snapshots, staged services, lifecycle, and last-good recovery.
|
|
import * as durable from '@earendil-works/pi-durable';
|
|
import * as yaml from 'yaml';
|
|
import * as tools from '@earendil-works/pi-durable/tools';
|
|
import * as ai from '@earendil-works/pi-ai';
|
|
import * as modelApi from '@earendil-works/pi-ai/models';
|
|
import * as chord from '@earendil-works/chord';
|
|
import * as contextApi from '@earendil-works/chord/context';
|
|
import * as schema from '@earendil-works/pi-schema';
|
|
import * as events from './events';
|
|
import { conversationChains } from './environment';
|
|
const ops = Deno.core.ops;
|
|
const encoder = new TextEncoder();
|
|
const decoder = new TextDecoder();
|
|
export const ExtensionState = durable.defineDoc({ kind: 'sausage.extensions', version: 1, scope: 'session', checkpointWhen: () => false, initial: () => ({good:null, current:null, error:null}) });
|
|
// The host database retains the last valid bytes independently of editable world files.
|
|
const ExtensionFiles = durable.defineDoc({kind:'sausage.extensions.files',version:1,scope:'session',history:'latest',checkpointWhen:()=>false,initial:()=>({hash:null,files:[]})});
|
|
export const Metadata = durable.defineDoc({ kind: 'sausage.metadata', version: 1, scope: 'conversation', history:'latest', fork:'current', initial:()=>({label:'',title:''}), checkpointWhen:()=>false });
|
|
export type Host = events.EventHost & {
|
|
harness(): durable.Harness; storage: Pick<durable.Storage, 'scanSubmissions'>;
|
|
models(): modelApi.Models; settings(): durable.JsonObject; metadata: typeof Metadata;
|
|
fs: typeof fs; exec: typeof exec; git: typeof git;
|
|
chain(conversationId:number):Promise<string[]>;
|
|
implement<S>(api: {name:string;readonly service?:S}, value:NoInfer<S>):void;
|
|
need<S>(api:{name:string;readonly service?:S}):S; want<S>(api:{name:string;readonly service?:S}):S|undefined;
|
|
resource<T>(key:string, create:()=>T):T;
|
|
onOpen(cb:()=>Promise<void>):void; onClose(cb:()=>Promise<void>):void;
|
|
notice(message:string):void;
|
|
};
|
|
async function file(kind:string, path:string, bytes:number[]=[]):Promise<any> {
|
|
const result = await ops.op_file(kind,path,bytes);
|
|
if (!result.ok) throw Object.assign(new Error(result.error.message), result.error);
|
|
return result.value;
|
|
}
|
|
export const fs = {
|
|
read: async (path:string) => decoder.decode(new Uint8Array(await file('read',path))),
|
|
write: (path:string, text:string) => file('write',path,Array.from(encoder.encode(text))),
|
|
append: (path:string, text:string) => file('append',path,Array.from(encoder.encode(text))),
|
|
list: (path:string) => file('list',path), stat: (path:string) => file('stat',path),
|
|
lstat: (path:string) => file('lstat',path), mkdir: (path:string) => file('mkdir',path),
|
|
remove: (path:string) => file('remove',path),
|
|
rename: (from:string,to:string) => file('rename',from,Array.from(encoder.encode(to))),
|
|
};
|
|
export async function exec(command:string|string[], options:{cwd:string;env?:[string,string][];timeout?:number;conversations?:string[];signal?:AbortSignal;toolCall?:string;task?:string|number;extension?:string;onOutput?:(text:string,stream:string)=>void}) {
|
|
options.signal?.throwIfAborted();
|
|
const tag = options.extension ? {id:options.toolCall ?? `extension:${options.extension}:${options.task ?? crypto.randomUUID()}`,
|
|
pid:0, program:`extension:${options.extension}`, conversations:options.conversations ?? []} : undefined;
|
|
const rid = ops.op_exec(command, options.cwd, options.timeout, undefined, options.env, options.conversations, tag);
|
|
const abort = () => Deno.core.close(rid);
|
|
options.signal?.addEventListener('abort',abort,{once:true});
|
|
const decoders:any = {stdout:new TextDecoder(),stderr:new TextDecoder()};
|
|
const result = {code:0,stdout:'',stderr:''};
|
|
try {
|
|
for (;;) {
|
|
const chunk = await ops.op_exec_next(rid);
|
|
if (!chunk.stream) {
|
|
if (chunk.error) throw new Error(chunk.error);
|
|
result.code = chunk.exitCode;
|
|
for (const stream of ['stdout','stderr']) { const text = decoders[stream].decode(); result[stream] += text; if(text) options.onOutput?.(text,stream); }
|
|
return result;
|
|
}
|
|
const text = decoders[chunk.stream].decode(new Uint8Array(chunk.bytes),{stream:true});
|
|
result[chunk.stream] += text;
|
|
options.onOutput?.(text,chunk.stream);
|
|
}
|
|
} finally { options.signal?.removeEventListener('abort',abort); }
|
|
}
|
|
export async function git(cwd:string,...args:string[]) {
|
|
const result = await exec(['git',...args],{cwd,timeout:30});
|
|
if (result.code) throw new Error(result.stderr || `git exited ${result.code}`);
|
|
return result.stdout.trim();
|
|
}
|
|
export function registerModules() {
|
|
const modules = {'yaml':yaml,'@earendil-works/sausage-events':events,'@earendil-works/pi-schema':schema,'@earendil-works/pi-durable':durable,'@earendil-works/pi-durable/tools':tools,'@earendil-works/pi-ai':ai,'@earendil-works/pi-ai/models':modelApi,'@earendil-works/chord':chord,'@earendil-works/chord/context':contextApi};
|
|
globalThis.__sausage = {modules};
|
|
ops.op_modules(Object.fromEntries(Object.entries(modules).map(([name,value])=>[name,Object.keys(value)])));
|
|
}
|
|
export async function loadExtensions(harness:any, registry:any, storage:any, models:any, context:any, builtins:any[]=[]) {
|
|
registerModules();
|
|
const eventService = events.eventLog(()=>harness);
|
|
const chain = conversationChains(harness,context);
|
|
const repo = '/src/sausage';
|
|
let providers = new Map<string,any>();
|
|
const resources = new Map<string,any>();
|
|
let facets:any[] = [];
|
|
let state = {...(await harness.snapshot(ExtensionState,context) ?? {good:null,current:null,error:null})};
|
|
let settings:any = {};
|
|
let active = true;
|
|
let installed=false;
|
|
let queue = Promise.resolve();
|
|
type Copy = {hash:string;files:[string,number[]][]};
|
|
const publish = async (copy?:Copy) => {
|
|
await harness.commit(async (tx:any)=>{
|
|
Object.assign(await tx.doc(ExtensionState),state);
|
|
if(copy)Object.assign(await tx.doc(ExtensionFiles),copy);
|
|
},context);
|
|
ops.op_extension_status(JSON.stringify(state));
|
|
};
|
|
const readCopy = async ():Promise<Copy> => {
|
|
const files:[string,number[]][]=[];
|
|
const walk = async (relative:string) => {
|
|
const path=`${repo}/extensions${relative}`;
|
|
const entries=await fs.list(path).catch(error=>{if(error.code==='not_found')return [];throw error;});
|
|
entries.sort((a:any,b:any)=>a.name<b.name?-1:a.name>b.name?1:0);
|
|
for(const entry of entries) {
|
|
const name=`${relative}/${entry.name}`;
|
|
if(entry.kind==='directory')await walk(name);
|
|
else files.push([name,await file('read',`${repo}/extensions${name}`)]);
|
|
}
|
|
};
|
|
await walk('');
|
|
const digest=await crypto.subtle.digest('SHA-256',encoder.encode(JSON.stringify(files)));
|
|
const hash=Array.from(new Uint8Array(digest),byte=>byte.toString(16).padStart(2,'0')).join('');
|
|
return {hash,files};
|
|
};
|
|
const mkdir = async (path:string) => {try {await fs.mkdir(path);} catch(error) {if((error as any).code!=='already_exists')throw error;}};
|
|
const snapshot = async ({hash,files}:Copy) => {
|
|
const base=`/run/sausage/extensions/${hash}`;
|
|
for(const path of ['/run/sausage','/run/sausage/extensions',base,`${base}/extensions`])await mkdir(path);
|
|
for(const [name,bytes] of files) {
|
|
let parent=`${base}/extensions`;
|
|
for(const part of name.slice(1).split('/').slice(0,-1)) {parent+=`/${part}`;await mkdir(parent);}
|
|
await file('write',`${base}/extensions${name}`,bytes);
|
|
}
|
|
return base;
|
|
};
|
|
const dispose = async (entries:any[]) => { for(const facet of entries) for(const cleanup of facet.close) { try {await cleanup();} catch(error) {ops.op_output(`Extension cleanup: ${String(error)}\n`);} } };
|
|
const scope = (name:string, staged:Map<string,any>) => {
|
|
let opened=false;
|
|
const notices=new Set<string>();
|
|
const open:any[] = [async()=>{opened=true;}], close:any[] = [async()=>{opened=false;}];
|
|
const handle = (api:any) => new Proxy({}, {get:(_,key)=> {
|
|
if (typeof key === 'symbol' || key === 'then') return undefined;
|
|
const value = providers.get(api.name);
|
|
if(!value) throw new Error(`No provider for ${api.name}`);
|
|
const member = value[key];
|
|
return typeof member === 'function' ? (...args:any[]) => {
|
|
const current = providers.get(api.name);
|
|
if(!current) throw new Error(`No provider for ${api.name}`);
|
|
return Reflect.apply(current[key],current,args);
|
|
} : member;
|
|
}});
|
|
const host:Host = {
|
|
harness:()=>harness, storage, models:()=>models, settings:()=>settings, metadata:Metadata, fs,
|
|
exec:(command,options)=>exec(command,{...options,extension:name}),git,chain,
|
|
implement:(api,value)=>{staged.set(api.name,value);}, need:handle,
|
|
want:(api)=>providers.has(api.name)?handle(api):undefined,
|
|
resource:(key,create)=>{const id=`${name}/${key}`;if(!resources.has(id))resources.set(id,create());return resources.get(id);},
|
|
emit:eventService.emit,eventLog:eventService.eventLog,
|
|
react:(key,tokens,handler,options)=>{
|
|
const reactor=eventService.reactor(`${name}/${key}`,tokens,handler,options);
|
|
open.push(reactor.start);close.push(reactor.stop);
|
|
if(opened)void reactor.start();
|
|
},
|
|
onOpen:cb=>open.push(cb),onClose:cb=>close.push(cb),notice:message=>{
|
|
state.notice=`${name}: ${message}`;ops.op_extension_status(JSON.stringify(state));ops.op_output(`${name}: ${message}\n`);
|
|
if(!notices.has(message)){
|
|
notices.add(message);
|
|
void harness.commit((tx:any)=>eventService.emit(tx,events.NoticeEvent,{extension:name,message,level:'warning'}),context)
|
|
.catch((error:any)=>ops.op_output(`Recording a notice failed: ${String(error)}\n`));
|
|
}
|
|
},
|
|
};
|
|
return {host,open,close};
|
|
};
|
|
const load = async (copy:Copy) => {
|
|
const next:any[] = [];
|
|
const staged = new Map<string,any>();
|
|
try {
|
|
const pathRoot = await snapshot(copy);
|
|
const modules:any[] = [];
|
|
for(const entry of await fs.list(`${pathRoot}/extensions`).catch(error=>{if(error.code==='not_found')return [];throw error;})) {
|
|
if(entry.kind!=='directory')continue;
|
|
const path = `${pathRoot}/extensions/${entry.name}/server.ts`;
|
|
try {await fs.stat(path);} catch(error) {if((error as any).code==='not_found')continue;throw error;}
|
|
const module = await import(/* @vite-ignore */ `file://${path}`);
|
|
modules.push({name:entry.name,module});
|
|
}
|
|
modules.sort((a,b)=>(a.module.order??0)-(b.module.order??0)||a.name.localeCompare(b.name));
|
|
for(const {name,module} of modules) {
|
|
const facet = scope(name,staged); next.push(facet);
|
|
facet.actions = Object.fromEntries((module.userActions ?? []).filter((key:string)=>typeof module.actions?.[key]==='function').map((key:string)=>[key,module.actions[key]]));
|
|
facet.extension = typeof module.default==='function'?await module.default(facet.host):module.default;
|
|
if(!facet.extension || facet.extension.name!==name)throw new Error(`Extension name must match ${name}`);
|
|
}
|
|
// Validate the complete file set before the live registry changes.
|
|
const probe = durable.createRegistry();
|
|
for(const extension of [...builtins,...next.map(f=>f.extension)])probe.install(extension);
|
|
const names=new Set(next.map(f=>f.extension.name));
|
|
for(const old of facets)if(!names.has(old.extension.name))registry.uninstall(old.extension);
|
|
for(const extension of [...builtins,...next.map(f=>f.extension)])registry.install(extension);
|
|
const old = facets; facets=next; providers=staged;
|
|
await dispose(old);
|
|
for(const facet of next)for(const cb of facet.open) {try {await cb();} catch(error) {facet.host.notice(`onOpen: ${String(error)}`);} }
|
|
installed=true;
|
|
state.good=copy.hash;
|
|
return null;
|
|
} catch(error) {await dispose(next); return String(error);}
|
|
};
|
|
const reload = async () => {
|
|
let good:Copy|undefined;
|
|
try {settings=JSON.parse(await fs.read(`${repo}/settings.json`));} catch(error) {if((error as any).code==='not_found')settings={};else {state.notice=`Settings: ${String(error)}`;ops.op_extension_status(JSON.stringify(state));ops.op_output(`${state.notice}\n`);}}
|
|
try {
|
|
const copy=await readCopy();
|
|
if(state.current===copy.hash && installed)return;
|
|
state.current=copy.hash;
|
|
state.error=await load(copy);
|
|
if(!state.error)good=copy;
|
|
} catch(error) {state.error=String(error);}
|
|
if(state.error && !installed) {
|
|
const saved=await harness.snapshot(ExtensionFiles,context);
|
|
if(saved?.hash) {
|
|
const failed=state.error;
|
|
const fallback=await load(saved);
|
|
state.error=fallback?`${failed}\nLast good: ${fallback}`:failed;
|
|
}
|
|
}
|
|
await publish(good);
|
|
};
|
|
const enqueue = () => queue=queue.then(reload);
|
|
const rid = ops.op_watch([`${repo}/extensions`,`${repo}/settings.json`]);
|
|
let timer:ReturnType<typeof setTimeout>|undefined;
|
|
await enqueue();
|
|
const pending = (async()=> {while(active){
|
|
const event=await ops.op_watch_next(rid);
|
|
if(!event || !active)break;
|
|
clearTimeout(timer);
|
|
timer=setTimeout(()=>{timer=undefined;void enqueue();},75);
|
|
}})();
|
|
return {reload:enqueue, state:()=>({...state}), async mainAgent() {
|
|
if (!providers.has('models') || !providers.has('subagents')) return undefined;
|
|
const {main_agent_name:name}=await providers.get('models').discover(repo);
|
|
const conversationId=await providers.get('subagents').mainAgent(name,context);
|
|
return {name,conversationId};
|
|
}, async invoke(name:string,key:string,args:any,signal:AbortSignal,conversationId?:number) {
|
|
const facet = facets.find(f=>f.extension.name===name);
|
|
const action = facet?.actions?.[key];
|
|
if (!action) throw Object.assign(new Error('Unknown user action'),{status:404});
|
|
return action(args,{host:facet.host,signal,conversationId,source:'ui'});
|
|
}, close:async()=>{active=false;clearTimeout(timer);Deno.core.close(rid);await pending;await queue;await dispose(facets);}};
|
|
}
|