979 lines
43 KiB
TypeScript
979 lines
43 KiB
TypeScript
/** Conversation trees, liquid tally abstracts, prompt delivery, and completion notices. */
|
|
import { BACKGROUND_CONTEXT as context, awaitWithContext } from "@earendil-works/chord/context"
|
|
import { arr, bool, int, lit, obj, oneOf, opt, str, union } from "@earendil-works/pi-schema"
|
|
import { AgentDoc, CompactionTask, configure, defineDoc, defineExtension, defineTask, defineTool, GenerationTask, hook, ROOT_CONVERSATION_ID, section } from "@earendil-works/pi-durable"
|
|
import type { Context } from "@earendil-works/chord"
|
|
import type { ConversationId, ConversationRecord, Cursor, EntryId, JsonObject, ModelRef, TaskId, Tx } from "@earendil-works/pi-durable"
|
|
import type { ModelThinkingLevel } from "@earendil-works/pi-ai"
|
|
import { onCommit } from "../messages/compat.ts"
|
|
import { Started as BoardStarted } from "../messages/addresses.ts"
|
|
import { MainAgent, Started, Subagents } from "../subagents/api.ts"
|
|
import { Branch, Branched, LineageApi, Rooted, Roots } from "./api.ts"
|
|
import { TallyDoc, SubmissionLanded } from "./refs.ts"
|
|
import { readMemory, applyMemoryOps, memoryBrief, memoryIndex, memoryText, memorizeTask, Memorized, MEMORY_RULES, MEMORY_SLUG, LINE_CHARS, BODY_CHARS } from "./memory.ts"
|
|
import { catchup, Seen, seedSeen, mainOf, watchSeen } from "./catchup.ts"
|
|
import type { Tree } from "./catchup.ts"
|
|
type ExtensionHost = any
|
|
export type BranchImage = { data: string; mimeType: string }
|
|
|
|
/** A conversation's liquid abstract: its `anchor` entry keeps its place in the tally and shows the latest text. */
|
|
export type OpenAbstract = {
|
|
anchor: number
|
|
at: number
|
|
text: string
|
|
title?: string
|
|
feature?: string
|
|
}
|
|
export type TallyState = {
|
|
/** Each conversation's open abstract, until a landing freezes it. */
|
|
abstracts: Record<string, OpenAbstract>
|
|
/** The entry this root became a tally at; absent while a root is still an ordinary conversation. */
|
|
since?: number
|
|
}
|
|
/**
|
|
* A root's tally bookkeeping. Forks start from the initial value: a branch inherits the tally's entries, but it is no
|
|
* tally itself, and copying the map would let its own abstracts omit entries of another conversation.
|
|
*/
|
|
export const Tally = defineDoc<TallyState>({
|
|
kind: TallyDoc.kind,
|
|
checkpointWhen: () => false,
|
|
version: 1,
|
|
scope: 'conversation',
|
|
history: 'latest',
|
|
fork: 'initial',
|
|
initial: () => ({ abstracts: {} }),
|
|
})
|
|
|
|
/** A first input: the prompt, with the images pasted with it. */
|
|
function promptContent(prompt: string, images: readonly BranchImage[] = []) {
|
|
return images.length
|
|
? [
|
|
{ type: 'text' as const, text: prompt },
|
|
...images.map((image) => ({ type: 'image' as const, ...image })),
|
|
]
|
|
: prompt
|
|
}
|
|
|
|
export async function rootsOf(host: Pick<ExtensionHost, 'harness'>, ctx: Context) {
|
|
return ((await host.harness().snapshot(Roots, ROOT_CONVERSATION_ID, ctx))?.roots ?? [
|
|
ROOT_CONVERSATION_ID,
|
|
]) as ConversationId[]
|
|
}
|
|
|
|
/** Up from `record` through owners (subagents) and fork sources (branches) to the first of `roots`. */
|
|
async function climb(tx: Tx, record: ConversationRecord | undefined, roots: readonly number[]) {
|
|
while (record && !roots.includes(record.id)) {
|
|
const up = record.owner?.conversationId ?? record.parent?.conversationId
|
|
record = up === undefined ? undefined : await tx.conversation(up)
|
|
}
|
|
return record?.id
|
|
}
|
|
|
|
/** The root of `id`'s tree: up through owners (subagents) and fork sources (branches); undefined outside any tree. */
|
|
export async function rootOf(host: Pick<ExtensionHost, 'harness'>, id: number, ctx: Context) {
|
|
const roots = await rootsOf(host, ctx)
|
|
return host
|
|
.harness()
|
|
.commit(async (tx) => climb(tx, await tx.conversation(id as ConversationId), roots), ctx)
|
|
}
|
|
|
|
/** The root a top-level conversation's tally lives in; undefined for roots, owned conversations and outside trees. */
|
|
async function tallyRoot(tx: Tx, id: ConversationId, roots: readonly number[]) {
|
|
const record = await tx.conversation(id)
|
|
if (!record || record.owner || roots.includes(id)) return undefined
|
|
return climb(tx, record, roots)
|
|
}
|
|
|
|
/** The root and intent of a branch that writes abstracts into its tree's tally; undefined where none is written. */
|
|
export async function branchOf(host: Pick<ExtensionHost, 'harness'>, id: number, ctx: Context) {
|
|
const roots = await rootsOf(host, ctx)
|
|
const root = await host.harness().commit((tx) => tallyRoot(tx, id as ConversationId, roots), ctx)
|
|
if (root === undefined) return undefined
|
|
const intent = (await host.harness().snapshot(Branch, id as ConversationId, ctx))?.branch?.intent
|
|
return { root: root as number, ...(intent ? { intent } : {}) }
|
|
}
|
|
|
|
export type AbstractInput = { text: string; at: number; title?: string; feature?: string }
|
|
/**
|
|
* Write `id`'s abstract into its tree's root in the caller's transaction, before that transaction's first table write
|
|
* (it reads conversation records). An abstract is liquid until its conversation lands: while one is open, a carrier
|
|
* entry without model messages replaces the open anchor's text in place, so the tally never reorders. Otherwise it
|
|
* appends a model-visible abstract that becomes the anchor. A no-op where `branchOf` is undefined.
|
|
*/
|
|
export async function abstract(tx: Tx, id: number, input: AbstractInput) {
|
|
const roots = (await tx.doc(Roots, ROOT_CONVERSATION_ID)).roots
|
|
const root = await tallyRoot(tx, id as ConversationId, roots)
|
|
if (root === undefined) return
|
|
const tally = await tx.doc(Tally, root)
|
|
const open = tally.abstracts[String(id)] as OpenAbstract | undefined
|
|
// An anchor a compaction or genesis replaced is out of the context: an edit of it would change nothing.
|
|
const marker = open && (await tx.latestHeadMarker(root))
|
|
// A running compaction may replace the anchor too, after this commit: the update opens a new anchor below its range
|
|
// and omits the old one, which then either went into the summary or stays and is hidden.
|
|
const compacting = !!open && (await liveTask(tx, root, CompactionTask.definition.name))
|
|
const anchor =
|
|
open && !compacting && (!marker || open.anchor >= marker.head) ? open.anchor : undefined
|
|
const { text, at, title, feature } = input
|
|
const state = { at, text, ...(title ? { title } : {}), ...(feature ? { feature } : {}) }
|
|
const data = { conversation: id, ...state }
|
|
const model = [
|
|
{
|
|
role: 'user' as const,
|
|
content: `Abstract of #${id}${title ? ` ${title}` : ''} as of #${id}/${at}:\n${text}`,
|
|
timestamp: Date.now(),
|
|
},
|
|
]
|
|
const entry = await tx.appendEntry(
|
|
root,
|
|
anchor === undefined
|
|
? {
|
|
kind: 'lineage.abstract',
|
|
data,
|
|
model,
|
|
...(compacting
|
|
? { edits: [{ target: open.anchor as EntryId, action: 'omit' as const }] }
|
|
: {}),
|
|
}
|
|
: {
|
|
kind: 'lineage.abstract',
|
|
data: { ...data, anchor },
|
|
edits: [{ target: anchor as EntryId, action: 'replace', messages: model }],
|
|
},
|
|
)
|
|
tally.abstracts[String(id)] = { anchor: anchor ?? entry.id, ...state }
|
|
}
|
|
|
|
/**
|
|
* The last tally entry a fork of `from` at `at` has in its context: the newest root entry it inherits, or the position
|
|
* the newest catch-up it inherits advanced to. Undefined when it inherits neither.
|
|
*/
|
|
async function inheritedTally(tx: Tx, from: ConversationId, at: EntryId, root: ConversationId) {
|
|
let cursor: Cursor | undefined
|
|
do {
|
|
const page = await tx.scanEntries({ conversationId: from, maxEntryId: at }, 100, cursor)
|
|
for (const entry of page.items) {
|
|
if (entry.conversationId === root) return entry.id as number
|
|
const data = entry.data as
|
|
| { source?: string; tally?: number; lineage?: { brief?: { tally?: number | null } } }
|
|
| undefined
|
|
// A memory brief stands in for the tally up to the tip it covers.
|
|
if (entry.kind === 'pi.reset' && data?.lineage?.brief)
|
|
return data.lineage.brief.tally ?? undefined
|
|
if (entry.kind === 'lineage.note' && data?.source === 'main' && data.tally !== undefined)
|
|
return data.tally
|
|
}
|
|
cursor = page.next
|
|
} while (cursor)
|
|
return undefined
|
|
}
|
|
|
|
/** Whether a task of `kind` is live in conversation `id`. */
|
|
async function liveTask(tx: Tx, id: ConversationId, kind: string) {
|
|
for (const status of ['pending', 'running', 'waiting', 'completing'] as const)
|
|
if ((await tx.scanTasks({ conversationId: id, kind, status }, 1)).items.length) return true
|
|
return false
|
|
}
|
|
|
|
/** Include background reporters: their deliveries can start another coordinator turn. */
|
|
async function waitForBranchIdle(host: ExtensionHost, id: ConversationId, ctx: Context): Promise<void> {
|
|
const harness = host.harness()
|
|
const conversation = await harness.conversation(id, ctx)
|
|
if (!conversation) return
|
|
for (;;) {
|
|
await conversation.waitForIdle(ctx)
|
|
const state = await harness.snapshot(Subagents, id, ctx)
|
|
const actions = Object.values(state?.actions ?? {})
|
|
for (const action of actions) await harness.waitForTask(action.reporter, ctx)
|
|
for (const child of Object.values(state?.agents ?? {}))
|
|
await waitForBranchIdle(host, child.conversationId, ctx)
|
|
await conversation.waitForIdle(ctx)
|
|
const current = await harness.snapshot(Subagents, id, ctx)
|
|
if (Object.keys(current?.actions ?? {}).length === actions.length) return
|
|
}
|
|
}
|
|
|
|
function lineage(host: ExtensionHost) {
|
|
/**
|
|
* Delivers a new branch's prompt as its first input; created with the branch, so a crash cannot leave it without
|
|
* one. It is no background task, so waiting for the branch to go idle waits for its prompt and the run it starts.
|
|
*/
|
|
const Prompt = defineTask<
|
|
{ id: ConversationId; prompt: string; images?: BranchImage[] },
|
|
{ phase: 'deliver' },
|
|
null
|
|
>({
|
|
name: 'lineage.prompt',
|
|
version: 1,
|
|
initial: () => ({ phase: 'deliver' }),
|
|
phases: {
|
|
deliver: async (task, runtime, ctx) => {
|
|
const { id, prompt, images } = task.input
|
|
// The stable request id makes a rerun after a crash find the input it already submitted.
|
|
await (await host.harness().conversation(id, ctx))!.submit(
|
|
{ type: 'input', content: promptContent(prompt, images), requestId: 'lineage:prompt' },
|
|
ctx,
|
|
)
|
|
await runtime.commit(
|
|
() => ({ status: 'terminal', outcome: { status: 'completed', result: null } }),
|
|
ctx,
|
|
)
|
|
},
|
|
},
|
|
abort: (_task, runtime, ctx) =>
|
|
runtime.commit(() => ({ status: 'terminal', outcome: { status: 'aborted' } }), ctx),
|
|
})
|
|
const Notify = defineTask<
|
|
{ target: ConversationId; wake: boolean },
|
|
{ phase: 'wait' } | { phase: 'deliver'; title?: string; text: string; answer: number },
|
|
null
|
|
>({
|
|
name: 'lineage.notify',
|
|
version: 1,
|
|
initial: () => ({ phase: 'wait' }),
|
|
phases: {
|
|
wait: async (task, runtime, ctx) => {
|
|
const { target } = task.input
|
|
let said = ''
|
|
let answerId = 0
|
|
for (;;) {
|
|
// Subscribed before reading, so an entry the target writes after the read still ends the wait below.
|
|
const written = Promise.withResolvers<void>()
|
|
// Not a reactor: a short wait inside this task for any entry of the target, a Pi record.
|
|
const unsubscribe = onCommit(host.harness(), (publication) => {
|
|
if (
|
|
publication.changes.some(
|
|
(change) => change.type === 'entry' && change.value.conversationId === target,
|
|
)
|
|
)
|
|
written.resolve()
|
|
})
|
|
try {
|
|
await waitForBranchIdle(host, target, ctx)
|
|
// A fork inherits its source's entries; only an answer written in the target itself counts as its own.
|
|
const view = await runtime.context(target, ctx)
|
|
said = ''
|
|
answerId = 0
|
|
for (let index = view.entries.length - 1; index >= 0 && !said; index--) {
|
|
if (view.entries[index].conversationId !== target) continue
|
|
const answer = view.contributions[index].findLast(
|
|
(message) => message.role === 'assistant' && contentText(message.content).trim(),
|
|
)
|
|
if (answer) {
|
|
said = contentText(answer.content).trim()
|
|
answerId = view.entries[index].id
|
|
}
|
|
}
|
|
const delivered = (await host.harness().snapshot(Notified, runtime.conversationId, ctx))
|
|
?.answers[target]
|
|
// This caller already has that answer: wait for the target to write again, then for its next idle.
|
|
if (delivered === undefined || answerId > delivered) break
|
|
await awaitWithContext(written.promise, ctx)
|
|
} finally {
|
|
unsubscribe()
|
|
}
|
|
}
|
|
const title = (await host.harness().snapshot(host.metadata, target, ctx))?.title
|
|
await runtime.commit(
|
|
() => ({
|
|
status: 'running',
|
|
checkpoint: {
|
|
phase: 'deliver',
|
|
...(title ? { title } : {}),
|
|
text: said
|
|
? said.length > FINISHED_CHARS
|
|
? `${said.slice(0, FINISHED_CHARS)}…`
|
|
: said
|
|
: 'It gave no answer.',
|
|
answer: answerId,
|
|
},
|
|
}),
|
|
ctx,
|
|
)
|
|
},
|
|
deliver: async (task, runtime, ctx) => {
|
|
const { target, wake } = task.input
|
|
const { title, text, answer } = task.state.checkpoint as {
|
|
title?: string
|
|
text: string
|
|
answer: number
|
|
}
|
|
// Keyed by the answer, so a task that replaced one aborted between submit and record finds its delivery.
|
|
const requestId = answer
|
|
? `lineage:notify:${target}:answer:${answer}`
|
|
: `lineage:notify:${target}:${task.id}`
|
|
const caller = runtime.conversationId
|
|
// A delivery found here (follow-up or note: a re-arm may switch `wake`) only lacks its record. This task may
|
|
// have delivered under its own id before answer-keyed ids, and a restart resumes it here.
|
|
const ownId = `lineage:notify:${target}:${task.id}`
|
|
const delivered = await host
|
|
.harness()
|
|
.commit(
|
|
async (tx) =>
|
|
(await tx.submissionByRequest(caller, requestId)) ??
|
|
(await tx.submissionByRequest(caller, ownId)),
|
|
ctx,
|
|
)
|
|
if (!delivered && wake)
|
|
await (await runtime.conversation(caller, ctx))!.submit(
|
|
{
|
|
type: 'input',
|
|
content: finishedText(target, title, text),
|
|
whenBusy: 'followUp',
|
|
requestId,
|
|
entry: {
|
|
kind: 'lineage.finished',
|
|
data: { from: target, text, ...(title ? { title } : {}) },
|
|
},
|
|
},
|
|
ctx,
|
|
)
|
|
else if (!delivered)
|
|
await leaveNote(host, caller, text, {
|
|
from: target,
|
|
source: 'finished',
|
|
...(title ? { data: { title } } : {}),
|
|
requestId,
|
|
})
|
|
await runtime.commit(async (tx) => {
|
|
;(await tx.doc(Notified, caller)).answers[target] = answer
|
|
return { status: 'terminal', outcome: { status: 'completed', result: null } }
|
|
}, ctx)
|
|
},
|
|
},
|
|
abort: (_task, runtime, ctx) =>
|
|
runtime.commit(() => ({ status: 'terminal', outcome: { status: 'aborted' } }), ctx),
|
|
})
|
|
/** Delivers a landing note to a root that is no tally, which may be running a turn; its request id writes it once. */
|
|
const LandingNote = defineTask<
|
|
{ to: ConversationId; text: string; source: NoteSource },
|
|
{ phase: 'deliver' },
|
|
null
|
|
>({
|
|
name: 'lineage.landing-note',
|
|
version: 1,
|
|
initial: () => ({ phase: 'deliver' }),
|
|
phases: {
|
|
deliver: async (task, runtime, ctx) => {
|
|
await leaveNote(host, task.input.to, task.input.text, task.input.source)
|
|
await runtime.commit(
|
|
() => ({ status: 'terminal', outcome: { status: 'completed', result: null } }),
|
|
ctx,
|
|
)
|
|
},
|
|
},
|
|
abort: (_task, runtime, ctx) =>
|
|
runtime.commit(() => ({ status: 'terminal', outcome: { status: 'aborted' } }), ctx),
|
|
})
|
|
return {Notify, Prompt, LandingNote}
|
|
}
|
|
/** The note a landing leaves in its submitter's tree root: what landed, how it diverged, and the card's data. */
|
|
function landingNote(landing: any) {
|
|
const more = landing.commitCount - landing.commits.length
|
|
const text = [
|
|
// The submit-time summary is often stale and the tally carries the abstract: the card shows it, the model does not.
|
|
`Landed ${landing.sha.slice(0, 7)} on ${landing.target} from #${landing.conversationId}/${landing.at} ${landing.title}${landing.feature ? ` (${landing.feature})` : ''}:`,
|
|
`Diverged: ${landing.divergence}`,
|
|
...landing.commits.map((commit) => commit.subject),
|
|
...(more ? [`and ${more} more`] : []),
|
|
`Changed: ${landing.paths.join(', ')}`,
|
|
].join('\n')
|
|
const data = {
|
|
at: landing.at,
|
|
divergence: landing.divergence,
|
|
sha: landing.sha,
|
|
base: landing.base,
|
|
target: landing.target,
|
|
commits: landing.commits,
|
|
paths: landing.paths,
|
|
title: landing.title,
|
|
more,
|
|
lander: landing.lander,
|
|
...(landing.summary ? { summary: landing.summary } : {}),
|
|
...(landing.feature ? { feature: landing.feature } : {}),
|
|
...(landing.pushed ? { pushed: landing.pushed } : {}),
|
|
}
|
|
return { text, data }
|
|
}
|
|
|
|
/**
|
|
* Per target, the answer entry a notify last delivered to this conversation (`0`: it had none), so a re-armed notify
|
|
* waits for a newer one instead of repeating it. A fork keeps it with the deliveries it inherits.
|
|
*/
|
|
export const Notified = defineDoc<{ answers: Record<string, number> }>({
|
|
kind: 'lineage.notified',
|
|
checkpointWhen: () => false,
|
|
version: 1,
|
|
scope: 'conversation',
|
|
history: 'latest',
|
|
fork: 'current',
|
|
initial: () => ({ answers: {} }),
|
|
})
|
|
|
|
/** Created with the notify task: replay finishes cancellations and provenance without creating another task. */
|
|
const NotifyReceipts = defineDoc<{ calls: Record<string, TaskId[]> }>({
|
|
kind: 'lineage.notify-receipts',
|
|
checkpointWhen: () => false,
|
|
version: 1,
|
|
scope: 'task',
|
|
initial: () => ({ calls: {} }),
|
|
})
|
|
|
|
/** How much of the finished conversation's last answer a notify delivers. */
|
|
const FINISHED_CHARS = 1500
|
|
const finishedText = (id: number, title: string | undefined, text: string) =>
|
|
`#${id} finished${title ? ` · ${title}` : ''}\n\nIts last answer:\n${text}`
|
|
|
|
type PromptInput = {
|
|
prompt: string
|
|
/** One short line naming the branch until its summary renames it; without it, the prompt's first 80 characters. */
|
|
title?: string
|
|
images?: BranchImage[]
|
|
isolated?: boolean
|
|
model?: ModelRef
|
|
thinkingLevel?: ModelThinkingLevel
|
|
}
|
|
export type BranchInput = PromptInput & {
|
|
from: number
|
|
repo: string
|
|
at?: number
|
|
deliver?: false
|
|
by?: number
|
|
receipt?: { taskId: TaskId; callId: string }
|
|
}
|
|
|
|
export const BranchReceipts = defineDoc<{ branches: Record<string, number> }>({
|
|
kind: 'lineage.branch-receipts',
|
|
checkpointWhen: () => false,
|
|
version: 1,
|
|
scope: 'task',
|
|
initial: () => ({ branches: {} }),
|
|
})
|
|
|
|
/** `text` on one line of at most `max` characters, whitespace collapsed. */
|
|
function oneLine(text: string, max: number) {
|
|
const flat = text.replace(/\s+/g, ' ').trim()
|
|
return flat.length > max ? `${flat.slice(0, max - 1)}…` : flat
|
|
}
|
|
|
|
/**
|
|
* A branch's intent in one line: its title until its summary names it, and what its tree's tally reads. The title its
|
|
* starter gave (at most 60 characters), else the prompt's first 80.
|
|
*/
|
|
function intentLine(prompt: string, title?: string) {
|
|
const given = title === undefined ? '' : oneLine(title, 60)
|
|
return given || oneLine(prompt, 80)
|
|
}
|
|
|
|
/**
|
|
* The one line the tree root's model reads about a new branch: who started it, where, and its intent. The prompt
|
|
* itself stays the branch's own first input, so later branches never inherit it.
|
|
*/
|
|
function branchedText(branch: number, from: number, at: number, intent: string, caller?: string) {
|
|
return `#${branch} started${caller ? ` by ${caller}` : ''} from #${from}/${at}: ${intent}`
|
|
}
|
|
|
|
/**
|
|
* Fork `from` at `at` (default: its latest entry) into a branch whose first input is the prompt. The tree's root gets
|
|
* the prompt node after the fork point, so the branch reads its prompt once, as its own input.
|
|
*/
|
|
export async function startBranch(host: ExtensionHost, input: BranchInput, ctx: Context) {
|
|
if (!input.prompt.trim()) throw new Error('A branch needs a prompt')
|
|
const from = input.from as ConversationId
|
|
if (!input.repo?.startsWith("/")) throw new Error("repo must be an absolute world path")
|
|
await host.fs.stat(input.repo)
|
|
if (input.isolated) throw new Error("worktree isolation requires the workspaces extension (P3-k)")
|
|
const main = (await mainOf(host, input.from, ctx, undefined, input.repo))?.sha
|
|
const roots = await rootsOf(host, ctx)
|
|
// A first branch needs a stored entry to fork from. An empty root becomes a tally at that entry: its history builds
|
|
// up from branches and landings, while branches start from its memory.
|
|
if (input.at === undefined)
|
|
await host.harness().commit(async (tx) => {
|
|
if (!(await tx.conversation(from))) throw new Error(`No conversation #${from}`)
|
|
if ((await tx.scanEntries({ conversationId: from }, 1)).items.length) return
|
|
const entry = await tx.appendEntry(from, { kind: 'pi.reset', head: 'self' })
|
|
if (roots.includes(from)) (await tx.doc(Tally, from)).since = entry.id
|
|
}, ctx)
|
|
const id = await host.harness().commit((tx) => forkBranch(host, tx, input, main, roots), ctx)
|
|
host.harness().resume()
|
|
return { id }
|
|
}
|
|
|
|
/** `startBranch` inside the caller's transaction, before its first table write; `main` is the checkout's tip. */
|
|
async function forkBranch(
|
|
host: ExtensionHost,
|
|
tx: Tx,
|
|
input: BranchInput,
|
|
main: string | undefined,
|
|
roots: readonly number[],
|
|
) {
|
|
const receipts = input.receipt ? await tx.doc(BranchReceipts, input.receipt.taskId) : undefined
|
|
if (input.receipt && receipts?.branches[input.receipt.callId] !== undefined)
|
|
return receipts.branches[input.receipt.callId] as ConversationId
|
|
const prompt = input.prompt.trim()
|
|
const from = input.from as ConversationId
|
|
const { Prompt } = lineage(host)
|
|
if (!(await tx.conversation(from))) throw new Error(`No conversation #${from}`)
|
|
let at = input.at as EntryId | undefined
|
|
if (at === undefined) {
|
|
at = (await tx.scanEntries({ conversationId: from }, 1)).items[0]?.id
|
|
if (at === undefined) throw new Error(`Conversation #${from} has no entries to branch from`)
|
|
} else if (
|
|
!(await tx.scanEntries({ conversationId: from, maxEntryId: at, minEntryId: at }, 1)).items
|
|
.length
|
|
)
|
|
throw new Error(`Entry ${at} is not in conversation #${from}`)
|
|
// Table reads precede the transaction's first table write, the fork.
|
|
const root = await climb(tx, await tx.conversation(from), roots)
|
|
// A branch of a root with memories starts from their brief instead of the tally; a fork of another branch keeps that
|
|
// branch's context.
|
|
const brief = root !== undefined && root === from ? await memoryBrief(host, tx, root) : undefined
|
|
const tally = brief
|
|
? brief.tip
|
|
: root === undefined
|
|
? undefined
|
|
: await inheritedTally(tx, from, at, root)
|
|
let caller: string | undefined
|
|
if (input.by !== undefined) {
|
|
const index = await tx.doc(MainAgent, ROOT_CONVERSATION_ID)
|
|
const name = Object.keys(index).find((candidate) => index[candidate] === input.by)
|
|
caller = `#${input.by}${name ? ` (@${name})` : ''}`
|
|
}
|
|
const fork = await tx.forkConversation(from, at, { ownership: { kind: 'ownerless' } })
|
|
await configure(tx, fork.id, {cwd: input.repo, ...(input.model ? {model: input.model} : {}), ...(input.thinkingLevel ? {thinkingLevel: input.thinkingLevel} : {})})
|
|
if (input.by !== undefined) (await tx.doc(BoardStarted, fork.id)).by = input.by
|
|
// A branch gets ordinary tools whatever its source ran with.
|
|
const agent = await tx.doc(AgentDoc, fork.id)
|
|
delete agent.tools
|
|
delete agent.modelTools
|
|
|
|
if (main) await seedSeen(tx, fork.id, main, Date.now())
|
|
// The branch knows its tally as far as its context reaches; its own prompt node, appended below, is no news to it.
|
|
if (tally !== undefined) (await tx.doc(Seen, fork.id)).tally = tally
|
|
;(await tx.doc(Branch, fork.id)).branch = {
|
|
from,
|
|
at,
|
|
intent: prompt,
|
|
...(brief ? { brief: true } : {}),
|
|
}
|
|
// The reset starts the branch's model context here, before its prompt: the tally it forked stays in storage.
|
|
if (brief)
|
|
await tx.appendEntry(fork.id, {
|
|
kind: 'pi.reset',
|
|
head: 'self',
|
|
// The tally tip the brief covers: a fork of this branch inherits it, not the root entry this branch forked at.
|
|
data: { lineage: { brief: { tally: brief.tip ?? null } } },
|
|
model: [{ role: 'user', content: brief.text, timestamp: Date.now() }],
|
|
})
|
|
// Until the summary names it, a branch is called by the title its starter gave, else its intent, not by the first
|
|
// message it inherited.
|
|
const intent = intentLine(prompt, input.title)
|
|
const metadata = await tx.doc(host.metadata, fork.id)
|
|
metadata.title = intent
|
|
// The prompt is its first input, placed right after this commit: it sorts as asked for now.
|
|
metadata.inputAt = Date.now()
|
|
await host.emit(tx, Branched, {
|
|
id: fork.id,
|
|
from,
|
|
at,
|
|
intent: prompt,
|
|
...(input.title ? { title: input.title } : {}),
|
|
})
|
|
await host.emit(tx, Started, {
|
|
id: fork.id,
|
|
agent: 'branch',
|
|
kind: 'branch',
|
|
parent: from,
|
|
...(agent.model ? { model: `${agent.model.provider}/${agent.model.modelId}` } : {}),
|
|
...(agent.thinkingLevel ? { thinking: agent.thinkingLevel } : {}),
|
|
})
|
|
const images = input.images?.length ?? 0
|
|
const marker = { branch: fork.id, at, intent, ...(images ? { images } : {}) }
|
|
if (root !== undefined)
|
|
await tx.appendEntry(root, {
|
|
kind: 'lineage.branched',
|
|
data: {
|
|
...marker,
|
|
...(root === from ? {} : { from }),
|
|
},
|
|
model: [
|
|
{
|
|
role: 'user',
|
|
content: branchedText(fork.id, from, at, intent, caller),
|
|
timestamp: Date.now(),
|
|
},
|
|
],
|
|
})
|
|
// A source inside the tree keeps a display-only marker; its model never sees it.
|
|
if (root !== from) await tx.appendEntry(from, { kind: 'lineage.branched', data: marker })
|
|
if (input.deliver !== false)
|
|
await tx.createTask(
|
|
Prompt,
|
|
{ id: fork.id, prompt, ...(images ? { images: input.images } : {}) },
|
|
{ conversationId: fork.id, ownership: { kind: 'conversation' }, background: false },
|
|
)
|
|
if (receipts && input.receipt) receipts.branches[input.receipt.callId] = fork.id
|
|
return fork.id
|
|
}
|
|
|
|
type NoteSource = {
|
|
from?: number
|
|
source?: 'lander' | 'finished'
|
|
data?: JsonObject
|
|
requestId?: string
|
|
}
|
|
|
|
/** A note's entry data and the text its reader's model sees. */
|
|
function note(text: string, source?: NoteSource) {
|
|
const from = source?.from
|
|
const data = {
|
|
...source?.data,
|
|
text,
|
|
...(from === undefined ? {} : { from }),
|
|
...(source?.source ? { source: source.source } : {}),
|
|
}
|
|
const content =
|
|
source?.source === 'lander'
|
|
? `Note from the lander: ${text}`
|
|
: source?.source === 'finished'
|
|
? finishedText(from!, source.data?.title as string | undefined, text)
|
|
: `Note from #${from ?? '?'}: ${text}`
|
|
return { data, content }
|
|
}
|
|
|
|
/**
|
|
* Place landing note `text` from submitter `source.from` in `target` inside `tx`, when `target` is a tally; false when it
|
|
* is none. Placed with the check for an open abstract in one transaction, so none can open in between: the submitter's
|
|
* open abstract freezes into the note, and its anchor leaves the context. A note placed for `requestId` before counts.
|
|
*/
|
|
async function placeLanding(
|
|
tx: Tx,
|
|
target: ConversationId,
|
|
text: string,
|
|
source: NoteSource & { from: number },
|
|
) {
|
|
const tally = await tx.doc(Tally, target)
|
|
if (tally.since === undefined) return false
|
|
const { from, requestId } = source
|
|
// A rerun after a crash finds the landing it wrote; a newer anchor stays open.
|
|
if (requestId && (await tx.submissionByRequest(target, requestId))) return true
|
|
const { data, content } = note(text, source)
|
|
// A tally runs no turns, so the write is placed at once, as an idle conversation's would be.
|
|
const open = tally.abstracts[String(from)] as OpenAbstract | undefined
|
|
const model = { role: 'user' as const, content, timestamp: Date.now() }
|
|
let entry
|
|
if (open) {
|
|
const { anchor, ...state } = open
|
|
entry = await tx.appendEntry(target, {
|
|
kind: 'lineage.note',
|
|
data: { ...data, abstract: state },
|
|
model: [
|
|
{
|
|
...model,
|
|
content: `${content}\n\nState at landing (abstract as of #${from}/${state.at}):\n${state.text}`,
|
|
},
|
|
],
|
|
edits: [{ target: anchor as EntryId, action: 'omit' }],
|
|
})
|
|
delete tally.abstracts[String(from)]
|
|
} else entry = await tx.appendEntry(target, { kind: 'lineage.note', data, model: [model] })
|
|
await tx.createSubmission({
|
|
conversationId: target,
|
|
...(requestId ? { requestId } : {}),
|
|
type: 'write',
|
|
status: 'done',
|
|
entry: entry.id,
|
|
})
|
|
return true
|
|
}
|
|
|
|
/**
|
|
* Leave a note in `to` (a conversation id, or the root of `source.from`'s tree) without waking it. A landing in a tally
|
|
* freezes its submitter's open abstract: the note carries that state, and the anchor leaves the context with it.
|
|
*/
|
|
export async function leaveNote(
|
|
host: Pick<ExtensionHost, 'harness'>,
|
|
to: number | 'root',
|
|
text: string,
|
|
source?: NoteSource,
|
|
) {
|
|
let target = to as ConversationId
|
|
if (to === 'root') {
|
|
if (source?.from === undefined) throw new Error('A note to the root needs its source')
|
|
const root = await rootOf(host, source.from, context)
|
|
if (root === undefined) throw new Error(`#${source.from} has no root`)
|
|
target = root
|
|
}
|
|
const conversation = await host.harness().conversation(target, context)
|
|
if (!conversation) throw new Error(`No conversation #${target}`)
|
|
const from = source?.from
|
|
if (
|
|
source?.source === 'lander' &&
|
|
from !== undefined &&
|
|
(await host
|
|
.harness()
|
|
.commit((tx) => placeLanding(tx, target, text, { ...source, from }), context))
|
|
)
|
|
return target
|
|
const { data, content } = note(text, source)
|
|
const requestId = source?.requestId
|
|
await conversation.submit(
|
|
{
|
|
type: 'write',
|
|
...(requestId ? { requestId } : {}),
|
|
entry: {
|
|
kind: 'lineage.note',
|
|
data,
|
|
model: [{ role: 'user', content, timestamp: Date.now() }],
|
|
},
|
|
},
|
|
context,
|
|
)
|
|
return target
|
|
}
|
|
|
|
const ZOOM =
|
|
'Your tree root retains branch intents, abstracts, and landings. Leave a note for a conversation that your work affects.'
|
|
|
|
/** Whether each conversation is owned (a subagent, a lander), read once: ownership never changes after creation. */
|
|
const owned = new Map<ConversationId, boolean>()
|
|
|
|
/** The lineage section for one conversation; undefined for roots, which never request, and owned conversations. */
|
|
export async function lineageSection(
|
|
host: Pick<ExtensionHost, 'harness'>,
|
|
id: ConversationId,
|
|
ctx: Context,
|
|
) {
|
|
if (!owned.has(id)) {
|
|
const record = await host.harness().commit((tx) => tx.conversation(id), ctx)
|
|
if (!record) return undefined
|
|
owned.set(id, !!record.owner)
|
|
}
|
|
if (owned.get(id)) return undefined
|
|
const root = await rootOf(host, id, ctx)
|
|
if (id === root) return undefined
|
|
const rooted = root === undefined ? '' : `Root: #${root}.\n`
|
|
const state = (await host.harness().snapshot(Branch, id, ctx))?.branch
|
|
if (!state) return `${rooted}${ZOOM}`
|
|
if (state.brief)
|
|
return `Branched from #${state.from} at entry ${state.at} for: ${state.intent}\n${rooted}Your context starts with your project's memory index and recent landings, kept in root #${root}: read a memory in full with \`memory {read: slug}\`. Root #${root} retains the full tally and #<conversation>/<entry> references.\n${ZOOM}`
|
|
return `Branched from #${state.from} at entry ${state.at} for: ${state.intent}\n${rooted}Your context starts with the tally of your tree, kept in its root: one intent line per branch, the latest abstract of each, and every landing, with #<conversation>/<entry> references.\n${ZOOM}`
|
|
}
|
|
|
|
/** The tally `id` catches up on: its tree root's, once that root is a tally. */
|
|
async function treeOf(tx: Tx, id: ConversationId): Promise<Tree | undefined> {
|
|
const root = await tallyRoot(tx, id, (await tx.doc(Roots, ROOT_CONVERSATION_ID)).roots)
|
|
if (root === undefined) return undefined
|
|
const { since, abstracts } = await tx.doc(Tally, root)
|
|
return since === undefined ? undefined : { root, since, abstracts }
|
|
}
|
|
|
|
/** A tally runs no turns: an input placed in one is refused with its transaction. */
|
|
async function refuseTally(tx: Tx, id: ConversationId) {
|
|
if (!(await tx.doc(Roots, ROOT_CONVERSATION_ID)).roots.includes(id)) return
|
|
const tally = await tx.doc(Tally, id)
|
|
if (tally.since !== undefined)
|
|
throw new Error(`#${id} is a tally and runs no turns; ask the main agent to start a branch`)
|
|
}
|
|
|
|
|
|
function contentText(content: any) { return typeof content === "string" ? content : content.filter((p: any) => p.type === "text").map((p: any) => p.text).join("\n") }
|
|
export default function lineageExtension(host: any) {
|
|
host.onOpen(() => host.harness().commit(async (tx: Tx) => {
|
|
await tx.doc(Roots, ROOT_CONVERSATION_ID)
|
|
}, context))
|
|
const memoryRoot = async (caller: number, requested: number | undefined, ctx: Context) => {
|
|
const roots = await rootsOf(host, ctx)
|
|
const root = requested ?? await rootOf(host, caller, ctx) ?? (roots.length === 1 ? roots[0] : undefined)
|
|
if (root === undefined || !roots.includes(root as ConversationId)) throw new Error('Choose a listed root')
|
|
return root
|
|
}
|
|
const remember = async (root: number, ops: unknown[], ctx: Context, now?: number) => {
|
|
await memoryRoot(root, root, ctx)
|
|
return applyMemoryOps(host, root, ops, now)
|
|
}
|
|
host.implement(LineageApi, {
|
|
branch: (input: BranchInput, ctx: Context) => startBranch(host, input, ctx),
|
|
rootOf: (id: number, ctx: Context) => rootOf(host, id, ctx),
|
|
leaveNote: (to: any, text: string, source: any) => leaveNote(host, to, text, source),
|
|
remember,
|
|
memoryBrief: async (id: number, ctx: Context) => {
|
|
const root = await memoryRoot(id, undefined, ctx)
|
|
return host.harness().commit(async (tx: Tx) => {
|
|
const brief = await memoryBrief(host, tx, root)
|
|
return brief ? {root, text: brief.text, memories: Object.keys((await readMemory(host, root)).memories).length} : undefined
|
|
}, ctx)
|
|
},
|
|
memoriesOf: async (id: number, ctx: Context) => {
|
|
const root = await memoryRoot(id, undefined, ctx)
|
|
return {root, memories: (await readMemory(host, root)).memories}
|
|
},
|
|
})
|
|
host.implement({name: 'conversations.trees'}, {
|
|
rootOf: (id: number, ctx: Context) => rootOf(host, id, ctx),
|
|
branchOf: (id: number, ctx: Context) => branchOf(host, id, ctx), abstract,
|
|
})
|
|
host.react('named', [Branched, Rooted], async (tx: Tx, {data}: any) => {
|
|
await host.want({name: 'conversations'})?.generatedTitle(tx, data.id, intentLine(data.intent, data.title))
|
|
})
|
|
watchSeen(host, context)
|
|
const {Notify, Prompt, LandingNote} = lineage(host)
|
|
const Memorize = memorizeTask(host, async (tx, root, from) => (await tx.doc(Tally, root)).abstracts[String(from)]?.text)
|
|
host.react('landing-notes', [SubmissionLanded], async (tx: Tx, {data: landing}: any) => {
|
|
const roots = (await tx.doc(Roots, ROOT_CONVERSATION_ID)).roots
|
|
const root = await climb(tx, await tx.conversation(landing.conversationId), roots)
|
|
if (root === undefined || root === landing.conversationId) return
|
|
const {text, data} = landingNote(landing)
|
|
const source = {from: landing.conversationId, source: 'lander' as const, requestId: `lander-root:${landing.submission}`, data}
|
|
if (!(await placeLanding(tx, root, text, source)))
|
|
await tx.createTask(LandingNote, {to: root, text, source}, {conversationId: root, ownership: {kind: 'conversation'}, background: true})
|
|
})
|
|
host.react('memorize', [SubmissionLanded], async (tx: Tx, {data: landing}: any) => {
|
|
const root = await climb(tx, await tx.conversation(landing.conversationId), (await tx.doc(Roots, ROOT_CONVERSATION_ID)).roots)
|
|
if (root === undefined || root === landing.conversationId) return
|
|
const receipt = await tx.doc(Memorized, root, landing.submission, null)
|
|
if (receipt.task !== undefined) return
|
|
receipt.task = await tx.createTask(Memorize, {root, submission: landing.submission, from: landing.conversationId, at: landing.at, note: landingNote(landing).text}, {conversationId: root, ownership: {kind: 'conversation'}, background: true})
|
|
}, {from: 'now'})
|
|
return defineExtension({
|
|
name: 'lineage', tasks: [Notify, Prompt, LandingNote, Memorize],
|
|
sections: [section('lineage', (input, ctx) => lineageSection(host, input.conversationId, ctx))],
|
|
hooks: [hook(GenerationTask, {beforeInput: async (input, api, ctx) => {
|
|
await refuseTally(api.tx, api.conversationId)
|
|
return catchup(host, input, api, ctx, treeOf)
|
|
}})],
|
|
tools: [
|
|
defineTool({name: 'note', replay: 'safe', description: 'Leave a passive note for a conversation or your root.', parameters: obj({to: union([int({minimum: 1}), lit('root')]), text: str({minLength: 1})}), execute: async ({to, text}, api, ctx) => {
|
|
const target = await leaveNote(host, to, text, {from: api.conversationId, requestId: `lineage:note:${api.taskId}:${api.callId}`})
|
|
return {output: [{type: 'text', text: `Left a note in #${target}.`}]}
|
|
}}),
|
|
defineTool({
|
|
name: 'notify',
|
|
description:
|
|
'Get told when conversation `id` finishes: once it and its subagents are idle (at once if it already is), you receive "#id finished" with its title and last answer. An answer you already received is not sent again: you are told when it next finishes. `wake` (default true) starts your next run with it; `wake: false` leaves it as a note you read with your next input. Calling it again for the same id replaces the earlier request.',
|
|
parameters: obj({
|
|
id: int({ minimum: 1 }),
|
|
wake: opt(bool()),
|
|
}),
|
|
// The transactional receipt reuses the task; cancellations and the request-id note tolerate replay.
|
|
replay: 'safe',
|
|
async execute({ id, wake = true }, api, ctx) {
|
|
const target = id as ConversationId
|
|
const from = api.conversationId
|
|
if (target === from) throw new Error('A conversation cannot notify on itself')
|
|
const replaced = await api.commit(async (tx) => {
|
|
const receipt = await tx.doc(NotifyReceipts, api.taskId)
|
|
if (receipt.calls[api.callId]) return [...receipt.calls[api.callId]]
|
|
if (!(await tx.conversation(target))) throw new Error(`No conversation #${target}`)
|
|
const pending: TaskId[] = []
|
|
let cursor: Cursor | undefined
|
|
do {
|
|
const page = await tx.scanTasks(
|
|
{ conversationId: from, kind: Notify.definition.name },
|
|
100,
|
|
cursor,
|
|
)
|
|
for (const task of page.items)
|
|
if (
|
|
task.state.status !== 'terminal' &&
|
|
(task.input as { target: number }).target === target
|
|
)
|
|
pending.push(task.id)
|
|
cursor = page.next
|
|
} while (cursor)
|
|
await tx.createTask(
|
|
Notify,
|
|
{ target, wake },
|
|
{ conversationId: from, ownership: { kind: 'conversation' }, background: true },
|
|
)
|
|
receipt.calls[api.callId] = pending
|
|
return pending
|
|
}, ctx)
|
|
for (const task of replaced) await host.harness().abortTask(task, ctx)
|
|
host.harness().resume()
|
|
return {
|
|
output: [
|
|
{
|
|
type: 'text',
|
|
text: `You will ${wake ? 'be woken' : 'get a note'} when #${target} finishes.`,
|
|
},
|
|
],
|
|
}
|
|
},
|
|
}),
|
|
defineTool({
|
|
name: 'memory',
|
|
description:
|
|
"Read your project's memory, kept in your tree's root: without `read`, the index (one line per memory); with `read: slug`, that memory in full with its source. Outside a tree it reads the project's only root; `root` reads another root's.",
|
|
parameters: obj({
|
|
read: opt(str({ minLength: 1 })),
|
|
root: opt(int({ minimum: 1 })),
|
|
}),
|
|
async execute({ read, root }, api, ctx) {
|
|
const id = await memoryRoot(api.conversationId, root, ctx)
|
|
const { memories } = await readMemory(host, id)
|
|
if (read !== undefined) {
|
|
const item = Object.hasOwn(memories, read) ? memories[read] : undefined
|
|
if (!item) throw new Error(`#${id} has no memory ${read}`)
|
|
return { output: [{ type: 'text', text: memoryText(read, item) }] }
|
|
}
|
|
const index = memoryIndex(memories)
|
|
return {
|
|
output: [
|
|
{
|
|
type: 'text',
|
|
text: index.length
|
|
? `#${id}'s memory (${index.length}):\n${index.join('\n')}`
|
|
: `#${id} has no memories yet.`,
|
|
},
|
|
],
|
|
}
|
|
},
|
|
}),
|
|
defineTool({
|
|
name: 'remember',
|
|
description: `Change a root's project memory in one atomic batch. Limits, checked before anything is written: ${MEMORY_RULES}. One op over a limit refuses the whole batch, so count a line's characters before you send it and keep detail in the body. Ops are {op:'add', slug, kind: 'decision'|'fact'|'lesson', line, body, source?}, {op:'update', slug, kind?, line?, body?, source?} and {op:'delete', slug}; source is \`#<conversation>/<entry>\`. \`root\` defaults to your tree's root, or outside a tree to the project's only root.`,
|
|
parameters: obj({
|
|
root: opt(int({ minimum: 1 })),
|
|
ops: arr(
|
|
obj({
|
|
op: oneOf(['add', 'update', 'delete']),
|
|
slug: str({
|
|
pattern: MEMORY_SLUG.source,
|
|
maxLength: 48,
|
|
description: 'kebab-case, at most 48 characters',
|
|
}),
|
|
kind: opt(oneOf(['decision', 'fact', 'lesson'])),
|
|
line: opt(
|
|
str({
|
|
maxLength: LINE_CHARS,
|
|
description: `The index line: one line, at most ${LINE_CHARS} characters`,
|
|
}),
|
|
),
|
|
body: opt(
|
|
str({
|
|
maxLength: BODY_CHARS,
|
|
description: `The full memory, at most ${BODY_CHARS} characters; split longer text across memories`,
|
|
}),
|
|
),
|
|
source: opt(str({ description: '#<conversation>/<entry>' })),
|
|
}),
|
|
{ minItems: 1 },
|
|
),
|
|
}),
|
|
// Not replay-safe (the default): an interrupted call is never rerun, so a batch commits at most once.
|
|
async execute({ root, ops }, api, ctx) {
|
|
// Every conversation sees the tool, but only the primary agent can write memory.
|
|
const mainAgent = await host.harness().snapshot(MainAgent, ROOT_CONVERSATION_ID, ctx)
|
|
if (mainAgent?.['$main-agent'] !== api.conversationId)
|
|
throw new Error(
|
|
'Only the main agent writes project memory; tell the main agent what to remember (message, reply: false).',
|
|
)
|
|
const id = await memoryRoot(api.conversationId, root, ctx)
|
|
const summary = await applyMemoryOps(host, id, ops, Date.now(), `tool:${api.taskId}:${api.callId}`)
|
|
return { output: [{ type: 'text', text: summary }] }
|
|
},
|
|
}),
|
|
],
|
|
})
|
|
}
|