Skip to content

Commit a09fda7

Browse files
Aitomatesclaude
andcommitted
feat(integrations): add Obsidian orchestration, lesson extraction, and execution orchestrator
New integrations: - ObsidianOrchestrator: push session history and prompts to Obsidian vault - obsidian-sync.ts: historical migration script for EventStore → Obsidian - ExecutionOrchestrator: watcher-to-Obsidian pipeline with self-healing Enhancements: - Config: RefinerConfig now supports atlassian and obsidian config blocks - ConfigManager: getAtlassianConfig() and getObsidianConfig() static helpers - BackgroundService: watcher integration with execution orchestrator - Dashboard: additional operator controls and Obsidian sync status - Timeline: expanded session tracking and lesson correlation - LessonExtractor: derive and persist prompt improvement lessons from history Tests: background-service, config, lessons, obsidian-orchestrator, watcher-index, self-healing Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent fde69e0 commit a09fda7

21 files changed

Lines changed: 2033 additions & 26 deletions

universal-refiner/package-lock.json

Lines changed: 457 additions & 5 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

universal-refiner/package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -46,18 +46,19 @@
4646
"@hono/node-server": "^1.19.13",
4747
"@modelcontextprotocol/sdk": "^1.29.0",
4848
"better-sqlite3": "^12.8.0",
49-
"chokidar": "^5.0.0",
50-
"flexsearch": "^0.7.43",
5149
"hono": "^4.12.25",
5250
"typescript": "^5.9.3",
5351
"zod": "^4.3.6"
5452
},
5553
"devDependencies": {
5654
"@emnapi/core": "^1.11.1",
5755
"@emnapi/runtime": "^1.11.1",
56+
"@lancedb/lancedb": "^0.30.0",
5857
"@types/better-sqlite3": "^7.6.13",
5958
"@types/node": "^22.19.17",
6059
"@vitest/coverage-v8": "4.1.4",
60+
"chokidar": "^5.0.0",
61+
"flexsearch": "^0.8.212",
6162
"ts-node": "^10.9.2",
6263
"vitest": "^4.1.4"
6364
},
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import { EventStore } from "../src/history/event-store.js";
2+
import { ObsidianOrchestrator } from "../src/integrations/obsidian/obsidian-orchestrator.js";
3+
import { ConfigManager } from "../src/core/config.js";
4+
import * as path from "path";
5+
import * as fs from "fs";
6+
7+
/**
8+
* Historical Migration Script
9+
* Sweeps the EventStore (events.db) and pushes all old session data to the Obsidian Vault.
10+
*/
11+
async function migrate() {
12+
console.log("Starting Historical Migration to Obsidian...");
13+
14+
// Force the orchestrator to use the global obsidian vault
15+
ConfigManager.getObsidianConfig = () => ({ vaultPath: "C:\\repo\\global.obsidian" });
16+
17+
const store = EventStore.getInstance();
18+
const db = (store as any).db;
19+
20+
// 1. Get all unique projects (repo_ids)
21+
const repos = db.prepare("SELECT DISTINCT repo_id FROM prompts WHERE repo_id IS NOT NULL").all() as { repo_id: string }[];
22+
console.log(`Found ${repos.length} projects with history.`);
23+
24+
for (const { repo_id } of repos) {
25+
console.log(`\nMigrating project: ${repo_id}...`);
26+
27+
// Simulate a rootPath for the orchestrator (it uses basename(rootPath) as repoId)
28+
// We'll use a dummy path that ends with the repo_id
29+
const dummyRootPath = `C:\\repo\\${repo_id}`;
30+
31+
// 2. Fetch all successful executions for this repo
32+
const executions = db.prepare(`
33+
SELECT p.raw_prompt, e.result_summary, e.ended_at, e.executor_name
34+
FROM prompts p
35+
JOIN executions e ON p.id = e.prompt_id
36+
WHERE p.repo_id = ? AND e.status = 'completed'
37+
ORDER BY e.ended_at ASC
38+
`).all(repo_id) as any[];
39+
40+
console.log(`- Found ${executions.length} historical executions.`);
41+
42+
for (const exec of executions) {
43+
const summary = `Historical: ${exec.result_summary || "Agent execution"}`;
44+
const rationale = `Prompt: ${exec.raw_prompt}\n\nExecutor: ${exec.executor_name}\nDate: ${new Date(exec.ended_at).toLocaleString()}`;
45+
46+
await ObsidianOrchestrator.logActivity(dummyRootPath, summary, rationale);
47+
}
48+
49+
// 3. Fetch all commits for this repo
50+
const commits = db.prepare(`
51+
SELECT message, committed_at, author, sha
52+
FROM commits
53+
WHERE repo_id = ?
54+
ORDER BY committed_at ASC
55+
`).all(repo_id) as any[];
56+
57+
console.log(`- Found ${commits.length} historical commits.`);
58+
59+
for (const commit of commits) {
60+
const summary = `Historical Commit: ${commit.sha.substring(0, 7)} - ${commit.message}`;
61+
const rationale = `Author: ${commit.author}\nDate: ${new Date(commit.committed_at).toLocaleString()}`;
62+
63+
await ObsidianOrchestrator.logActivity(dummyRootPath, summary, rationale);
64+
}
65+
66+
// 4. Sync lessons (Engineering Mandates)
67+
await ObsidianOrchestrator.syncToWiki(dummyRootPath);
68+
}
69+
70+
console.log("\nMigration Complete!");
71+
process.exit(0);
72+
}
73+
74+
migrate().catch(err => {
75+
console.error("Migration failed:", err);
76+
process.exit(1);
77+
});

universal-refiner/src/core/background-service.ts

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,11 @@ import { AutoPilotStatus } from "./autopilot-status.js";
77
import { RuntimeLogger } from "./logger.js";
88
import { CommandCenterDashboard } from "./dashboard.js";
99
import { SerializedJobQueue } from "./job-queue.js";
10+
import { ExecutionOrchestrator } from "./execution-orchestrator.js";
11+
import { EventStore } from "../history/event-store.js";
12+
import { exec } from "child_process";
13+
import * as util from "util";
14+
const execAsync = util.promisify(exec);
1015

1116
export class BackgroundAutonomyService {
1217
private watcher: chokidar.FSWatcher | null = null;
@@ -100,14 +105,21 @@ export class BackgroundAutonomyService {
100105
const engine = new CorrelationEngine();
101106
const extractor = new LessonExtractor(this.requestModelText);
102107
const lessonsBefore = AutoPilotStatus.getSnapshot().stats.lessonsExtracted;
108+
const orchestrator = new ExecutionOrchestrator(EventStore.getInstance(), this.requestModelText);
109+
103110
const results = await Promise.allSettled([
104111
engine.correlateAll(),
105112
extractor.extractNewLessons(),
113+
extractor.extractFailureLessons(),
106114
]);
115+
107116
const rejected = results.find((result): result is PromiseRejectedResult => result.status === "rejected");
108117
if (rejected) {
109118
throw rejected.reason;
110119
}
120+
121+
await this.attemptSelfHealing(orchestrator);
122+
111123
const lessonsAfter = AutoPilotStatus.getSnapshot().stats.lessonsExtracted;
112124
if (lessonsAfter > lessonsBefore) {
113125
AutoPilotStatus.record(`Extracted ${lessonsAfter - lessonsBefore} lesson(s)`, "lesson");
@@ -116,7 +128,16 @@ export class BackgroundAutonomyService {
116128
AutoPilotStatus.incrementCycles();
117129
AutoPilotStatus.setActive();
118130
AutoPilotStatus.record("Cycle complete", "cycle_complete");
119-
CommandCenterDashboard.log("Background Autonomy: Correlation and lesson extraction complete.");
131+
CommandCenterDashboard.log("Background Autonomy: Correlation, lesson extraction, and self-healing complete.");
132+
133+
try {
134+
await execAsync("npx tsx scripts/obsidian-sync.ts", { cwd: this.rootPath });
135+
CommandCenterDashboard.log("Background Autonomy: Synced to Obsidian Vault.");
136+
AutoPilotStatus.record("Synced to Obsidian Vault", "cycle_complete");
137+
} catch (syncError) {
138+
RuntimeLogger.error("Failed to sync to Obsidian", syncError);
139+
CommandCenterDashboard.log("Background Autonomy: Failed to sync to Obsidian Vault.");
140+
}
120141

121142
} catch (error) {
122143
AutoPilotStatus.setIdle();
@@ -127,6 +148,27 @@ export class BackgroundAutonomyService {
127148
}
128149
}
129150

151+
private async attemptSelfHealing(orchestrator: ExecutionOrchestrator) {
152+
if (AutoPilotStatus.getSnapshot().state !== "active") return;
153+
154+
try {
155+
const db = (EventStore.getInstance() as any).db;
156+
// Find recently approved lessons that target an execution
157+
const lessons = db.prepare(`
158+
SELECT id, execution_id FROM lessons
159+
WHERE approved = 1 AND execution_id IS NOT NULL
160+
ORDER BY created_at DESC LIMIT 10
161+
`).all();
162+
163+
for (const lesson of lessons) {
164+
// Heal and retry
165+
await orchestrator.healAndRetry(lesson.execution_id, lesson.id);
166+
}
167+
} catch (error) {
168+
RuntimeLogger.error("Self-healing failed", error);
169+
}
170+
}
171+
130172
public stop() {
131173
if (this.watcher) {
132174
this.watcher.close();

universal-refiner/src/core/config.ts

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@ export interface RefinerConfig {
66
mandates?: string[];
77
ignoredPaths?: string[];
88
semantic?: Partial<SemanticConfig>;
9+
atlassian?: any;
10+
obsidian?: any;
911
}
1012

1113
export interface SemanticConfig {
@@ -88,6 +90,14 @@ export class ConfigManager {
8890
};
8991
}
9092

93+
static getAtlassianConfig(rootPath: string = "."): any | null {
94+
return this.loadConfig(rootPath).atlassian || null;
95+
}
96+
97+
static getObsidianConfig(rootPath: string = "."): any | null {
98+
return this.loadConfig(rootPath).obsidian || null;
99+
}
100+
91101
static getPredictiveMandates(): string[] {
92102
const logs = AgenticBlackboard.getLogs();
93103
const recent = logs.slice(0, 10).map(l => l.message.toLowerCase());

universal-refiner/src/core/dashboard.html

Lines changed: 39 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -214,14 +214,46 @@ <h2>Provider Metrics</h2>
214214
const data = await res.json();
215215
document.getElementById('timeline-terminal').innerHTML = data.map(e => {
216216
let icon = 'EVT', color = 'var(--text)';
217-
if (e.type === 'prompt') { icon = 'PRM'; color = 'var(--accent)'; }
217+
let extraHtml = '';
218+
219+
if (e.type === 'prompt') {
220+
icon = 'PRM'; color = 'var(--accent)';
221+
if (e.details && e.details.intent === 'self-heal') {
222+
icon = 'HEAL'; color = '#10b981';
223+
extraHtml += `<div class="badge" style="background: rgba(16, 185, 129, 0.15); color: #6ee7b7; border-color: rgba(16, 185, 129, 0.3); margin-top: 8px; display: inline-block;">AUTONOMOUS SELF-HEALING RETRY</div>`;
224+
}
225+
if (e.details && e.details.normalized_prompt) {
226+
extraHtml += `<div style="margin-top: 8px; padding: 10px; background: rgba(56, 189, 248, 0.05); border-left: 2px solid var(--accent); border-radius: 0 4px 4px 0; font-size: 0.75rem; white-space: pre-wrap; color: var(--dim);"><strong>IMPROVED PROMPT:</strong><br/>${escapeHtml(e.details.normalized_prompt)}</div>`;
227+
}
228+
}
218229
else if (e.type === 'commit') { icon = 'GIT'; color = '#f472b6'; }
219230
else if (e.type === 'log') { icon = 'LOG'; color = 'var(--dim)'; }
231+
else if (e.type === 'execution') {
232+
icon = 'EXEC';
233+
color = e.event_type === 'failed' ? '#ef4444' : '#22c55e';
234+
let execBadge = e.event_type === 'failed'
235+
? `<span class="badge" style="background: rgba(239, 68, 68, 0.15); color: #fca5a5; border-color: rgba(239, 68, 68, 0.3);">AI ERROR</span>`
236+
: `<span class="badge" style="background: rgba(34, 197, 94, 0.15); color: #86efac; border-color: rgba(34, 197, 94, 0.3);">SUCCESS</span>`;
237+
238+
extraHtml = `<div style="margin-top: 8px;">
239+
${execBadge} <span style="color: var(--dim); font-size: 0.7rem; margin-left: 6px;">EXECUTOR: ${escapeHtml(e.author || 'unknown')}</span>
240+
</div>`;
241+
242+
if (e.event_type === 'failed' && e.details && e.details.error) {
243+
extraHtml += `<div style="margin-top: 8px; padding: 10px; background: rgba(239, 68, 68, 0.05); border-left: 2px solid #ef4444; border-radius: 0 4px 4px 0; font-size: 0.75rem; white-space: pre-wrap; color: #fca5a5; font-family: monospace;"><strong>RAW ERROR:</strong><br/>${escapeHtml(typeof e.details.error === 'object' ? JSON.stringify(e.details.error, null, 2) : String(e.details.error))}</div>`;
244+
} else if (e.event_type === 'completed' && e.details && e.details.healedResponse) {
245+
extraHtml += `<div style="margin-top: 8px; padding: 10px; background: rgba(34, 197, 94, 0.05); border-left: 2px solid #22c55e; border-radius: 0 4px 4px 0; font-size: 0.75rem; white-space: pre-wrap; color: #86efac;"><strong>HEALED RESPONSE:</strong><br/>${escapeHtml(e.details.healedResponse)}</div>`;
246+
}
247+
}
248+
220249
return `
221250
<div class="log-line">
222251
<span class="log-ts">${new Date(e.timestamp).toLocaleTimeString()}</span>
223252
<span class="log-icon" style="color:${color}">${icon}</span>
224-
<span>${escapeHtml(e.summary)}</span>
253+
<div style="flex-grow: 1;">
254+
<div style="white-space: pre-wrap; font-family: monospace; font-size: 0.85rem;">${escapeHtml(e.summary)}</div>
255+
${extraHtml}
256+
</div>
225257
</div>
226258
`;
227259
}).join('');
@@ -328,9 +360,13 @@ <h2>Provider Metrics</h2>
328360
}
329361
}
330362

363+
// Auto-poll every 3 seconds so the user never has to click anything
364+
setInterval(() => {
365+
refreshData().catch(err => console.error("Auto-refresh failed:", err));
366+
}, 3000);
367+
331368
// Initial Load
332369
refreshData();
333-
setInterval(refreshData, 10000);
334370
</script>
335371
</body>
336372
</html>

universal-refiner/src/core/dashboard.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -241,8 +241,10 @@ export class CommandCenterDashboard {
241241

242242
app.get("/api/timeline", async (c) => {
243243
try {
244+
const store = EventStore.getInstance();
245+
const repoId = store.ensureRepository(this.resolveSelectedPath(c.req.query("project"))).id;
244246
const provider = new TimelineProvider();
245-
const timeline = provider.getUnifiedTimeline(50);
247+
const timeline = provider.getUnifiedTimeline(50, repoId);
246248
return c.json(timeline);
247249
} catch (error) {
248250
this.logRouteError("api/timeline", error);
Lines changed: 119 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
import { EventStore } from "../history/event-store.js";
2+
import { randomUUID } from "crypto";
3+
import { RuntimeLogger } from "./logger.js";
4+
import { CommandCenterDashboard } from "./dashboard.js";
5+
import { AutoPilotStatus } from "./autopilot-status.js";
6+
7+
export class ExecutionOrchestrator {
8+
constructor(
9+
private eventStore: EventStore,
10+
private requestModelText: (taskName: string, userPrompt: string, maxTokens: number) => Promise<string | null>
11+
) {}
12+
13+
async healAndRetry(executionId: string, lessonId: string): Promise<boolean> {
14+
const db = (this.eventStore as any).db;
15+
16+
// Fetch execution and lesson
17+
const execution = this.eventStore.getExecutionByPromptId(
18+
db.prepare("SELECT prompt_id FROM executions WHERE id = ?").get(executionId)?.prompt_id || ""
19+
);
20+
if (!execution) {
21+
RuntimeLogger.warn(`Execution ${executionId} not found for healing.`);
22+
return false;
23+
}
24+
25+
const lesson = db.prepare("SELECT * FROM lessons WHERE id = ?").get(lessonId);
26+
if (!lesson) {
27+
RuntimeLogger.warn(`Lesson ${lessonId} not found for healing.`);
28+
return false;
29+
}
30+
31+
// Fetch the original prompt
32+
const prompt = db.prepare("SELECT * FROM prompts WHERE id = ?").get(execution.prompt_id);
33+
if (!prompt) {
34+
RuntimeLogger.warn(`Original prompt not found for execution ${executionId}.`);
35+
return false;
36+
}
37+
38+
const retryCount = db.prepare(`SELECT COUNT(*) as count FROM prompts WHERE intent = 'self-heal' AND raw_prompt LIKE ?`).get(`%[HEALING: ${executionId}]%`)?.count || 0;
39+
40+
const MAX_RETRIES = 2;
41+
if (retryCount >= MAX_RETRIES) {
42+
RuntimeLogger.warn(`Max retries (${MAX_RETRIES}) reached for execution ${executionId}.`);
43+
return false;
44+
}
45+
46+
CommandCenterDashboard.log(`[Auto-Heal] Retrying execution ${executionId} using lesson: ${lesson.title}`);
47+
48+
// Create the healed prompt structure
49+
const healedPromptContent = `[HEALING: ${executionId}]\nThe previous execution of this prompt failed.
50+
51+
Original Request:
52+
${prompt.raw_prompt}
53+
54+
Extracted Lesson to Apply:
55+
Title: ${lesson.title}
56+
Rule: ${lesson.summary}
57+
58+
Please re-execute the original request while strictly adhering to the lesson above to avoid the previous failure.`;
59+
60+
const newPromptId = `prm_heal_${randomUUID()}`;
61+
62+
this.eventStore.recordPrompt({
63+
id: newPromptId,
64+
client: "Auto-Heal",
65+
agent_name: "ExecutionOrchestrator",
66+
raw_prompt: healedPromptContent,
67+
intent: "self-heal",
68+
repo_id: prompt.repo_id,
69+
});
70+
71+
const newExecId = `exec_heal_${randomUUID()}`;
72+
const now = new Date().toISOString();
73+
74+
this.eventStore.recordExecution({
75+
id: newExecId,
76+
prompt_id: newPromptId,
77+
workflow_name: "self-healing",
78+
executor_name: "ExecutionOrchestrator",
79+
status: "running",
80+
started_at: now,
81+
});
82+
83+
try {
84+
const responseText = await this.requestModelText(
85+
"self_heal",
86+
healedPromptContent,
87+
4000
88+
);
89+
90+
if (!responseText) {
91+
throw new Error("Semantic provider returned null response.");
92+
}
93+
94+
this.eventStore.updateExecution({
95+
id: newExecId,
96+
status: "completed",
97+
ended_at: new Date().toISOString(),
98+
result_summary: `Healed execution succeeded.`,
99+
artifacts_json: JSON.stringify({ healedResponse: responseText })
100+
});
101+
102+
AutoPilotStatus.record(`Healed execution ${executionId} successfully.`, "cycle_complete");
103+
CommandCenterDashboard.log(`[Auto-Heal] Healed execution ${newExecId} succeeded.`);
104+
return true;
105+
106+
} catch (error) {
107+
const errorMessage = error instanceof Error ? error.message : String(error);
108+
this.eventStore.updateExecution({
109+
id: newExecId,
110+
status: "failed",
111+
ended_at: new Date().toISOString(),
112+
result_summary: `Healed execution failed: ${errorMessage}`,
113+
});
114+
AutoPilotStatus.record(`Healed execution ${executionId} failed: ${errorMessage}`, "error");
115+
CommandCenterDashboard.log(`[Auto-Heal] Healed execution ${newExecId} failed again.`);
116+
return false;
117+
}
118+
}
119+
}

0 commit comments

Comments
 (0)