Skip to content

Commit 69ee272

Browse files
HopsonOHopsonO
authored
Merge pull request #12 from O-Labz/memory-improvements
minor fix
2 parents 065f72a + 5642d1e commit 69ee272

8 files changed

Lines changed: 255 additions & 1 deletion

File tree

‎dashboard/src/pages/Repositories/Repositories.tsx‎

Lines changed: 180 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,15 @@ interface Repository {
1515
lastIndexedAt?: string;
1616
}
1717

18+
interface IndexQueueStatus {
19+
inFlight: number;
20+
queued: number;
21+
maxConcurrent: number;
22+
maxDepth: number;
23+
capacityUsed: number;
24+
capacityTotal: number;
25+
}
26+
1827
export default function Repositories() {
1928
const [repos, setRepos] = useState<Repository[]>([]);
2029
const [loading, setLoading] = useState(true);
@@ -30,15 +39,21 @@ export default function Repositories() {
3039
const [showWorkspaceDialog, setShowWorkspaceDialog] = useState(false);
3140
const [newWorkspacePath, setNewWorkspacePath] = useState('');
3241
const [changingWorkspace, setChangingWorkspace] = useState(false);
42+
const [indexQueue, setIndexQueue] = useState<IndexQueueStatus | null>(null);
3343
const { toasts, push: toast, dismiss: dismissToast } = useToast();
3444

3545
useEffect(() => {
3646
loadRepositories();
47+
loadIndexQueue();
3748
fetch('/api/health')
3849
.then((r) => r.json())
3950
.then((d) => setWorkspaceRoot(d.workspaceRoot || ''))
4051
.catch((err) => console.warn('Failed to load workspace info:', err));
4152
loadConfig();
53+
54+
// Poll queue status every 2 seconds for real-time updates
55+
const interval = setInterval(loadIndexQueue, 2000);
56+
return () => clearInterval(interval);
4257
}, []);
4358

4459
const loadConfig = async () => {
@@ -64,6 +79,16 @@ export default function Repositories() {
6479
}
6580
};
6681

82+
const loadIndexQueue = async () => {
83+
try {
84+
const response = await fetch('/api/metrics');
85+
const data = await response.json();
86+
setIndexQueue(data.indexQueue || null);
87+
} catch (error) {
88+
console.error('Failed to load index queue status:', error);
89+
}
90+
};
91+
6792
const handleReindex = async (repoId: string) => {
6893
setBusyAction(`reindex-${repoId}`);
6994
try {
@@ -507,6 +532,161 @@ export default function Repositories() {
507532
</div>
508533
)}
509534

535+
{/* Indexing Queue Status Card */}
536+
{indexQueue && (
537+
<div className="md:col-span-12 bg-surface-container-low p-6 rounded-xl">
538+
<div className="flex items-center justify-between mb-4">
539+
<div className="flex items-center gap-3">
540+
<div className="w-10 h-10 bg-tertiary-container flex items-center justify-center rounded-xl">
541+
<span className="material-symbols-outlined text-on-tertiary-container">
542+
{indexQueue.inFlight > 0 ? 'progress_activity' : 'task_alt'}
543+
</span>
544+
</div>
545+
<div>
546+
<h4 className="text-[0.875rem] font-bold text-on-surface">Indexing Queue</h4>
547+
<p className="text-[0.75rem] text-on-surface-variant">
548+
{indexQueue.inFlight === 0 && indexQueue.queued === 0
549+
? 'No repositories currently indexing'
550+
: `${indexQueue.inFlight} indexing · ${indexQueue.queued} queued`}
551+
</p>
552+
</div>
553+
</div>
554+
<div className="flex items-center gap-2">
555+
{indexQueue.inFlight > 0 && (
556+
<span className="px-3 py-1.5 bg-green-500/10 text-green-600 text-xs font-semibold rounded-lg flex items-center gap-1.5">
557+
<span className="w-2 h-2 rounded-full bg-green-500 animate-pulse"></span>
558+
Active
559+
</span>
560+
)}
561+
<span className="text-[0.75rem] text-on-surface-variant">
562+
Capacity: {indexQueue.capacityUsed}/{indexQueue.capacityTotal}
563+
</span>
564+
</div>
565+
</div>
566+
567+
{/* Progress Bar */}
568+
<div className="mb-4">
569+
<div className="flex justify-between text-xs text-on-surface-variant mb-2">
570+
<span>Queue Usage</span>
571+
<span>{Math.round((indexQueue.capacityUsed / indexQueue.capacityTotal) * 100)}%</span>
572+
</div>
573+
<div className="w-full bg-surface-container h-2.5 rounded-full overflow-hidden">
574+
<div
575+
className="h-full rounded-full transition-all duration-500"
576+
style={{
577+
width: `${Math.min(100, (indexQueue.capacityUsed / indexQueue.capacityTotal) * 100)}%`,
578+
background: indexQueue.capacityUsed >= indexQueue.capacityTotal
579+
? 'linear-gradient(90deg, var(--error), var(--error-dim))'
580+
: indexQueue.capacityUsed > indexQueue.capacityTotal * 0.7
581+
? 'linear-gradient(90deg, #f59e0b, #d97706)'
582+
: 'linear-gradient(90deg, var(--tertiary), var(--tertiary-dim))',
583+
}}
584+
/>
585+
</div>
586+
</div>
587+
588+
{/* Stats Grid */}
589+
<div className="grid grid-cols-2 md:grid-cols-4 gap-4">
590+
<div className="bg-surface-container-highest p-4 rounded-lg">
591+
<div className="flex items-center gap-2 mb-2">
592+
<span className="material-symbols-outlined text-[16px] text-green-600">
593+
{indexQueue.inFlight > 0 ? 'sync' : 'check_circle'}
594+
</span>
595+
<span className="text-[0.6875rem] font-semibold uppercase tracking-wider text-outline">
596+
Active
597+
</span>
598+
</div>
599+
<span className={`text-2xl font-bold ${indexQueue.inFlight > 0 ? 'text-green-600' : 'text-on-surface'}`}>
600+
{indexQueue.inFlight}
601+
</span>
602+
<p className="text-[0.6875rem] text-on-surface-variant mt-1">
603+
Max: {indexQueue.maxConcurrent}
604+
</p>
605+
</div>
606+
607+
<div className="bg-surface-container-highest p-4 rounded-lg">
608+
<div className="flex items-center gap-2 mb-2">
609+
<span className="material-symbols-outlined text-[16px] text-orange-500">
610+
schedule
611+
</span>
612+
<span className="text-[0.6875rem] font-semibold uppercase tracking-wider text-outline">
613+
Queued
614+
</span>
615+
</div>
616+
<span className={`text-2xl font-bold ${indexQueue.queued > 0 ? 'text-orange-500' : 'text-on-surface'}`}>
617+
{indexQueue.queued}
618+
</span>
619+
<p className="text-[0.6875rem] text-on-surface-variant mt-1">
620+
Max: {indexQueue.maxDepth}
621+
</p>
622+
</div>
623+
624+
<div className="bg-surface-container-highest p-4 rounded-lg">
625+
<div className="flex items-center gap-2 mb-2">
626+
<span className="material-symbols-outlined text-[16px] text-tertiary">
627+
inventory_2
628+
</span>
629+
<span className="text-[0.6875rem] font-semibold uppercase tracking-wider text-outline">
630+
Available
631+
</span>
632+
</div>
633+
<span className="text-2xl font-bold text-on-surface">
634+
{indexQueue.capacityTotal - indexQueue.capacityUsed}
635+
</span>
636+
<p className="text-[0.6875rem] text-on-surface-variant mt-1">
637+
Slots free
638+
</p>
639+
</div>
640+
641+
<div className="bg-surface-container-highest p-4 rounded-lg">
642+
<div className="flex items-center gap-2 mb-2">
643+
<span className="material-symbols-outlined text-[16px] text-on-surface-variant">
644+
info
645+
</span>
646+
<span className="text-[0.6875rem] font-semibold uppercase tracking-wider text-outline">
647+
Status
648+
</span>
649+
</div>
650+
<span className={`text-lg font-bold ${
651+
indexQueue.capacityUsed >= indexQueue.capacityTotal
652+
? 'text-error'
653+
: indexQueue.capacityUsed > 0
654+
? 'text-green-600'
655+
: 'text-on-surface'
656+
}`}>
657+
{indexQueue.capacityUsed >= indexQueue.capacityTotal
658+
? 'Full'
659+
: indexQueue.capacityUsed > 0
660+
? 'Processing'
661+
: 'Idle'}
662+
</span>
663+
{indexQueue.capacityUsed >= indexQueue.capacityTotal && (
664+
<p className="text-[0.6875rem] text-error mt-1">
665+
New requests will be rejected
666+
</p>
667+
)}
668+
</div>
669+
</div>
670+
671+
{/* Info Banner */}
672+
{indexQueue.capacityUsed >= indexQueue.capacityTotal && (
673+
<div className="mt-4 bg-error/10 border border-error/20 rounded-lg p-3 flex gap-3">
674+
<span className="material-symbols-outlined text-error text-[18px] shrink-0">
675+
warning
676+
</span>
677+
<div className="text-xs">
678+
<p className="text-error font-semibold mb-1">Queue at capacity</p>
679+
<p className="text-on-surface-variant leading-relaxed">
680+
The indexing queue is full. New repository additions will return HTTP 429
681+
until current jobs complete. Wait for active indexing to finish or increase
682+
INDEX_QUEUE_MAX_DEPTH environment variable.
683+
</p>
684+
</div>
685+
</div>
686+
)}
687+
</div>
688+
)}
689+
510690
{/* Standard Repository Cards */}
511691
{repos.slice(1).map((repo) => (
512692
<div

‎src/api/routes/metrics.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export interface MetricsRouteOptions {
2727
vectorStore?: any;
2828
embeddingProvider?: any;
2929
mcpServer?: any;
30+
indexQueue?: any;
3031
}
3132

3233
/**
@@ -143,6 +144,23 @@ export async function registerMetricsRoutes(
143144
totalDiskUsage: dbStats.databaseSize + (options.vectorStore ? await options.vectorStore.getSize().catch(() => 0) : 0),
144145
},
145146

147+
// Index queue status
148+
indexQueue: options.indexQueue ? {
149+
inFlight: options.indexQueue.getStats().inFlight,
150+
queued: options.indexQueue.getStats().queued,
151+
maxConcurrent: options.indexQueue.maxConcurrent || 1,
152+
maxDepth: options.indexQueue.maxDepth || 16,
153+
capacityUsed: options.indexQueue.getStats().inFlight + options.indexQueue.getStats().queued,
154+
capacityTotal: (options.indexQueue.maxConcurrent || 1) + (options.indexQueue.maxDepth || 16),
155+
} : {
156+
inFlight: 0,
157+
queued: 0,
158+
maxConcurrent: 1,
159+
maxDepth: 16,
160+
capacityUsed: 0,
161+
capacityTotal: 17,
162+
},
163+
146164
// Timestamp
147165
timestamp: new Date().toISOString(),
148166
};

‎src/api/routes/repositories.ts‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ export interface RepositoryRouteOptions {
3838
watcher?: any;
3939
vectorStore?: any;
4040
indexQueue?: any;
41+
autoWatch?: boolean;
4142
}
4243

4344
/**
@@ -140,6 +141,20 @@ export async function registerRepositoryRoutes(
140141
nodesCreated: job.nodesCreated,
141142
edgesCreated: job.edgesCreated,
142143
});
144+
145+
// Auto-watch after successful indexing if enabled
146+
if (options.watcher && options.autoWatch) {
147+
try {
148+
const repo = options.storage.listRepositories().find(r => r.id === job.repositoryId);
149+
if (repo && !repo.isWatched) {
150+
options.watcher.watch(repo.path, job.repositoryId);
151+
options.storage.updateRepositoryWatchStatus(job.repositoryId, true);
152+
console.log(`Auto-watch enabled for ${repo.name}`);
153+
}
154+
} catch (watchError) {
155+
console.warn(`Failed to auto-watch repository ${job.repositoryId}:`, watchError);
156+
}
157+
}
143158
} catch (error) {
144159
options.broadcaster.broadcast(WebSocketEvents.INDEX_ERROR, {
145160
path: input.path,
@@ -308,6 +323,20 @@ export async function registerRepositoryRoutes(
308323
nodesCreated: job.nodesCreated,
309324
edgesCreated: job.edgesCreated,
310325
});
326+
327+
// Auto-watch after successful reindexing if enabled
328+
if (options.watcher && options.autoWatch) {
329+
try {
330+
const repoAfter = options.storage.listRepositories().find(r => r.id === job.repositoryId);
331+
if (repoAfter && !repoAfter.isWatched) {
332+
options.watcher.watch(repoAfter.path, job.repositoryId);
333+
options.storage.updateRepositoryWatchStatus(job.repositoryId, true);
334+
console.log(`Auto-watch enabled for ${repoAfter.name}`);
335+
}
336+
} catch (watchError) {
337+
console.warn(`Failed to auto-watch repository ${job.repositoryId}:`, watchError);
338+
}
339+
}
311340
} catch (error) {
312341
options.broadcaster.broadcast(WebSocketEvents.INDEX_ERROR, {
313342
repositoryId: id,

‎src/api/server.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@ export interface APIServerOptions {
5050
configManager?: any;
5151
eml?: EmlServices;
5252
indexQueue?: any;
53+
config?: any;
5354
}
5455

5556
export interface APIServer {
@@ -202,6 +203,7 @@ export async function createAPIServer(
202203
watcher: options.watcher,
203204
vectorStore: options.vectorStore,
204205
indexQueue: options.indexQueue,
206+
autoWatch: options.config ? options.config.autoWatch.value : true,
205207
});
206208

207209
await registerSearchRoutes(fastify, {
@@ -225,6 +227,7 @@ export async function createAPIServer(
225227
vectorStore: options.vectorStore,
226228
embeddingProvider: options.embeddingProvider,
227229
mcpServer: options.mcpServer,
230+
indexQueue: options.indexQueue,
228231
});
229232

230233
await registerMcpConfigRoutes(fastify, {

‎src/core/config.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ const DEFAULT_CONFIG = {
3636
dataDir: '/data',
3737
autoIndex: true,
3838
watchEnabled: true,
39+
autoWatch: true,
3940
logLevel: 'info' as const,
4041
embeddingConcurrency: 5,
4142
embeddingBatchSize: 20,
@@ -68,6 +69,7 @@ const ENV_VAR_MAP = {
6869
dataDir: 'CONTEXT_SIMPLO_DATA_DIR',
6970
autoIndex: 'CONTEXT_SIMPLO_AUTO_INDEX',
7071
watchEnabled: 'CONTEXT_SIMPLO_WATCH',
72+
autoWatch: 'CONTEXT_SIMPLO_AUTO_WATCH',
7173
logLevel: 'CONTEXT_SIMPLO_LOG_LEVEL',
7274
embeddingConcurrency: 'EMBEDDING_CONCURRENCY',
7375
embeddingBatchSize: 'EMBEDDING_BATCH_SIZE',
@@ -103,6 +105,8 @@ export interface DashboardConfig {
103105
llmEmbeddingModel?: string;
104106
embeddingConcurrency?: number;
105107
embeddingBatchSize?: number;
108+
autoIndex?: boolean;
109+
autoWatch?: boolean;
106110
}
107111

108112
function parseEnvValue(key: ConfigKey, envValue: string | undefined): unknown {
@@ -192,6 +196,9 @@ export function loadConfig(dashboardConfig?: DashboardConfig): AppConfig {
192196
const envWatchEnabled = parseEnvValue('watchEnabled', process.env[ENV_VAR_MAP.watchEnabled]) as
193197
| boolean
194198
| undefined;
199+
const envAutoWatch = parseEnvValue('autoWatch', process.env[ENV_VAR_MAP.autoWatch]) as
200+
| boolean
201+
| undefined;
195202
const envLogLevel = parseEnvValue('logLevel', process.env[ENV_VAR_MAP.logLevel]) as
196203
| 'error' | 'warn' | 'info' | 'debug'
197204
| undefined;
@@ -356,6 +363,13 @@ export function loadConfig(dashboardConfig?: DashboardConfig): AppConfig {
356363
DEFAULT_CONFIG.watchEnabled
357364
);
358365

366+
const autoWatch = createConfigValue(
367+
'autoWatch',
368+
envAutoWatch,
369+
dashboardConfig?.autoWatch,
370+
DEFAULT_CONFIG.autoWatch
371+
);
372+
359373
const logLevel = createConfigValue(
360374
'logLevel',
361375
envLogLevel,
@@ -496,6 +510,7 @@ export function loadConfig(dashboardConfig?: DashboardConfig): AppConfig {
496510
dataDir,
497511
autoIndex,
498512
watchEnabled,
513+
autoWatch,
499514
logLevel,
500515
embeddingConcurrency,
501516
embeddingBatchSize,

‎src/core/parse-pool.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ export class ParsePool {
7676

7777
// Handle worker exit (crash or termination)
7878
worker.on('exit', (code) => {
79-
if (code !== 0 && !this.isTerminated) {
79+
if (code !== 0 && !this.isTerminated && !(poolWorker as any).recycling) {
8080
console.warn(`index.worker.crashed`, {
8181
workerId,
8282
exitCode: code,
@@ -105,6 +105,9 @@ export class ParsePool {
105105
parsedCount: poolWorker.parsedCount,
106106
});
107107

108+
// Mark worker for recycling so exit handler doesn't log it as a crash
109+
(poolWorker as any).recycling = true;
110+
108111
// Terminate the old worker and create a new one
109112
poolWorker.worker.terminate();
110113
const index = this.workers.findIndex((w) => w.workerId === poolWorker.workerId);

0 commit comments

Comments
 (0)