Incremental Parsing

package.json

@@ -207,6 +207,7 @@
     "command-line-usage": "^7.0.3",
     "commonmark": "^0.31.2",
     "dagre": "^0.8.5",
+    "diff": "^8.0.3",
     "gray-matter": "^4.0.3",
     "joi": "^18.0.1",
     "lz-string": "^1.5.0",

src/config.ts

@@ -250,7 +250,7 @@ export const FlowrConfig = {
 	 * The default configuration for flowR, used when no config file is found or when a config file is missing some options.
 	 * You can use this as a base for your own config and only specify the options you want to change.
 	 */
-	default(): FlowrConfig {
+	default(this: void): FlowrConfig {
 		return {
 			ignoreSourceCalls: false,
 			semantics:         {
@@ -364,7 +364,7 @@ export const FlowrConfig = {
 	/**
 	 * Parses the given JSON string as a flowR config file, returning the resulting config object if the parsing and validation were successful, or `undefined` if there was an error.
 	 */
-	parse(jsonString: string): FlowrConfig | undefined {
+	parse(this: void, jsonString: string): FlowrConfig | undefined {
 		try {
 			const parsed   = JSON.parse(jsonString) as FlowrConfig;
 			const validate = FlowrConfig.Schema.validate(parsed);
@@ -383,14 +383,14 @@ export const FlowrConfig = {
 	 * Creates a new flowr config that has the updated values.
 	 */
 	// eslint-disable-next-line @typescript-eslint/no-invalid-void-type
-	amend(config: FlowrConfig, amendmentFunc: (config: DeepWritable<FlowrConfig>) => FlowrConfig | void): FlowrConfig {
+	amend(this: void, config: FlowrConfig, amendmentFunc: (config: DeepWritable<FlowrConfig>) => FlowrConfig | void): FlowrConfig {
 		const newConfig = FlowrConfig.clone(config);
 		return amendmentFunc(newConfig as DeepWritable<FlowrConfig>) ?? newConfig;
 	},
 	/**
 	 * Clones the given flowr config object.
 	 */
-	clone(config: FlowrConfig): FlowrConfig {
+	clone(this: void, config: FlowrConfig): FlowrConfig {
 		return deepClonePreserveUnclonable(config);
 	},
 	/**
@@ -399,7 +399,7 @@ export const FlowrConfig = {
 	 * infer the config from flowR's default locations.
 	 * This is mostly useful for user-facing features.
 	 */
-	fromFile(configFile?: string, configWorkingDirectory = process.cwd()): FlowrConfig {
+	fromFile(this: void, configFile?: string, configWorkingDirectory = process.cwd()): FlowrConfig {
 		try {
 			return loadConfigFromFile(configFile, configWorkingDirectory);
 		} catch(e) {
@@ -410,7 +410,7 @@ export const FlowrConfig = {
 	/**
 	 * Gets the configuration for the given engine type from the config.
 	 */
-	getForEngine<T extends EngineConfig['type']>(config: FlowrConfig, engine: T): EngineConfig & { type: T } | undefined {
+	getForEngine<T extends EngineConfig['type']>(this: void, config: FlowrConfig, engine: T): EngineConfig & { type: T } | undefined {
 		const engines = config.engines;
 		if(engines.length > 0) {
 			return engines.find(e => e.type === engine) as EngineConfig & { type: T } | undefined;
@@ -429,7 +429,7 @@ export const FlowrConfig = {
 	 * console.log(newConfig.solver.variables); // Output: "builtin"
 	 * ```
 	 */
-	setInConfig<Path extends ValidFlowrConfigPaths>(config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): FlowrConfig {
+	setInConfig<Path extends ValidFlowrConfigPaths>(this: void, config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): FlowrConfig {
 		const clone = FlowrConfig.clone(config);
 		objectPath.set(clone, key, value);
 		return clone;
@@ -438,7 +438,7 @@ export const FlowrConfig = {
 	 * Modifies the given config object in place by setting the given value at the given key, where the key is a dot-separated path to the value in the config object.
 	 * @see {@link setInConfig} for a version that returns a new config object instead of modifying the given one in place.
 	 */
-	setInConfigInPlace<Path extends ValidFlowrConfigPaths>(config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): void {
+	setInConfigInPlace<Path extends ValidFlowrConfigPaths>(this: void, config: FlowrConfig, key: Path, value: PathValue<FlowrConfig, Path>): void {
 		objectPath.set(config, key, value);
 	}
 } as const;

src/dataflow/internal/process/functions/call/built-in/built-in-source.ts

@@ -246,7 +246,7 @@ export function sourceRequest<OtherInfo>(rootId: NodeId, request: RParseRequest
 		} else {
 			guard(textRequest !== undefined, `Expected text request to be defined for sourced file ${JSON.stringify(request)}`);
 		}
-		const parsed = (!data.parser.async ? data.parser : new RShellExecutor()).parse(textRequest.r);
+		const parsed = (!data.parser.async ? data.parser : new RShellExecutor()).parse(textRequest.r, data.ctx);
 		const normalized = (typeof parsed !== 'string' ?
 			normalizeTreeSitter({ files: [{ parsed, filePath: textRequest.path }] }, getId, data.ctx.config)
 			: normalize({ files: [{ parsed, filePath: textRequest.path }] }, getId)) as NormalizedAst<OtherInfo & ParentInformation>;

src/documentation/wiki-analyzer.ts

@@ -39,7 +39,9 @@ import { FlowrAnalyzerPlugin } from '../project/plugins/flowr-analyzer-plugin';
 import { FlowrAnalyzerEnvironmentContext } from '../project/context/flowr-analyzer-environment-context';
 import { FlowrAnalyzerFunctionsContext } from '../project/context/flowr-analyzer-functions-context';
 import { FlowrAnalyzerMetaContext } from '../project/context/flowr-analyzer-meta-context';
+import { FlowrAnalyzerIncrementalAnalysisContext } from '../project/context/flowr-analyzer-incremental-analysis-context';
 import { FlowrConfig } from '../config';
+import { FlowrInlineTextFile } from '../project/context/flowr-file';
 
 async function analyzerQuickExample() {
 	const analyzer = await new FlowrAnalyzerBuilder()
@@ -99,11 +101,12 @@ ${
 			'How to add a new plugin': undefined,
 		},
 		'Context Information': {
-			'Files Context':         undefined,
-			'Loading Order Context': undefined,
-			'Dependencies Context':  undefined,
-			'Environment Context':   undefined,
-			'Meta Context':          undefined,
+			'Files Context':                undefined,
+			'Loading Order Context':        undefined,
+			'Dependencies Context':         undefined,
+			'Environment Context':          undefined,
+			'Meta Context':                 undefined,
+			'Incremental Analysis Context': undefined,
 		},
 		'Caching': undefined
 	})
@@ -478,6 +481,50 @@ and the project namespace via
 ${ctx.linkM(FlowrAnalyzerMetaContext, 'getNamespace', { codeFont: true, realNameWrapper: 'i' })}.
 
 
+${section('Incremental Analysis Context', 3)}
+
+The ${ctx.link(FlowrAnalyzerIncrementalAnalysisContext)} is a context that stores analysis information needed for making the next analysis run incremental by reusing the previous analysis results:
+
+${ctx.hierarchy(FlowrAnalyzerIncrementalAnalysisContext, { showImplSnippet: false })}
+
+This context is not an analysis-result cache by itself.
+Instead, it carries forward the minimal state needed by future incremental phases after an invalidation happened.
+At the moment, it is used for incremental parsing with Tree-sitter, but it is intended to become the shared context for additional incremental analysis stages as well.
+
+If the analyzer or context is reset, the incremental information is discarded via
+${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'reset', { codeFont: true, realNameWrapper: 'i' })}.
+In other words, this context only transports incremental handoff state between analysis runs.
+
+${section('Incremental Parsing', 4)}
+
+Currently, the implemented use of this context is Tree-sitter's incremental parsing support.
+When a file is represented by a mutable file provider such as ${ctx.link('FlowrInlineTextFile')} and its content is invalidated via
+${ctx.linkM(FlowrInlineTextFile, 'invalidate', { codeFont: true, realNameWrapper: 'i' })},
+the analyzer receives a file invalidation event.
+At that point, the incremental context only records the file path together with the old source text.
+No edit region is computed eagerly during invalidation.
+
+After a successful parse-oriented analysis run, the analyzer cache stores the latest Tree-sitter parse trees in this context via
+${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'storeOldParseResults', { codeFont: true, realNameWrapper: 'i' })}.
+This gives the next parse run access to the last completed parse snapshot for each file path.
+
+On the next parse run, Tree-sitter combines both pieces of information lazily:
+
+* the previous parse tree obtained from
+  ${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'getOldParseResultOf', { codeFont: true, realNameWrapper: 'i' })}
+* the old source text obtained from
+  ${ctx.linkM(FlowrAnalyzerIncrementalAnalysisContext, 'getAndRemoveOldContentOf', { codeFont: true, realNameWrapper: 'i' })}
+
+Using these together with the current file content, flowR computes a minimal ${ctx.link('Parser.Edit')} only when a new parse is actually requested.
+If the file content did not change, the previous tree can be reused directly.
+Otherwise, the edit is applied to the previous tree and Tree-sitter reparses incrementally instead of starting from scratch.
+The stored old-content entry is consumed when it is used, so invalidation state only survives until the next relevant parse.
+
+${section('Incremental Dataflow', 4)}
+
+This context is planned to also support future incremental dataflow graph computation.
+
+
 ${section('Caching', 2)}
 
 To speed up analyses, flowR provides a caching mechanism that stores intermediate results of the analysis.

src/project/cache/flowr-analyzer-cache.ts

@@ -1,5 +1,5 @@
-import type { KnownParser } from '../../r-bridge/parser';
-import { type CacheInvalidationEvent, CacheInvalidationEventType, FlowrCache } from './flowr-cache';
+import type { KnownParser, ParseStepOutput } from '../../r-bridge/parser';
+import { type InvalidationEvent, InvalidationEventType, FlowrCache } from './flowr-cache';
 import {
 	createDataflowPipeline,
 	type DEFAULT_DATAFLOW_PIPELINE,
@@ -18,7 +18,7 @@ import type { FlowrAnalyzerContext } from '../context/flowr-analyzer-context';
 import { FlowrAnalyzerControlFlowCache } from './flowr-analyzer-controlflow-cache';
 import type { CallGraph } from '../../dataflow/graph/call-graph';
 import { computeCallGraph } from '../../dataflow/graph/call-graph';
-
+import type { Tree } from 'web-tree-sitter';
 interface FlowrAnalyzerCacheOptions<Parser extends KnownParser> {
 	parser:  Parser;
 	context: FlowrAnalyzerContext;
@@ -56,30 +56,33 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 		}) as AnalyzerPipelineExecutor<Parser>;
 		this.controlFlowCache = new FlowrAnalyzerControlFlowCache();
 		this.callGraphCache = undefined;
+		this.computeIfAbsent(true, () => this.pipeline?.getResults(true));
 	}
 
 	public static create<Parser extends KnownParser>(data: FlowrAnalyzerCacheOptions<Parser>): FlowrAnalyzerCache<Parser> {
 		return new FlowrAnalyzerCache<Parser>(data);
 	}
 
-	public override receive(event: CacheInvalidationEvent): void {
+	public override receive(event: InvalidationEvent): void {
 		super.receive(event);
-		switch(event.type) {
-			case CacheInvalidationEventType.Full:
+		const type = event.type;
+		switch(type) {
+			case InvalidationEventType.Full:
+			case InvalidationEventType.FileInvalidate:
 				this.initCacheProviders();
 				break;
 			default:
-				assertUnreachable(event.type);
+				assertUnreachable(type);
 		}
 	}
 
 	private get(): AnalyzerCacheType<Parser> {
 		/* this will do a ref assignment, so indirect force */
-		return this.computeIfAbsent(false, () => this.pipeline.getResults(true));
+		return this.computeIfAbsent(false, () => this.pipeline?.getResults(true));
 	}
 
 	public reset() {
-		this.receive({ type: CacheInvalidationEventType.Full });
+		this.receive({ type: InvalidationEventType.Full });
 	}
 
 	private async runTapeUntil<T>(force: boolean | undefined, until: () => T | undefined): Promise<T> {
@@ -92,10 +95,26 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 		while((g = until()) === undefined && this.pipeline.hasNextStep()) {
 			await this.pipeline.nextStep();
 		}
+
+		this.storeIncrementalSnapshotIfAvailable();
+
 		guard(g !== undefined, 'Could not reach the desired pipeline step, invalid cache state(?)');
 		return g;
 	}
 
+	private storeIncrementalSnapshotIfAvailable(): void {
+		if(this.args.parser.name !== 'tree-sitter') {
+			return;
+		}
+
+		const parse = this.peekParse();
+		if(parse !== undefined) {
+			this.args.context.inc.storeOldParseResults(
+				parse as ParseStepOutput<Tree> // cast needed because of TypeScript's limited narrowing capabilities
+			);
+		}
+	}
+
 	/**
 	 * Get the parse output for the request, parsing if necessary.
 	 * @param force - Do not use the cache, instead force a new parse.
@@ -112,7 +131,7 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 	 * @see {@link FlowrAnalyzerCache#parse} - to get the parse output, parsing if necessary.
 	 */
 	public peekParse(): NonNullable<AnalyzerCacheType<Parser>['parse']> | undefined {
-		return this.get().parse;
+		return this.get()?.parse;
 	}
 
 	/**
@@ -131,7 +150,7 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 	 * @see {@link FlowrAnalyzerCache#normalize} - to get the normalized AST, normalizing if necessary.
 	 */
 	public peekNormalize(): NonNullable<AnalyzerCacheType<Parser>['normalize']> | undefined {
-		return this.get().normalize;
+		return this.get()?.normalize;
 	}
 
 	/**
@@ -150,7 +169,7 @@ export class FlowrAnalyzerCache<Parser extends KnownParser> extends FlowrCache<A
 	 * @see {@link FlowrAnalyzerCache#dataflow} - to get the dataflow graph, computing if necessary.
 	 */
 	public peekDataflow(): NonNullable<AnalyzerCacheType<Parser>['dataflow']> | undefined {
-		return this.get().dataflow;
+		return this.get()?.dataflow;
 	}
 
 	/**

src/project/cache/flowr-cache.ts

@@ -1,37 +1,52 @@
 import { assertUnreachable } from '../../util/assert';
+import type { StringableContent } from '../context/flowr-file';
 
-export const enum CacheInvalidationEventType {
-	Full = 'full'
+export const enum InvalidationEventType {
+	Full = 'full',
+	FileInvalidate = 'file-invalidate',
 }
-export type CacheInvalidationEvent =
-	{ type: CacheInvalidationEventType.Full };
 
-export interface CacheInvalidationEventReceiver {
-	receive(event: CacheInvalidationEvent): void
+export interface FileContentInvalidateEvent<Content extends StringableContent = StringableContent> {
+	readonly type:       InvalidationEventType.FileInvalidate;
+	readonly oldContent: Content | undefined;
+	readonly filePath:   string;
+}
+
+export type InvalidationEvent<Content extends StringableContent = StringableContent> =
+	{ type: InvalidationEventType.Full }
+ |  FileContentInvalidateEvent<Content>;
+
+
+export type InvalidationEventHandler<Content extends StringableContent = StringableContent> = (event: InvalidationEvent<Content>) => void;
+
+export interface InvalidationEventReceiver<Content extends StringableContent = StringableContent> {
+	receive: InvalidationEventHandler<Content>
 }
 
 /**
  * Central class for caching analysis results in FlowR.
  */
-export abstract class FlowrCache<Cache> implements CacheInvalidationEventReceiver {
+export abstract class FlowrCache<Cache> implements InvalidationEventReceiver {
 	private value:      Cache | undefined = undefined;
-	private dependents: CacheInvalidationEventReceiver[] = [];
+	private dependents: InvalidationEventReceiver[] = [];
 
-	public registerDependent(dependent: CacheInvalidationEventReceiver) {
+	public registerDependent(dependent: InvalidationEventReceiver) {
 		this.dependents.push(dependent);
 	}
-	public removeDependent(dependent: CacheInvalidationEventReceiver) {
+	public removeDependent(dependent: InvalidationEventReceiver) {
 		this.dependents = this.dependents.filter(d => d !== dependent);
 	}
 
-	receive(event: CacheInvalidationEvent): void {
+	receive(event: InvalidationEvent): void {
+		const type = event.type;
 		/* we will update this as soon as we support incremental update patterns */
-		switch(event.type) {
-			case CacheInvalidationEventType.Full:
+		switch(type) {
+			case InvalidationEventType.Full:
+			case InvalidationEventType.FileInvalidate:
 				this.value = undefined;
 				break;
 			default:
-				assertUnreachable(event.type);
+				assertUnreachable(type);
 		}
 		/* in the future we want to defer this *after* the dataflow is re-computed, then all receivers can decide whether they need to update */
 		this.notifyDependents(event);
@@ -40,7 +55,7 @@ export abstract class FlowrCache<Cache> implements CacheInvalidationEventReceive
 	/**
 	 * Notify all dependents of a cache invalidation event.
 	 */
-	public notifyDependents(event: CacheInvalidationEvent) {
+	public notifyDependents(event: InvalidationEvent) {
 		for(const dependent of this.dependents) {
 			dependent.receive(event);
 		}