Initial port to new monolithic sourcelib package

Author: StefanH-AT

package-lock.json

@@ -439,10 +439,7 @@
         "typescript": "^6.0.3"
     },
     "dependencies": {
-        "@sourcelib/captions": "^0.1.1",
-        "@sourcelib/fs": "^0.1.1",
-        "@sourcelib/kv": "^0.9.1",
-        "@sourcelib/vmt": "^0.4.0",
-        "list-files-in-dir": "^0.1.5"
+        "list-files-in-dir": "^0.1.5",
+        "sourcelib": "^0.1.1"
     }
 }

package.json

@@ -1,5 +1,5 @@
 import vscode from "vscode";
-import { isWhitespace } from "@sourcelib/kv/";
+import * as sourcelib from "sourcelib";
 import * as main from "../main";
 import * as fs from "fs";
 import { execFile } from "child_process";
@@ -33,7 +33,7 @@ export async function compileSomething(settings: CompileSettings): Promise<void>
         filePath = filePath.toLowerCase();
         workDir = workDir.toLowerCase();
     }
-    if( exePath == null || isWhitespace(exePath) ) {
+    if( exePath == null || sourcelib.kv.isWhitespace(exePath) ) {
         vscode.window.showErrorMessage(`${settings.compilerName} path is empty. Please configure!`);
         return;
     }

src/compiler/compiler-base.ts

@@ -1,6 +1,6 @@
 import vscode from "vscode";
 import KvDocument from "./KvDocument";
-import {  populateColorTagMatches, ClrTagInfo } from "@sourcelib/captions";
+import * as sourcelib from "sourcelib";
 
 export class CaptionsColorsProvider implements vscode.DocumentColorProvider {
 
@@ -22,7 +22,7 @@ export class CaptionsColorsProvider implements vscode.DocumentColorProvider {
             if (kv == null || kv.values.length === 0)
                 continue;
 
-            const clrInfo: ClrTagInfo[] = populateColorTagMatches(kv.value.content);
+            const clrInfo: sourcelib.captions.ClrTagInfo[] = sourcelib.captions.populateColorTagMatches(kv.value.content);
             clrInfo.forEach((clr) => {
                 const colorInfo = new vscode.ColorInformation(kv.value.range.with(kv.value.range.start.translate(0, clr.start), kv.value.range.start.translate(0, clr.end)),
                     new vscode.Color(clr.color.r / 255, clr.color.g / 255, clr.color.b / 255, 1.0));

src/language/CaptionsColorsProvider.ts

@@ -1,23 +1,23 @@
 import * as vscode from "vscode";
 import { KvPair, KvPiece } from "../Kv";
-import { tokenize, TokenList, Token, TokenType } from "@sourcelib/kv";
+import * as sourcelib from "sourcelib";
 import { KvTokensProviderBase } from "./KvTokensProviderBase";
 
 export default class KvDocument {
 
     protected _document: vscode.TextDocument;
-    protected _tokens: TokenList;
+    protected _tokens: sourcelib.kv.TokenList;
 
     public get document(): vscode.TextDocument {
         return this._document;
     }
 
-    public get tokens(): TokenList {
+    public get tokens(): sourcelib.kv.TokenList {
         return this._tokens;
     }
 
     public static from(document: vscode.TextDocument): KvDocument {
-        return new KvDocument(document, tokenize(document.getText()));
+        return new KvDocument(document, sourcelib.kv.tokenize(document.getText()));
     }
 
     public static tokenLegend = new vscode.SemanticTokensLegend([
@@ -36,7 +36,7 @@ export default class KvDocument {
         "readonly"
     ]);
 
-    private constructor(document: vscode.TextDocument, tks: TokenList) {
+    private constructor(document: vscode.TextDocument, tks: sourcelib.kv.TokenList) {
         this._document = document;
         this._tokens = tks;
     }
@@ -56,9 +56,9 @@ export default class KvDocument {
         const valuePieces: KvPiece[] = [];
         for (const token of tokens) {
             switch (token.type) {
-            case TokenType.Key:
+            case sourcelib.kv.TokenType.Key:
                 keyPiece = this.getUnquotedToken(token); break;
-            case TokenType.Value:
+            case sourcelib.kv.TokenType.Value:
                 valuePieces.push(this.getUnquotedToken(token)); break;
             }
         }
@@ -69,13 +69,13 @@ export default class KvDocument {
         return new KvPair(keyPiece, valuePieces);
     }
 
-    public getTokenRange(token: Token): vscode.Range {
+    public getTokenRange(token: sourcelib.kv.Token): vscode.Range {
         const start = new vscode.Position(token.line, token.range.getStart());
         const end = new vscode.Position(token.line, token.range.getEnd());
         return new vscode.Range(start, end);
     }
 
-    private getUnquotedToken(token: Token): KvPiece {
+    private getUnquotedToken(token: sourcelib.kv.Token): KvPiece {
         const range = this.getTokenRange(token);
         return KvTokensProviderBase.unquoteToken(token, range);
     }

src/language/KvDocument.ts

@@ -2,7 +2,6 @@
 
 /*
 import vscode from "vscode";
-import {  } from "@sourcelib/kv";
 import KvDocument from "./KvDocument";
 import * as main from "../main";
 

src/language/KvFormatter.ts

@@ -1,6 +1,6 @@
 import * as vscode from "vscode";
+import * as sourcelib from "sourcelib";
 import { KvPiece } from "../Kv";
-import { isQuoted, stripQuotes, Token, TokenList, TokenType } from "@sourcelib/kv";
 import KvDocument from "./KvDocument";
 import { KvSemanticProcessor, KvSemanticProcessorParams } from "./KvSemanticProcessor";
 
@@ -18,14 +18,14 @@ export abstract class KvTokensProviderBase implements vscode.DocumentSemanticTok
     diagnostics: vscode.Diagnostic[] = [];
     bracketStack = 0;
 
-    protected tokens: TokenList;
+    protected tokens: sourcelib.kv.TokenList;
 
     protected abstract valueProcessors: KvSemanticProcessor[];
 
     protected abstract keyProcessors: KvSemanticProcessor[];
 
     constructor(legend: vscode.SemanticTokensLegend, diagnosticCollection: vscode.DiagnosticCollection) {
-        this.tokens = new TokenList;
+        this.tokens = new sourcelib.kv.TokenList;
         this.legend = legend;
         this.diagnosticCollection = diagnosticCollection;
     }
@@ -51,50 +51,50 @@ export abstract class KvTokensProviderBase implements vscode.DocumentSemanticTok
             const tokenRange = kvDoc.getTokenRange(token);
 
             // No further processing on comments
-            if (token.type === TokenType.Comment) {
+            if (token.type === sourcelib.kv.TokenType.Comment) {
                 tokensBuilder.push(tokenRange, "comment", []);
                 continue;
             }
 
-            if (token.type === TokenType.Key) {
+            if (token.type === sourcelib.kv.TokenType.Key) {
 
                 // Get next token that isn't a comment 
                 const interestingToken = this.getNextInterestingToken(kvDoc.tokens, i);
 
                 // We're an object key
-                if (interestingToken?.token.type === TokenType.ObjectStart) {
+                if (interestingToken?.token.type === sourcelib.kv.TokenType.ObjectStart) {
                     tokensBuilder.push(tokenRange, "struct", []);
                     this.bracketStack++;
-                    currentScope += "." + stripQuotes(token.value.toLowerCase());
+                    currentScope += "." + sourcelib.kv.stripQuotes(token.value.toLowerCase());
                     continue;
                 }
 
                 this.processKvKey(token, tokenRange, tokensBuilder, kvDoc, currentScope);
 
                 // Is this key not followed by a value?
                 const interestingTokenType = interestingToken?.token.type;
-                if (interestingTokenType !== TokenType.Value && interestingTokenType !== TokenType.Conditional) {
+                if (interestingTokenType !== sourcelib.kv.TokenType.Value && interestingTokenType !== sourcelib.kv.TokenType.Conditional) {
                     this.diagnostics.push(new vscode.Diagnostic(tokenRange, "Expecting value to this key", vscode.DiagnosticSeverity.Error));
                 }
                 continue;
             }
 
-            if (token.type === TokenType.Value) {
+            if (token.type === sourcelib.kv.TokenType.Value) {
                 this.processKvValue(token, tokenRange, tokensBuilder, kvDoc, currentScope);
                 continue;
             }            
 
-            if (token.type === TokenType.ObjectEnd) {
+            if (token.type === sourcelib.kv.TokenType.ObjectEnd) {
                 this.bracketStack--;
                 currentScope = currentScope.substring(0, currentScope.lastIndexOf("."));
                 continue;
             }
 
-            if (token.type === TokenType.PreprocessorKey) {
+            if (token.type === sourcelib.kv.TokenType.PreprocessorKey) {
 
                 // Get next token that isn't a comment 
                 const interestingToken = this.getNextInterestingToken(kvDoc.tokens, i);
-                if (interestingToken?.token.type === TokenType.Value) {
+                if (interestingToken?.token.type === sourcelib.kv.TokenType.Value) {
 
                     const nextTokenRange = kvDoc.getTokenRange(token);
 
@@ -108,7 +108,7 @@ export abstract class KvTokensProviderBase implements vscode.DocumentSemanticTok
                 continue;
             }
 
-            if(token.type === TokenType.Conditional) {
+            if(token.type === sourcelib.kv.TokenType.Conditional) {
                 tokensBuilder.push(tokenRange, "keyword", []);
             }
         }
@@ -123,13 +123,13 @@ export abstract class KvTokensProviderBase implements vscode.DocumentSemanticTok
         return tokensBuilder.build();
     }
 
-    getNextInterestingToken(tokens: TokenList, i: number): { token: Token; offset: number; } | null {
+    getNextInterestingToken(tokens: sourcelib.kv.TokenList, i: number): { token: sourcelib.kv.Token; offset: number; } | null {
         let n = 1;
         if(tokens.length - 1 < i + n) {
             return null;
         }
         let nextToken = tokens[i + n];
-        while (nextToken.type === TokenType.Comment) {
+        while (nextToken.type === sourcelib.kv.TokenType.Comment) {
             nextToken = tokens[i + n++];
             if (nextToken == null)
                 break;
@@ -138,18 +138,18 @@ export abstract class KvTokensProviderBase implements vscode.DocumentSemanticTok
         return { token: nextToken, offset: n };
     }
 
-    protected processKvKey(token: Token, range: vscode.Range, tokensBuilder: vscode.SemanticTokensBuilder, kvDocument: KvDocument, scope: string): void {
+    protected processKvKey(token: sourcelib.kv.Token, range: vscode.Range, tokensBuilder: vscode.SemanticTokensBuilder, kvDocument: KvDocument, scope: string): void {
         const processed = this.processString(token, range, tokensBuilder, this.keyProcessors, kvDocument, scope);
         if (!processed)
             tokensBuilder.push(range, "variable", ["declaration"]);
     }
-    protected processKvValue(token: Token, range: vscode.Range, tokensBuilder: vscode.SemanticTokensBuilder, kvDocument: KvDocument, scope: string): void {
+    protected processKvValue(token: sourcelib.kv.Token, range: vscode.Range, tokensBuilder: vscode.SemanticTokensBuilder, kvDocument: KvDocument, scope: string): void {
         const processed = this.processString(token, range, tokensBuilder, this.valueProcessors, kvDocument, scope);
         if (!processed)
             tokensBuilder.push(range, "string", []);
     }
 
-    processString(token: Token, range: vscode.Range, tokensBuilder: vscode.SemanticTokensBuilder, processors: KvSemanticProcessor[], kvDocument: KvDocument, scope: string): boolean {
+    processString(token: sourcelib.kv.Token, range: vscode.Range, tokensBuilder: vscode.SemanticTokensBuilder, processors: KvSemanticProcessor[], kvDocument: KvDocument, scope: string): boolean {
         const unquoted = KvTokensProviderBase.unquoteToken(token, range);
 
         const processed = processors.some(processor => {
@@ -170,15 +170,15 @@ export abstract class KvTokensProviderBase implements vscode.DocumentSemanticTok
         return processed;
     }
 
-    protected disallowDuplicate(scopedKey: string, depth: number, token: Token): boolean {
+    protected disallowDuplicate(scopedKey: string, depth: number, token: sourcelib.kv.Token): boolean {
         return false; // Duplicate keys are allowed by default.
     }
 
-    public static unquoteToken(token: Token, range: vscode.Range): KvPiece {
+    public static unquoteToken(token: sourcelib.kv.Token, range: vscode.Range): KvPiece {
         // Quote tokens
         let unquotedContent: string = token.value;
         let unquotedRange: vscode.Range = range;
-        if (isQuoted(token.value)) {
+        if (sourcelib.kv.isQuoted(token.value)) {
             unquotedContent = token.value.substring(1, token.value.length - 1);
             unquotedRange = new vscode.Range(range.start.translate(0, 1), range.end.translate(0, -1));
         }

src/language/KvTokensProviderBase.ts

@@ -1,17 +1,11 @@
-// ==========================================================================
-// Purpose:
-// Implementations of language utility providers for the keyvalues language.
-//
 // This is NOT a base for other formats!
-// ==========================================================================
 
 import * as shared from "./Shared";
 import * as vscode from "vscode";
 import { KvTokensProviderBase } from "./KvTokensProviderBase";
-//import { KvDocumentFormatter } from "./KvFormatter";
 import { KvSemanticProcessor, KvSemanticProcessorParams } from "./KvSemanticProcessor";
 import KvDocument from "./KvDocument";
-import { matrixRegExp } from "@sourcelib/vmt";
+import * as sourcelib from "sourcelib";
 
 export const selectorAll: ReadonlyArray<vscode.DocumentFilter> = [ shared.filterKvSaved, 
     shared.filterKvUnsaved, 
@@ -32,7 +26,7 @@ export class KeyvalueSemanticTokensProvider extends KvTokensProviderBase {
     protected valueProcessors: KvSemanticProcessor[] =
         [
             { regex: /^-?\d+(\.\d+)?$/, processor: this.processValueNumber },
-            { regex: matrixRegExp, processor: this.processValueArray }
+            { regex: sourcelib.kv.matrixRegExp, processor: this.processValueArray }
         ];
 
     constructor() {