Remove syntactic colorization in favor of text mate colorization (#3821)

Author: Colengms

Documentation/LanguageServer/colorization.md

@@ -1,7 +1,7 @@
 
 # VS Code C/C++ Extension - Enhanced Colorization
 
-The VS Code C/C++ extension now supports lexical/syntactic and semantic colorization, when IntelliSense is enabled.  Enhanced colorization can be enabled using the enhancedColorization setting:
+The VS Code C/C++ extension now supports semantic colorization, when IntelliSense is enabled.  Enhanced colorization can be enabled using the enhancedColorization setting:
 
 ```
     "C_Cpp.enhancedColorization": "Enabled"
@@ -20,41 +20,32 @@ Colors are associated with [TextMate scopes](https://macromates.com/manual/en/la
 | Token         | Scope         |
 | ------------- |:-------------:|
 | Class Template | entity.name.class.template |
-| Comment | comment |
 | Enumerator | variable.other.enummember |
 | Event  (C++/CLI) | variable.other.event |
 | Function | entity.name.function |
 | Function Template | entity.name.function.template |
 | Generic Type (C++/CLI) | entity.name.class.generic |
 | Global Variable | variable.other.global |
 | Identifier | <span>entity.name</span> |
-| Keyword | keyword.control |
 | Label | entity.name.label |
 | Local Variable | variable.other.local |
 | Macro | entity.name.function.preprocessor |
 | Member Field  | variable.other.member |
 | Member Function | entity.name.function.member |
 | Member Operator | keyword.operator.member |
-| Namespace | entity.name.namespace |
+| Namespace | entity.name.type.namespace |
 | New / Delete | keyword.operator.new |
-| Number Literal | constant.numeric |
-| Operator | keyword.operator |
 | Operator Function | entity.name.function.operator |
 | Parameter | variable.parameter |
-| Preprocessor Keyword | keyword.control.directive |
 | Property (C++/CLI) | variable.other.property |
 | Reference Type (C++/CLI) | entity.name.class.reference |
 | Static Member Field | variable.other.member.static |
 | Static Member Function | entity.name.function.member.static |
-| String Literal | string.quoted |
 | Type | entity.name.type |
-| User-Defined Literal – Number | entity.name.user-defined-literal.number |
+| User-Defined Literal - Number | entity.name.user-defined-literal.number |
 | User-Defined Literal - Raw | entity.name.user-defined-literal |
 | User-Defined Literal - String | entity.name.user-defined-literal.string |
 | Value Type (C++/CLI) | entity.name.class.value |
-| Variable | variable |
-| Xml Doc Comment | comment.xml.doc |
-| Xml Doc Tag | comment.xml.doc.tag |
 
 Many of the tokens recognized by IntelliSense do not directly map to existing scopes in the VS Code's default C/C++ TextMate grammar, so are likely not colored by existing VS Code themes.
 
@@ -259,7 +250,7 @@ Use the following to augment the Visual Studio Dark theme to match what Visual S
                     }
                 },
                 {
-                    "scope": "entity.name.namespace",
+                    "scope": "entity.name.type.namespace",
                     "settings": {
                         "foreground": "#C8C8C8"
                     }
@@ -480,7 +471,7 @@ Use the following to augment the Visual Studio Light theme to match what Visual
                     }
                 },
                 {
-                    "scope": "entity.name.namespace",
+                    "scope": "entity.name.type.namespace",
                     "settings": {
                         "foreground": "#000000"
                     }

Extension/CHANGELOG.md

@@ -1,8 +1,21 @@
 # C/C++ for Visual Studio Code Change Log
 
+## Version 0.24.0-insiders2: June 25, 2019
+### Minor Changes
+* Change `C_Cpp.clang_format_path` to `machine` scope. [#3774](https://github.com/microsoft/vscode-cpptools/issues/3774)
+* Removed syntactic/lexical colorization from `enhancedColorization`.  [PR #3821](https://github.com/microsoft/vscode-cpptools/pull/3821)
+
+### Bug Fixes
+* Fix crash when tag parsing Objective-C code. [#3776](https://github.com/microsoft/vscode-cpptools/issues/3776)
+* Fix duplicate slashes getting added to `c_cpp_properties.json`. [PR #3778](https://github.com/microsoft/vscode-cpptools/pull/3778)
+* Fix incorrect "file already opened in editor" message on Linux/Mac. [#3786](https://github.com/microsoft/vscode-cpptools/issues/3786)
+* Fix colorization for themes with background colors equal to the editor background color. [#3780](https://github.com/microsoft/vscode-cpptools/issues/3780)
+* Improve performance of colorization. [#3781](https://github.com/microsoft/vscode-cpptools/issues/3781)
+* Fix regression crash on hover. [#3792](https://github.com/microsoft/vscode-cpptools/issues/3792)
+
 ## Version 0.24.0-insiders: June 14, 2019
 ### New Features
-* Syntactic/lexical and semantic colorization [PR #3651](https://github.com/microsoft/vscode-cpptools/pull/3651) [Documentation](https://github.com/microsoft/vscode-cpptools/blob/master/Documentation/LanguageServer/colorization.md)
+* Semantic colorization [Documentation](https://github.com/microsoft/vscode-cpptools/blob/master/Documentation/LanguageServer/colorization.md)  [#230](https://github.com/microsoft/vscode-cpptools/issues/230)
 * Add `Rescan Workspace` command. [microsoft/vscode-cpptools-api#11](https://github.com/microsoft/vscode-cpptools-api/issues/11)
 
 ### Minor Changes

Extension/package.json

@@ -146,7 +146,7 @@
         "C_Cpp.dimInactiveRegions": {
           "type": "boolean",
           "default": true,
-          "description": "Controls whether inactive preprocessor blocks are colored differently than active code. This setting is ignored by the Tag Parser engine.",
+          "description": "Controls whether inactive preprocessor blocks are colored differently than active code. This setting has no effect if IntelliSense is disabled or if using the Default High Contrast theme.",
           "scope": "resource"
         },
         "C_Cpp.inactiveRegionOpacity": {
@@ -504,7 +504,7 @@
             "Disabled"
           ],
           "default": "Disabled",
-          "description": "If enabled, code is colorized based on IntelliSense. This setting has no effect if IntelliSense is disabled.",
+          "description": "If enabled, code is colorized based on IntelliSense. This setting has no effect if IntelliSense is disabled or if using the Default High Contrast theme.",
           "scope": "resource"
         }
       }

Extension/src/LanguageServer/client.ts

@@ -82,12 +82,6 @@ interface OutputNotificationBody {
     output: string;
 }
 
-interface SyntacticColorizationRegionsParams {
-    uri: string;
-    regions: InputColorizationRegion[];
-    editVersion: number;
-}
-
 interface SemanticColorizationRegionsParams {
     uri: string;
     regions: InputColorizationRegion[];
@@ -159,12 +153,12 @@ interface DidChangeVisibleRangesParams {
     ranges: Range[];
 }
 
-interface SyntacticColorizationRegionsReceiptParams {
+interface SemanticColorizationRegionsReceiptParams {
     uri: string;
 }
 
-interface SemanticColorizationRegionsReceiptParams {
-    uri: string;
+interface ColorThemeChangedParams {
+    name: string;
 }
 
 // Requests
@@ -193,8 +187,8 @@ const CustomBrowseConfigurationNotification: NotificationType<CustomBrowseConfig
 const ClearCustomConfigurationsNotification: NotificationType<void, void> = new NotificationType<void, void>('cpptools/clearCustomConfigurations');
 const RescanFolderNotification: NotificationType<void, void> = new NotificationType<void, void>('cpptools/rescanFolder');
 const DidChangeVisibleRangesNotification: NotificationType<DidChangeVisibleRangesParams, void> = new NotificationType<DidChangeVisibleRangesParams, void>('cpptools/didChangeVisibleRanges');
-const SyntacticColorizationRegionsReceiptNotification: NotificationType<SyntacticColorizationRegionsReceiptParams, void> = new NotificationType<SyntacticColorizationRegionsReceiptParams, void>('cpptools/syntacticColorizationRegionsReceipt');
 const SemanticColorizationRegionsReceiptNotification: NotificationType<SemanticColorizationRegionsReceiptParams, void> = new NotificationType<SemanticColorizationRegionsReceiptParams, void>('cpptools/semanticColorizationRegionsReceipt');
+const ColorThemeChangedNotification: NotificationType<ColorThemeChangedParams, void> = new NotificationType<ColorThemeChangedParams, void>('cpptools/colorThemeChanged');
 
 // Notifications from the server
 const ReloadWindowNotification: NotificationType<void, void> = new NotificationType<void, void>('cpptools/reloadWindow');
@@ -204,7 +198,6 @@ const ReportTagParseStatusNotification: NotificationType<ReportStatusNotificatio
 const ReportStatusNotification: NotificationType<ReportStatusNotificationBody, void> = new NotificationType<ReportStatusNotificationBody, void>('cpptools/reportStatus');
 const DebugProtocolNotification: NotificationType<OutputNotificationBody, void> = new NotificationType<OutputNotificationBody, void>('cpptools/debugProtocol');
 const DebugLogNotification:  NotificationType<OutputNotificationBody, void> = new NotificationType<OutputNotificationBody, void>('cpptools/debugLog');
-const SyntacticColorizationRegionsNotification:  NotificationType<SyntacticColorizationRegionsParams, void> = new NotificationType<SyntacticColorizationRegionsParams, void>('cpptools/syntacticColorizationRegions');
 const SemanticColorizationRegionsNotification:  NotificationType<SemanticColorizationRegionsParams, void> = new NotificationType<SemanticColorizationRegionsParams, void>('cpptools/semanticColorizationRegions');
 const CompileCommandsPathsNotification:  NotificationType<CompileCommandsPaths, void> = new NotificationType<CompileCommandsPaths, void>('cpptools/compileCommandsPaths');
 const UpdateClangFormatPathNotification: NotificationType<string, void> = new NotificationType<string, void>('cpptools/updateClangFormatPath');
@@ -479,7 +472,6 @@ class DefaultClient implements Client {
                 autocomplete: settings.autoComplete,
                 errorSquiggles: settings.errorSquiggles,
                 dimInactiveRegions: settings.dimInactiveRegions,
-                textMateColorization: settings.textMateColorization,
                 enhancedColorization: settings.enhancedColorization,
                 suggestSnippets: settings.suggestSnippets,
                 loggingLevel: settings.loggingLevel,
@@ -528,9 +520,6 @@ class DefaultClient implements Client {
     }
 
     public onDidChangeSettings(event: vscode.ConfigurationChangeEvent): { [key: string] : string } {
-        if (event.affectsConfiguration("C_Cpp.textMateColorization", this.RootUri)) {
-            this.colorizationSettings.updateGrammars();
-        }
         let colorizationNeedsReload: boolean = event.affectsConfiguration("workbench.colorTheme")
             || event.affectsConfiguration("editor.tokenColorCustomizations");
 
@@ -541,6 +530,12 @@ class DefaultClient implements Client {
             || event.affectsConfiguration("C_Cpp.inactiveRegionForegroundColor", this.RootUri)
             || event.affectsConfiguration("C_Cpp.inactiveRegionBackgroundColor", this.RootUri);
 
+        let colorThemeChanged: boolean = event.affectsConfiguration("workbench.colorTheme", this.RootUri);
+        if (colorThemeChanged) {
+            let otherSettings: OtherSettings = new OtherSettings(this.RootUri);
+            this.languageClient.sendNotification(ColorThemeChangedNotification, { name: otherSettings.colorTheme } );
+        }
+
         if (colorizationNeedsReload) {
             this.colorizationSettings.reload();
         }
@@ -1000,7 +995,6 @@ class DefaultClient implements Client {
         this.languageClient.onNotification(ReportNavigationNotification, (e) => this.navigate(e));
         this.languageClient.onNotification(ReportStatusNotification, (e) => this.updateStatus(e));
         this.languageClient.onNotification(ReportTagParseStatusNotification, (e) => this.updateTagParseStatus(e));
-        this.languageClient.onNotification(SyntacticColorizationRegionsNotification, (e) => this.updateSyntacticColorizationRegions(e));
         this.languageClient.onNotification(SemanticColorizationRegionsNotification, (e) => this.updateSemanticColorizationRegions(e));
         this.languageClient.onNotification(CompileCommandsPathsNotification, (e) => this.promptCompileCommands(e));
         this.setupOutputHandlers();
@@ -1216,21 +1210,6 @@ class DefaultClient implements Client {
         return colorizationState;
     }
 
-    private updateSyntacticColorizationRegions(params: SyntacticColorizationRegionsParams): void {
-        // Convert the params to vscode.Range's before passing to colorizationState.updateSyntactic()
-        let syntacticRanges: vscode.Range[][] = new Array<vscode.Range[]>(TokenKind.Count);
-        for (let i: number = 0; i < TokenKind.Count; i++) {
-            syntacticRanges[i] = [];
-        }
-        params.regions.forEach(element => {
-            let newRange : vscode.Range = new vscode.Range(element.range.start.line, element.range.start.character, element.range.end.line, element.range.end.character);
-            syntacticRanges[element.kind].push(newRange);
-        });
-        let colorizationState: ColorizationState = this.getColorizationState(params.uri);
-        colorizationState.updateSyntactic(params.uri, syntacticRanges, params.editVersion);
-        this.languageClient.sendNotification(SyntacticColorizationRegionsReceiptNotification, { uri: params.uri });
-    }
-
     private updateSemanticColorizationRegions(params: SemanticColorizationRegionsParams): void {
         // Convert the params to vscode.Range's before passing to colorizationState.updateSemantic()
         let semanticRanges: vscode.Range[][] = new Array<vscode.Range[]>(TokenKind.Count);