Replace utf8 package with using TextDecoder API (#466)

This fixes the assumption that every GDB on every host supports
and starts up with UTF-8 host charset. This caused errors if this is
not the case.

* Replace use of utf8 package with TextDecoder API.
* Detect and use host charset encoding and apply
  it to octal char sequences. Updates with each charset change.
* Synchronize changed charsets between MAIN and AUX GDB
  backends.
* Add fixup map for encodings not known to whatwg standard,
  e.g. Windows CP65001 which effectively is utf-8.
* Explicitly set charset to UTF-8 for Windows unicode path test,
we may have a machine/GDB combination which uses CP1252 if 'auto'.

---------

Signed-off-by: Jens Reinecke <jens.reinecke@arm.com>

Author: jreineckearm

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Change Log
 
+## Unreleased
+
+- Fixes [`465`](https://github.com/eclipse-cdt-cloud/cdt-gdb-adapter/issues/465): UTF-8 'Failed to decode cstring' errors for GDB with CP1252 support only.
+
 ## 1.4.1
 
 - Fixes [`400`](https://github.com/eclipse-cdt-cloud/cdt-gdb-adapter/issues/400): Evaluation of variables to support RTOS Views extension.

package.json

@@ -64,8 +64,7 @@
     "@vscode/debugadapter": "^1.68.0",
     "@vscode/debugprotocol": "^1.68.0",
     "node-addon-api": "^8.4.0",
-    "serialport": "^13.0.0",
-    "utf8": "^3.0.0"
+    "serialport": "^13.0.0"
   },
   "devDependencies": {
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
@@ -76,7 +75,6 @@
     "@types/serialport": "^10.2.0",
     "@types/sinon": "^17.0.4",
     "@types/tmp": "^0.2.6",
-    "@types/utf8": "^3.0.3",
     "@typescript-eslint/eslint-plugin": "^8.35.1",
     "@typescript-eslint/parser": "^8.35.1",
     "@vscode/debugadapter-testsupport": "^1.68.0",

src/MIParser.ts

@@ -9,7 +9,6 @@
  *********************************************************************/
 import { Readable } from 'stream';
 import { IGDBBackend } from './types/gdb';
-import * as utf8 from 'utf8';
 import { NamedLogger } from './namedLogger';
 
 interface Command {
@@ -21,8 +20,22 @@ type CommandQueue = {
     [key: string]: Command;
 };
 
+const DEFAULT_CHARSET = 'utf-8';
+/**
+ * Fixups for non-whatwg charset encodings in GDB known to effectively being
+ * aliases for whatwg encodings.
+ */
+const FIXUPS_CHARSET = new Map<string, string>([['cp65001', 'utf-8']]);
+
 export class MIParser {
-    protected logger;
+    /**
+     * Encoding for GDB/MI cstring characters outside standard ASCII range.
+     * Value has to match names known to TextDecoder API, see also
+     * whatwg encodings (https://encoding.spec.whatwg.org/).
+     * Defaults to 'utf-8' to match previous behavior.
+     */
+    protected _hostCharset = DEFAULT_CHARSET;
+    protected logger: NamedLogger;
     protected line = '';
     protected pos = 0;
 
@@ -36,6 +49,19 @@ export class MIParser {
         this.logger = new NamedLogger(name);
     }
 
+    public set hostCharset(value: string | undefined) {
+        // Empty string not included in map, returns undefined.
+        const fixup = FIXUPS_CHARSET.get(value ?? '');
+        this._hostCharset = fixup ?? value ?? DEFAULT_CHARSET;
+        this.logger.verbose(
+            `MIParser: Decoding GDB host character set as '${this._hostCharset}'`
+        );
+    }
+
+    public get hostCharset(): string | undefined {
+        return this._hostCharset;
+    }
+
     public cancelQueue() {
         // Entries in the form of [key, callback]
         const entries = Object.entries(this.commandQueue);
@@ -120,6 +146,21 @@ export class MIParser {
         return token;
     }
 
+    protected decodeCStringBytes(encodedBytes: number[]): string {
+        const buffer = Buffer.from(encodedBytes);
+        try {
+            return new TextDecoder(this._hostCharset, { fatal: true }).decode(
+                buffer
+            );
+        } catch (err) {
+            this.logger.error(
+                `Failed to decode code points (${this._hostCharset}) '${encodedBytes}'. ${JSON.stringify(err)}`
+            );
+        }
+        // Return something even if garbage
+        return String.fromCodePoint(...encodedBytes);
+    }
+
     protected handleCString() {
         let c = this.next();
         if (!c || c !== '"') {
@@ -128,21 +169,33 @@ export class MIParser {
 
         let cstring = '';
         let octal = '';
+        let encodedBytes = [];
         mainloop: for (c = this.next(); c; c = this.next()) {
             if (octal) {
                 octal += c;
                 if (octal.length == 3) {
-                    cstring += String.fromCodePoint(parseInt(octal, 8));
+                    // Octal sequence complete, save code point
+                    encodedBytes.push(parseInt(octal, 8));
                     octal = '';
                 }
                 continue;
             }
+            // End of octal sequence, convert accumulated code points
+            if (c !== '\\' && encodedBytes.length) {
+                cstring += this.decodeCStringBytes(encodedBytes);
+                encodedBytes = [];
+            }
             switch (c) {
                 case '"':
                     break mainloop;
                 case '\\':
                     c = this.next();
                     if (c) {
+                        // End of octal sequence, convert and add accumulated code points
+                        if (encodedBytes.length && (c < '0' || c > '7')) {
+                            cstring += this.decodeCStringBytes(encodedBytes);
+                            encodedBytes = [];
+                        }
                         switch (c) {
                             case 'n':
                                 cstring += '\n';
@@ -173,15 +226,12 @@ export class MIParser {
                     cstring += c;
             }
         }
-
-        try {
-            return utf8.decode(cstring);
-        } catch (err) {
-            this.logger.error(
-                `Failed to decode cstring '${cstring}'. ${JSON.stringify(err)}`
-            );
-            return cstring;
+        // Remaining code points, convert and add accumulated code points
+        if (encodedBytes.length) {
+            cstring += this.decodeCStringBytes(encodedBytes);
+            encodedBytes = [];
         }
+        return cstring;
     }
 
     protected handleString() {

src/constants/gdb.ts

@@ -48,3 +48,15 @@ export const RESUME_COMMANDS = [
     'until',
     'u',
 ];
+
+export const SET_HOSTCHARSET_REGEXPS = [
+    /-gdb-set\s+(host-charset|charset)\s+.*/,
+    /-interpreter-exec\s+console\s+"set\s+(host-charset|charset)\s+.*/,
+    /set\s+(host-charset|charset)\s+.*/,
+];
+
+export const SET_ALL_CHARSET_REGEXPS = [
+    /-gdb-set\s+(charset|host-charset|target-charset|target-wide-charset)\s+.*/,
+    /-interpreter-exec\s+console\s+"set\s+(charset|host-charset|target-charset|target-wide-charset)\s+.*/,
+    /set\s+(charset|host-charset|target-charset|target-wide-charset)\s+.*/,
+];

src/gdb/GDBBackend.ts

@@ -22,12 +22,18 @@ import {
     sendExecInterrupt,
 } from '../mi';
 import { VarManager } from '../varManager';
+import { SET_HOSTCHARSET_REGEXPS } from '../constants/gdb';
 import { IGDBBackend, IGDBProcessManager, IStdioProcess } from '../types/gdb';
 import { MIParser } from '../MIParser';
 import { compareVersions } from '../util/compareVersions';
 import { isProcessActive } from '../util/processes';
 import { NamedLogger } from '../namedLogger';
 
+// Expected console output for interpreter command 'show host-charset'
+// if setting is 'auto'.
+const HOST_CHARSET_REGEXP =
+    /The host character set is \"auto; currently (.+)\"./;
+
 type WriteCallback = (error: Error | null | undefined) => void;
 
 export class GDBBackend extends events.EventEmitter implements IGDBBackend {
@@ -57,6 +63,90 @@ export class GDBBackend extends events.EventEmitter implements IGDBBackend {
         return this.varMgr;
     }
 
+    /**
+     * Send interpreter command to show host charset and wait until/resolve when
+     * stdout contains expected line or timeout. Don't throw, but return undefined
+     * in case of error/timeout.
+     */
+    private async getAutoHostCharsetFromConsole(): Promise<string | undefined> {
+        return new Promise<string | undefined>(async (resolve) => {
+            let resolved = false;
+            let charset: string | undefined;
+
+            // Call when promise is read to resolve,
+            // cleans up lister and timeout and calls resolve.
+            const done = () => {
+                if (!resolved) {
+                    resolved = true;
+                    this.off('consoleStreamOutput', logListener);
+                    clearTimeout(timeout);
+                    resolve(charset);
+                }
+            };
+
+            // Temporary listener looking out for console output
+            const logListener = (output: string, category: string) => {
+                if (category !== 'stdout') {
+                    return; // Expected output only on stdout
+                }
+                const match = HOST_CHARSET_REGEXP.exec(output);
+                if (!match) {
+                    return; // No match, continue waiting
+                }
+                // Match, call it done even if no valid charset.
+                // Use lower case encoding name as defined used TextDecoder.
+                charset = match[1]?.toLowerCase();
+                done();
+            };
+
+            this.on('consoleStreamOutput', logListener);
+            // Timeout to avoid lockup if something's wrong or stdout is missing.
+            const timeout = setTimeout(() => {
+                this.logger.error(
+                    'Error detecting host character set from stdout: timeout'
+                );
+                done();
+            }, 500);
+
+            try {
+                await this.sendCommand(
+                    '-interpreter-exec console "show host-charset"'
+                );
+            } catch (error) {
+                // Command failed
+                this.logger.error(
+                    `Error detecting host character set from stdout: ${error}`
+                );
+                done();
+            }
+        });
+    }
+
+    /**
+     * Get host character set encoding. Try MI -gdb-show first, then interpreter
+     * console if 'auto' as this is the only found way to get the actual encoding.
+     */
+    private async getHostCharset(): Promise<string | undefined> {
+        try {
+            const charsetResponse = await this.sendGDBShow('host-charset');
+            // Use lower case as defined for later used TextDecoder
+            const charset = charsetResponse?.value?.toLowerCase();
+            if (charset !== 'auto') {
+                // undefined or value other than 'auto'
+                return charset;
+            }
+        } catch (error) {
+            this.logger.error(`Error getting GDB host-charset: ${error}`);
+            return undefined;
+        }
+        // 'auto' detected, get actual charset through interpreter console
+        return await this.getAutoHostCharsetFromConsole();
+    }
+
+    private async updateCStringDecoder(): Promise<void> {
+        this.parser.hostCharset = await this.getHostCharset();
+    }
+
     public async spawn(
         requestArgs: LaunchRequestArguments | AttachRequestArguments
     ) {
@@ -92,6 +182,7 @@ export class GDBBackend extends events.EventEmitter implements IGDBBackend {
         this.asyncRequestedExplicitly = !!(
             requestArgs.gdbAsync || requestArgs.gdbNonStop
         );
+        await this.updateCStringDecoder();
         await this.setNonStopMode(requestArgs.gdbNonStop);
         await this.setAsyncMode(requestArgs.gdbAsync);
     }
@@ -216,10 +307,21 @@ export class GDBBackend extends events.EventEmitter implements IGDBBackend {
         }
     }
 
-    public sendCommand<T>(command: string): Promise<T> {
+    /**
+     * Post-processes a GDB command.
+     * @param expression Command to be executed
+     */
+    private async postProcessCommand(expression: string): Promise<void> {
+        if (SET_HOSTCHARSET_REGEXPS.some((regex) => regex.test(expression))) {
+            // Update GDB host charset info
+            await this.updateCStringDecoder();
+        }
+    }
+
+    public async sendCommand<T>(command: string): Promise<T> {
         const token = this.nextToken();
         this.logger.verbose(`GDB command: ${token} ${command}`);
-        return new Promise<T>((resolve, reject) => {
+        const result = await new Promise<T>((resolve, reject) => {
             if (this.out) {
                 /* Set error to capture the stack where the request originated,
                    not the stack of reading the stream and parsing the message.
@@ -275,6 +377,9 @@ export class GDBBackend extends events.EventEmitter implements IGDBBackend {
                 reject(new Error('gdb is not running.'));
             }
         });
+        // Post-process command after successful execution
+        await this.postProcessCommand(command);
+        return result;
     }
 
     public sendEnablePrettyPrint() {