support UTF-8 encoding in Query responses for Minecraft versions starting from 1.21.11

Author: KurtThiemann

README.md

@@ -16,7 +16,7 @@ Note that only servers with the `server.properties` option `enable-query` set to
 The main advantage of the Query protocol over the ping protocol ist that it returns the
 full list of players on the server, not just a small sample. It does, however, also have a few disadvantages, 
 like the fact that servers will return broken response packets if the MOTD (or any other string) [contains null bytes](https://bugs.mojang.com/browse/MC-221987)
-or [some other special characters](https://bugs.mojang.com/browse/MC-231035).  
+or [some other special characters](https://bugs.mojang.com/browse/MC-231035) in versions before 1.21.11.  
 This library makes an effort to interpret these broken response packets correctly, but it is not always possible to do so.
 
 ```js
@@ -34,6 +34,23 @@ and [`FullStatResponse`](src/Packet/Query/FullStatResponse.js) object respective
 
 Note that the query client needs to be closed manually, since it keeps its UDP socket open to reuse it for future queries.
 
+#### Query string encoding
+
+Since string encoding in query responses switched to UTF-8 in Minecraft 1.21.11, this library makes an effort to detect
+the correct encoding automatically.  
+- In full stat responses, this works pretty reliably based on the version field.
+- Basic stat responses do not include the server version, so the library has to make a guess based on the content of the string.
+If the whole string is valid UTF-8, it will be decoded as UTF-8. If it contains invalid UTF-8 sequences, it will be decoded as ISO-8859-1 instead.
+
+You can force the string encoding using the `useLegacyStringEncoding` option, where `true` will force ISO-8859-1 encoding and `false` will force UTF-8 encoding.
+By default, this option is set to `null`, which means that the library will try to detect the correct encoding automatically.
+
+```js
+let useLegacyStringEncoding = true; // true, false, or null
+let basic = await client.queryBasic('localhost', 25565, AbortSignal.timeout(5000), useLegacyStringEncoding);
+let full = await client.queryFull('localhost', 25565, AbortSignal.timeout(5000), useLegacyStringEncoding);
+```
+
 ### Java Edition Ping
 The [Server List Ping protocol](https://wiki.vg/Server_List_Ping) is what the Minecraft client uses to show the server status in the in-game server list.
 This protocol changed multiple times over the years, so you'd ideally want to know the version of the server you are pinging to use the correct protocol version.

package.json

@@ -1,7 +1,7 @@
 {
   "name": "craftping",
   "type": "module",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "main": "index.js",
   "repository": "github:aternosorg/craftping",
   "scripts": {

src/Packet/Query/BasicStatResponse.js

@@ -2,6 +2,17 @@ import ProtocolError from "../../Error/ProtocolError.js";
 import StatResponse from "./StatResponse.js";
 
 export default class BasicStatResponse extends StatResponse {
+    /** @type {?boolean} */ useLegacyEncoding;
+
+    /**
+     * @param {?boolean} useLegacyEncoding - Whether to use the legacy encoding method for strings.
+     * If null, it will be determined automatically.
+     */
+    constructor(useLegacyEncoding = null) {
+        super();
+        this.useLegacyEncoding = useLegacyEncoding;
+    }
+
     /**
      * @inheritDoc
      */
@@ -29,17 +40,17 @@ export default class BasicStatResponse extends StatResponse {
             [hostip, offset] = this.readStringNT(data, offset);
         } while (offset < data.length);
 
-        this.hostname = motd;
-        this.gametype = gametype;
-        this.map = map;
-        this.numplayers = parseInt(numplayers);
-        this.maxplayers = parseInt(maxplayers);
+        this.hostname = this.decodeString(motd, this.useLegacyEncoding);
+        this.gametype = this.decodeString(gametype, this.useLegacyEncoding);
+        this.map = this.decodeString(map, this.useLegacyEncoding);
+        this.numplayers = parseInt(this.decodeString(numplayers, this.useLegacyEncoding));
+        this.maxplayers = parseInt(this.decodeString(maxplayers, this.useLegacyEncoding));
         if (isNaN(this.numplayers) || isNaN(this.maxplayers)) {
             throw new ProtocolError("Player count is not a number");
         }
 
         this.hostport = hostport;
-        this.hostip = hostip;
+        this.hostip = this.decodeString(hostip, this.useLegacyEncoding);
         return this;
     }
 
@@ -59,13 +70,13 @@ export default class BasicStatResponse extends StatResponse {
         portBuffer.writeUInt16LE(hostport, 0);
 
         return Buffer.concat([
-            this.createStringNT(motd),
-            this.createStringNT(gametype),
-            this.createStringNT(map),
-            this.createStringNT(numplayers),
-            this.createStringNT(maxplayers),
+            this.createStringNT(motd, this.useLegacyEncoding ?? true),
+            this.createStringNT(gametype, this.useLegacyEncoding ?? true),
+            this.createStringNT(map, this.useLegacyEncoding ?? true),
+            this.createStringNT(numplayers, this.useLegacyEncoding ?? true),
+            this.createStringNT(maxplayers, this.useLegacyEncoding ?? true),
             portBuffer,
-            this.createStringNT(hostip)
+            this.createStringNT(hostip, this.useLegacyEncoding ?? true)
         ]);
     }
 }

src/Packet/Query/FullStatResponse.js

@@ -26,6 +26,16 @@ export default class FullStatResponse extends StatResponse {
     /** @type {string} */ version;
     /** @type {string} */ plugins;
     /** @type {string[]} */ players;
+    /** @type {?boolean} */ useLegacyEncoding;
+
+    /**
+     * @param {?boolean} useLegacyEncoding - Whether to use the legacy encoding method for strings.
+     * If null, it will be determined automatically.
+     */
+    constructor(useLegacyEncoding = null) {
+        super();
+        this.useLegacyEncoding = useLegacyEncoding;
+    }
 
     /**
      * @return {string[]}
@@ -101,7 +111,7 @@ export default class FullStatResponse extends StatResponse {
     /**
      * @param {Buffer} data
      * @param {number} offset
-     * @return {[Map<string, string>, number]}
+     * @return {[Map<string, Buffer>, number]}
      */
     readKeyValueSection(data, offset) {
         let values = new Map();
@@ -114,7 +124,7 @@ export default class FullStatResponse extends StatResponse {
 
             [value, offset] = this.readStringNTFollowedBy(data, offset, FullStatResponse.VALID_KEYS_WITH_PLAYERS);
 
-            values.set(key, value);
+            values.set(key.toString("ascii"), value);
         }
 
         return [values, offset];
@@ -123,7 +133,7 @@ export default class FullStatResponse extends StatResponse {
     /**
      * @param {Buffer} data
      * @param {number} offset
-     * @return {[string[], number]}
+     * @return {[Buffer[], number]}
      */
     readPlayers(data, offset) {
         let players = [];
@@ -140,6 +150,42 @@ export default class FullStatResponse extends StatResponse {
         return [players, offset];
     }
 
+    /**
+     * Minecraft switched to UTF-8 encoding in version 1.21.11
+     * @param {Buffer|string} version
+     * @return {boolean}
+     */
+    shouldUseLegacyStringEncoding(version) {
+        if (typeof version !== "string") {
+            version = version.toString("ascii");
+        }
+        let parts = version.split(".")
+            .map(s => parseInt(s))
+            .map(n => isNaN(n) ? 0 : n);
+
+        let modernVersion = [1, 21, 11];
+        return this.compareVersions(parts, modernVersion) < 0;
+    }
+
+    /**
+     *
+     * @param {number[]} a
+     * @param {number[]} b
+     * @return {number}
+     */
+    compareVersions(a, b) {
+        for (let i = 0; i < Math.max(a.length, b.length); i++) {
+            let partA = i < a.length ? a[i] : 0;
+            let partB = i < b.length ? b[i] : 0;
+            if (partA < partB) {
+                return -1;
+            } else if (partA > partB) {
+                return 1;
+            }
+        }
+        return 0;
+    }
+
     /**
      * @inheritDoc
      */
@@ -150,6 +196,12 @@ export default class FullStatResponse extends StatResponse {
         }
 
         let [values, offset] = this.readKeyValueSection(data, this.constructor.START_PADDING.length);
+        let legacyEncoding = null;
+        if (this.useLegacyEncoding !== null) {
+            legacyEncoding = this.useLegacyEncoding;
+        } else if (values.has('version')) {
+            legacyEncoding = this.shouldUseLegacyStringEncoding(values.get('version'));
+        }
 
         let playerPadding = Buffer.from(data.buffer, data.byteOffset + offset, this.constructor.PLAYERS_PADDING.length);
         if (!playerPadding.equals(this.constructor.PLAYERS_PADDING)) {
@@ -164,46 +216,48 @@ export default class FullStatResponse extends StatResponse {
             throw new Error("Unexpected data after full stat response");
         }
 
-        this.gametype = values.get('gametype');
-        this.game_id = values.get('game_id');
-        this.version = values.get('version');
-        this.plugins = values.get('plugins');
-        this.map = values.get('map');
-        this.numplayers = parseInt(values.get('numplayers'));
-        this.maxplayers = parseInt(values.get('maxplayers'));
+        this.gametype = this.decodeString(values.get('gametype'), legacyEncoding);
+        this.game_id = this.decodeString(values.get('game_id'), legacyEncoding);
+        this.version = this.decodeString(values.get('version'), legacyEncoding);
+        this.plugins = this.decodeString(values.get('plugins'), legacyEncoding);
+        this.map = this.decodeString(values.get('map'), legacyEncoding);
+        this.numplayers = parseInt(this.decodeString(values.get('numplayers'), legacyEncoding));
+        this.maxplayers = parseInt(this.decodeString(values.get('maxplayers'), legacyEncoding));
         if (isNaN(this.numplayers) || isNaN(this.maxplayers)) {
             throw new ProtocolError("Player count is not a number");
         }
-        this.hostport = parseInt(values.get('hostport'));
-        this.hostip = values.get('hostip');
-        this.hostname = values.get('hostname');
-        this.players = players;
+        this.hostport = parseInt(this.decodeString(values.get('hostport'), legacyEncoding));
+        this.hostip = this.decodeString(values.get('hostip'), legacyEncoding);
+        this.hostname = this.decodeString(values.get('hostname'), legacyEncoding);
+        this.players = players.map(p => this.decodeString(p, legacyEncoding));
 
         return this;
     }
 
     /**
      * @param {Map<string, string>} values
+     * @param {boolean} legacyEncoding
      * @return {Buffer}
      */
-    writeKeyValueSection(values) {
+    writeKeyValueSection(values, legacyEncoding) {
         let parts = [];
         for (let [key, value] of values) {
-            parts.push(this.createStringNT(key));
-            parts.push(this.createStringNT(value));
+            parts.push(this.createStringNT(key, legacyEncoding));
+            parts.push(this.createStringNT(value, legacyEncoding));
         }
         parts.push(Buffer.alloc(1));
         return Buffer.concat(parts);
     }
 
     /**
      * @param {string[]} players
+     * @param {boolean} legacyEncoding
      * @return {Buffer}
      */
-    writePlayers(players) {
+    writePlayers(players, legacyEncoding) {
         let parts = [];
         for (let player of players) {
-            parts.push(this.createStringNT(player));
+            parts.push(this.createStringNT(player, legacyEncoding));
         }
         parts.push(Buffer.alloc(1));
         return Buffer.concat(parts);
@@ -213,6 +267,13 @@ export default class FullStatResponse extends StatResponse {
      * @inheritDoc
      */
     writePayload() {
+        let legacyEncoding = true;
+        if (this.useLegacyEncoding !== null) {
+            legacyEncoding = this.useLegacyEncoding;
+        } else if (this.version) {
+            legacyEncoding = this.shouldUseLegacyStringEncoding(this.version);
+        }
+
         let values = new Map([
             ['hostname', this.hostname || ''],
             ['gametype', this.gametype || ''],
@@ -228,9 +289,9 @@ export default class FullStatResponse extends StatResponse {
 
         return Buffer.concat([
             this.constructor.START_PADDING,
-            this.writeKeyValueSection(values),
+            this.writeKeyValueSection(values, legacyEncoding),
             this.constructor.PLAYERS_PADDING,
-            this.writePlayers(this.players)
+            this.writePlayers(this.players, legacyEncoding)
         ]);
     }
 }