fix(transport): decode BOM-less utf-16 response bodies as big-endian

When a response declares Content-Type charset=utf-16 with no byte-order mark,
the SDKs disagreed on byte order — several decoded little-endian — so identical
bytes produced different strings across languages. These are HTTP clients
decoding by the IANA/MIME charset registration, and RFC 2781 specifies UTF-16
with no BOM defaults to big-endian.

Standardize on big-endian for a BOM-less utf-16 label across all transports
(csharp, go, python, ruby, rust, swift, node were little-endian; dart, java,
kotlin, php, elixir already big-endian). An explicit utf-16le/utf-16be label and
a present BOM are still honored. All 12 SDKs gain a parity test decoding the
BOM-less UTF-16BE bytes of "Pet" and asserting the result, plus that the
little-endian interpretation differs (so the big-endian choice is load-bearing).

Also drops a stray HTML entity (>) from the C# concurrent-redirect test's doc
comment, restoring CSharpFormattingSpec.

Author: mridang

src/main/resources/templates/csharp/default_api_client.mustache

@@ -1033,7 +1033,20 @@ public sealed class DefaultApiClient : IApiClient, IDisposable
             System.Net.Mime.ContentType media = new(contentType);
             if (!string.IsNullOrEmpty(media.CharSet))
             {
-                return Encoding.GetEncoding(media.CharSet);
+                string charset = media.CharSet.Trim();
+                /* A bare "utf-16" / "utf16" label (no endianness suffix, no BOM)
+                   must decode big-endian per RFC 2781: a UTF-16 stream lacking a
+                   BOM defaults to big-endian. .NET's Encoding.GetEncoding("utf-16")
+                   returns a little-endian UnicodeEncoding, which would disagree
+                   with the other SDKs, so resolve bare utf-16 to UTF-16BE here.
+                   Explicit "utf-16le" / "utf-16be" labels fall through to
+                   Encoding.GetEncoding so their declared byte order is honored. */
+                if (string.Equals(charset, "utf-16", StringComparison.OrdinalIgnoreCase)
+                    || string.Equals(charset, "utf16", StringComparison.OrdinalIgnoreCase))
+                {
+                    return new UnicodeEncoding(bigEndian: true, byteOrderMark: false);
+                }
+                return Encoding.GetEncoding(charset);
             }
         }
         catch (FormatException)

src/main/resources/templates/csharp/test/DefaultApiClientTest.mustache

@@ -700,7 +700,7 @@ public class DefaultApiClientTest
     }
 
     /// <summary>
-    /// Stateless redirect-chain handler: a request to /chain/{n} with n &gt; 0
+    /// Stateless redirect-chain handler: a request to /chain/{n} with n above 0
     /// returns a 302 pointing at /chain/{n-1}; /chain/0 returns 200 OK. The
     /// chain length lives entirely in the URL, so the handler holds no
     /// per-request state and can serve many concurrent chains of different

src/main/resources/templates/csharp/test/DefaultApiClientUnitTest.mustache

@@ -598,6 +598,31 @@ public class DefaultApiClientUnitTest
         Assert.Equal("héllo", response.Body);
     }
 
+    [Fact]
+    public async Task DecodesBomlessUtf16BodyAsBigEndian()
+    {
+        // RFC 2781: a UTF-16 stream with no BOM defaults to big-endian. A bare
+        // "charset=utf-16" (no endianness suffix, no BOM) must therefore decode
+        // big-endian, matching the other SDKs. .NET's Encoding.GetEncoding(
+        // "utf-16") is little-endian, so this would have decoded to garbage
+        // before the fix.
+        byte[] utf16BeBytes = { 0x00, 0x50, 0x00, 0x65, 0x00, 0x74 }; // "Pet" UTF-16BE
+        var handler = new RawByteHandler(
+            HttpStatusCode.OK,
+            utf16BeBytes,
+            "text/plain; charset=utf-16");
+        var client = new DefaultApiClient(new HttpClient(handler));
+        var response = await client.SendRequestAsync(
+            "GET", new Uri("http://example.com/utf16-no-bom"),
+            new Dictionary<string, string>(), null);
+        Assert.Equal("Pet", response.Body);
+
+        // Sanity check that the choice is load-bearing: interpreting the very
+        // same bytes little-endian yields a different (non-"Pet") string, so a
+        // passing assertion above can only mean big-endian decoding was used.
+        Assert.NotEqual("Pet", Encoding.Unicode.GetString(utf16BeBytes));
+    }
+
     // ---- 3.1: SensitiveHeaderNames default contents ----
 
     [Fact]

src/main/resources/templates/dart/test/default_api_client_unit_test.mustache

@@ -843,6 +843,47 @@ void main() {
       }
     });
 
+    /* utf16-no-bom-big-endian: a charset=utf-16 response body WITHOUT a
+     * byte-order mark must default to big-endian (RFC 2781), uniformly
+     * with the other 11 SDKs. The bytes 00 50 00 65 00 74 are "Pet" when
+     * read big-endian; read little-endian they would decode to U+5000
+     * U+6500 U+7400 instead, so a correct decode proves the byte order. */
+    test('decodes a BOM-less charset=utf-16 body as big-endian', () async {
+      final server = await HttpServer.bind(InternetAddress.loopbackIPv4, 0);
+      // UTF-16BE for "Pet", no BOM.
+      final payload = <int>[
+        0x00, 0x50, // P
+        0x00, 0x65, // e
+        0x00, 0x74, // t
+      ];
+      server.listen((request) {
+        request.response
+          ..statusCode = 200
+          ..headers.set('Content-Type', 'text/plain; charset=utf-16')
+          ..add(payload)
+          ..close();
+      });
+
+      try {
+        final client = DefaultApiClient();
+        final resp = await client.sendRequest(
+          'GET',
+          'http://127.0.0.1:${server.port}/utf16-nobom',
+          {},
+          null,
+        );
+        expect(resp.statusCode, equals(200));
+        expect(resp.body, equals('Pet'));
+        // Prove the big-endian choice: the same bytes read little-endian
+        // would not yield "Pet".
+        expect(resp.body, isNot(equals(String.fromCharCodes(
+          <int>[0x5000, 0x6500, 0x7400],
+        ))));
+      } finally {
+        await server.close();
+      }
+    });
+
     /* dart-charset-utf16-mojibake (lenient fallback): an unrecognised
      * charset with non-UTF-8 bytes must fall back to a lossy UTF-8
      * decode rather than throwing a FormatException. */

src/main/resources/templates/elixir/test/base_api_test.mustache

@@ -770,6 +770,32 @@ defmodule {{moduleName}}.Api.BaseApiTest do
     assert result == bytes
   end
 
+  # WAVE F: a BOM-less utf-16 response body must decode as big-endian per
+  # RFC 2781. The bytes 00 50 00 65 00 74 are "Pet" in UTF-16BE; the same
+  # bytes read little-endian would be a different (CJK) string, so decoding
+  # to exactly "Pet" proves the big-endian default. Erlang's bare :utf16
+  # encoding atom is {:utf16, :big}, so this SDK already honours the rule.
+  test "decode_text_body decodes BOM-less utf-16 as big-endian (RFC 2781)" do
+    bytes = <<0x00, 0x50, 0x00, 0x65, 0x00, 0x74>>
+
+    result =
+      {{moduleName}}.DefaultApiClient.decode_text_body(
+        bytes,
+        "text/plain; charset=utf-16"
+      )
+
+    assert result == "Pet"
+
+    # Prove the choice: the same bytes read little-endian are NOT "Pet".
+    little =
+      {{moduleName}}.DefaultApiClient.decode_text_body(
+        bytes,
+        "text/plain; charset=utf-16le"
+      )
+
+    refute little == "Pet"
+  end
+
   test "empty JSON object body includes Content-Type" do
     {:ok, name} = CapturingHeadersApiClient.start()
     config = {{moduleName}}.Configuration.new(base_url: "http://localhost")

src/main/resources/templates/go/default_api_client.mustache

@@ -455,9 +455,10 @@ func decodeBodyByCharset(body []byte, contentType string) string {
 		return string(runes)
 	case "windows-1252", "cp1252":
 		return decodeWindows1252(body)
-	case "utf-16", "utf16", "utf-16le", "utf16le":
+	case "utf-16le", "utf16le":
 		return decodeUtf16(body, false)
-	case "utf-16be", "utf16be":
+	case "utf-16", "utf16", "utf-16be", "utf16be":
+		// A BOM-less utf-16 body defaults to big-endian per RFC 2781.
 		return decodeUtf16(body, true)
 	default:
 		return string(body)

src/main/resources/templates/go/test/default_api_client_unit_test.mustache

@@ -433,6 +433,34 @@ func TestDefaultApiClient_DecodesIso88591ResponseBody(t *testing.T) {
 	}
 }
 
+func TestDefaultApiClient_DecodesBomlessUtf16AsBigEndian(t *testing.T) {
+	t.Parallel()
+	/* A BOM-less utf-16 body must decode as big-endian per RFC 2781. The
+	 * bytes below are "Pet" in UTF-16BE with no byte-order mark; decoded
+	 * little-endian they would yield CJK code points, not "Pet". */
+	bigEndian := []byte{0x00, 0x50, 0x00, 0x65, 0x00, 0x74}
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "text/plain; charset=utf-16")
+		w.WriteHeader(200)
+		_, _ = w.Write(bigEndian)
+	}))
+	defer server.Close()
+
+	client := NewDefaultApiClient(nil)
+	resp, err := client.SendRequest("GET", server.URL+"/utf16-no-bom", map[string]string{}, nil)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if resp.Body != "Pet" {
+		t.Errorf("expected BOM-less utf-16 to decode big-endian to \"Pet\", got %q", resp.Body)
+	}
+	/* Prove the byte order matters: the same bytes read little-endian must
+	 * not equal "Pet", so a default of utf-16le would fail this test. */
+	if le := decodeUtf16(bigEndian, false); le == "Pet" {
+		t.Errorf("expected little-endian decode of big-endian bytes to differ from \"Pet\", got %q", le)
+	}
+}
+
 func TestDefaultApiClient_DefaultsToUtf8WhenCharsetMissing(t *testing.T) {
 	t.Parallel()
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

src/main/resources/templates/java/test/DefaultApiClientUnitTest.mustache

@@ -144,6 +144,18 @@ class DefaultApiClientUnitTest {
             os.write(body);
           }
         });
+    server.createContext(
+        "/utf16-no-bom",
+        exchange -> {
+          // RFC 2781: a UTF-16 body with NO byte-order mark defaults to
+          // big-endian. "Pet" as BOM-less UTF-16BE is 00 50 00 65 00 74.
+          byte[] body = new byte[] {0x00, 0x50, 0x00, 0x65, 0x00, 0x74};
+          exchange.getResponseHeaders().add("Content-Type", "text/plain; charset=utf-16");
+          exchange.sendResponseHeaders(200, body.length);
+          try (OutputStream os = exchange.getResponseBody()) {
+            os.write(body);
+          }
+        });
     server.createContext(
         "/content-encoding-lie",
         exchange -> {
@@ -838,6 +850,27 @@ class DefaultApiClientUnitTest {
     assertEquals("héllo", response.body());
   }
 
+  {{! -- WAVE F: a BOM-less utf-16 response body must decode as big-endian -- }}
+  {{! -- (RFC 2781). The bytes 00 50 00 65 00 74 are "Pet" in UTF-16BE; the -- }}
+  {{! -- same bytes read little-endian would be a different (CJK) string, so -- }}
+  {{! -- decoding to exactly "Pet" proves the big-endian default. Java's -- }}
+  {{! -- UTF-16 charset already honours this, so this guard is green here. -- }}
+
+  @Test
+  void decodesBomlessUtf16AsBigEndian() throws Exception {
+    DefaultApiClient client = new DefaultApiClient();
+    ApiHttpResponse response = client.sendRequest("GET", baseUrl + "/utf16-no-bom", Map.of(), null);
+    assertEquals(200, response.statusCode());
+    assertEquals("Pet", response.body(),
+        "a BOM-less utf-16 body must decode as big-endian per RFC 2781");
+    // Prove the choice: the same bytes read little-endian are NOT "Pet".
+    byte[] sameBytes = new byte[] {0x00, 0x50, 0x00, 0x65, 0x00, 0x74};
+    assertNotEquals(
+        "Pet",
+        new String(sameBytes, java.nio.charset.StandardCharsets.UTF_16LE),
+        "little-endian interpretation of the same bytes must NOT equal \"Pet\"");
+  }
+
   {{! -- Gap AL: Content-Encoding lie (server claims gzip, sends plaintext) -- }}
 
   @Test

src/main/resources/templates/kotlin/test/DefaultApiClientUnitTest.mustache

@@ -493,6 +493,40 @@ class DefaultApiClientUnitTest {
             assertEquals("é", response.body, "ISO-8859-1 body must decode using the declared charset")
         }
 
+        @Test
+        @DisplayName("decodes BOM-less utf-16 response body as big-endian (RFC 2781)")
+        fun decodesBomlessUtf16AsBigEndian() {
+            // A response declaring charset=utf-16 with NO byte-order mark must
+            // decode as UTF-16 BIG-ENDIAN per RFC 2781. The bytes 00 50 00 65
+            // 00 74 are "Pet" in UTF-16BE; the same bytes read little-endian
+            // would yield the wrong string, proving the byte-order choice.
+            val bigEndianNoBom = byteArrayOf(0x00, 0x50, 0x00, 0x65, 0x00, 0x74)
+            val engine = MockEngine { _ ->
+                respond(
+                    content = bigEndianNoBom,
+                    status = HttpStatusCode.OK,
+                    headers = headersOf("Content-Type", "text/plain; charset=utf-16"),
+                )
+            }
+            val apiClient = DefaultApiClient(HttpClient(engine) { followRedirects = false })
+            val response = runBlocking {
+                apiClient.sendRequest("GET", "http://localhost/utf16-no-bom", emptyMap(), null)
+            }
+            assertEquals(
+                "Pet",
+                response.body,
+                "BOM-less utf-16 must decode as big-endian",
+            )
+            // Same bytes interpreted little-endian must NOT equal "Pet",
+            // confirming the big-endian default is what produced the match.
+            val asLittleEndian = String(bigEndianNoBom, Charsets.UTF_16LE)
+            assertNotEquals(
+                "Pet",
+                asLittleEndian,
+                "little-endian interpretation of the same bytes must differ",
+            )
+        }
+
         @Test
         @DisplayName("defaults to UTF-8 when Content-Type has no charset")
         fun defaultsToUtf8WhenNoCharset() {

src/main/resources/templates/node/abstract_api_client.mustache

@@ -540,13 +540,23 @@ export abstract class AbstractApiClient implements ApiClient {
    * present, and falls back to UTF-8 if the declared charset is not
    * recognised by the runtime's `TextDecoder`.
    *
+   * A bare `utf-16`/`utf16` label (no explicit byte order) is decoded as
+   * big-endian. Node's `TextDecoder` maps the WHATWG `'utf-16'` label to
+   * little-endian, but for an HTTP client resolving the IANA/MIME charset
+   * `UTF-16`, RFC 2781 specifies big-endian as the default when no BOM is
+   * present. A leading BOM is still honoured by `TextDecoder` and overrides
+   * the byte order; explicit `utf-16le`/`utf-16be` labels are left untouched.
+   *
    * @param buf the raw response bytes
    * @param contentType the response Content-Type header value
    * @returns the decoded text
    */
   static decodeBody(buf: Buffer, contentType?: string): string {
     const m = /charset=([^;]+)/i.exec(contentType ?? '');
-    const cs = m ? m[1].trim().toLowerCase().replace(/^["']|["']$/g, '') : 'utf-8';
+    let cs = m ? m[1].trim().toLowerCase().replace(/^["']|["']$/g, '') : 'utf-8';
+    if (cs === 'utf-16' || cs === 'utf16') {
+      cs = 'utf-16be';
+    }
     try {
       return new TextDecoder(cs, { fatal: false }).decode(buf);
     } catch {