fix(appsec): use per-part charset for files_content in commons-fileupload

Author: jandro996

dd-java-agent/instrumentation/commons-fileupload-1.5/src/main/java/datadog/trace/instrumentation/commons/fileupload/FileItemContentReader.java

@@ -2,9 +2,11 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Locale;
 import org.apache.commons.fileupload.FileItem;
 
 /** Reads uploaded file content for WAF inspection. */
@@ -35,11 +37,33 @@ public static String readContent(FileItem fileItem) {
           && (n = is.read(buf, total, MAX_CONTENT_BYTES - total)) != -1) {
         total += n;
       }
-      return new String(buf, 0, total, StandardCharsets.ISO_8859_1);
+      return decodeBytes(buf, total, fileItem.getContentType());
     } catch (IOException ignored) {
       return "";
     }
   }
 
+  static String decodeBytes(byte[] buf, int length, String contentType) {
+    Charset charset = extractCharset(contentType);
+    if (charset == null) charset = StandardCharsets.UTF_8;
+    try {
+      return new String(buf, 0, length, charset);
+    } catch (Exception e) {
+      return new String(buf, 0, length, StandardCharsets.ISO_8859_1);
+    }
+  }
+
+  static Charset extractCharset(String contentType) {
+    if (contentType == null) return null;
+    int idx = contentType.toLowerCase(Locale.ROOT).indexOf("charset=");
+    if (idx < 0) return null;
+    String name = contentType.substring(idx + 8).split("[;, ]")[0].trim();
+    try {
+      return Charset.forName(name);
+    } catch (IllegalArgumentException e) {
+      return null;
+    }
+  }
+
   private FileItemContentReader() {}
 }

dd-java-agent/instrumentation/commons-fileupload-1.5/src/test/groovy/FileItemContentReaderTest.groovy

@@ -39,6 +39,33 @@ class FileItemContentReaderTest extends Specification {
     FileItemContentReader.readContent(item) == ''
   }
 
+  void 'readContent decodes UTF-8 when Content-Type specifies charset=UTF-8'() {
+    given:
+    def text = 'héllo wörld'
+    def item = fileItemFromBytes(text.getBytes('UTF-8'), 'file.txt', 'text/plain; charset=UTF-8')
+
+    expect:
+    FileItemContentReader.readContent(item) == text
+  }
+
+  void 'readContent falls back to UTF-8 when Content-Type has no charset'() {
+    given:
+    def text = 'hello world'
+    def item = fileItemFromBytes(text.getBytes('UTF-8'), 'file.txt', 'text/plain')
+
+    expect:
+    FileItemContentReader.readContent(item) == text
+  }
+
+  void 'readContent falls back to UTF-8 when Content-Type is null'() {
+    given:
+    def text = 'hello world'
+    def item = fileItemFromBytes(text.getBytes('UTF-8'), 'file.txt', null)
+
+    expect:
+    FileItemContentReader.readContent(item) == text
+  }
+
   void 'readContents returns content for each non-form file with a name'() {
     given:
     def items = [fileItem('content-a', 'file-a.txt'), fileItem('content-b', 'file-b.txt'),]
@@ -96,15 +123,54 @@ class FileItemContentReaderTest extends Specification {
     FileItemContentReader.readContents([]) == []
   }
 
+  void 'extractCharset returns null for null contentType'() {
+    expect:
+    FileItemContentReader.extractCharset(null) == null
+  }
+
+  void 'extractCharset returns null for contentType without charset'() {
+    expect:
+    FileItemContentReader.extractCharset('text/plain') == null
+    FileItemContentReader.extractCharset('image/jpeg') == null
+    FileItemContentReader.extractCharset('application/octet-stream') == null
+  }
+
+  void 'extractCharset returns null for invalid charset name'() {
+    expect:
+    FileItemContentReader.extractCharset('text/plain; charset=NOTACHARSET') == null
+  }
+
+  void 'extractCharset extracts charset case-insensitively'() {
+    expect:
+    FileItemContentReader.extractCharset('text/plain; CHARSET=UTF-8').name() == 'UTF-8'
+    FileItemContentReader.extractCharset('text/plain; Charset=UTF-8').name() == 'UTF-8'
+    FileItemContentReader.extractCharset('text/plain; charset=utf-8').name() == 'UTF-8'
+  }
+
+  void 'extractCharset extracts charset from standard Content-Type'() {
+    expect:
+    FileItemContentReader.extractCharset('text/plain; charset=UTF-8').name() == 'UTF-8'
+    FileItemContentReader.extractCharset('text/xml; charset=ISO-8859-1').name() == 'ISO-8859-1'
+  }
+
   private FileItem fileItem(String content) {
     fileItem(content, 'file.txt')
   }
 
   private FileItem fileItem(String content, String name) {
+    fileItem(content, name, null)
+  }
+
+  private FileItem fileItem(String content, String name, String contentType) {
+    fileItemFromBytes((content ?: '').getBytes('ISO-8859-1'), name, contentType)
+  }
+
+  private FileItem fileItemFromBytes(byte[] bytes, String name, String contentType) {
     FileItem item = Stub(FileItem)
     item.isFormField() >> false
     item.getName() >> name
-    item.getInputStream() >> new ByteArrayInputStream((content ?: '').getBytes('ISO-8859-1'))
+    item.getContentType() >> contentType
+    item.getInputStream() >> new ByteArrayInputStream(bytes)
     return item
   }
 }