fix(appsec): gate FileChannel write callback on write-capable open options

FileChannel.open() with READ-only options was incorrectly triggering the
fileWritten callback, which could cause false positives in the zipslip
rule (dog-920-110) when a read-only channel open with a traversal path
coincided with a multipart zip upload in the same request.

Split beforeOpen into two overload-specific methods so the OpenOption
arguments can be inspected at the call site, mirroring the existing
pattern in beforeRandomAccessFileOpened.

Also fix a latent bug in AdviceGeneratorImpl: .sorted() without a
comparator on ArgumentSpecification (which does not implement Comparable)
would ClassCastException when an advice method captures a strict subset
of a target method's arguments. Fixed with Comparator.comparingInt.

Author: jandro996

dd-java-agent/instrumentation/java/java-io-1.8/src/main/java/datadog/trace/instrumentation/java/lang/FileChannelCallSite.java

@@ -2,7 +2,10 @@
 
 import datadog.trace.agent.tooling.csi.CallSite;
 import datadog.trace.api.appsec.RaspCallSites;
+import java.nio.file.OpenOption;
 import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Set;
 import javax.annotation.Nullable;
 
 @CallSite(
@@ -12,14 +15,61 @@ public class FileChannelCallSite {
 
   @CallSite.Before(
       "java.nio.channels.FileChannel java.nio.channels.FileChannel.open(java.nio.file.Path, java.nio.file.OpenOption[])")
+  public static void beforeOpenArray(
+      @CallSite.Argument(0) @Nullable final Path path,
+      @CallSite.Argument(1) @Nullable final OpenOption[] options) {
+    if (path != null) {
+      String pathStr = path.toString();
+      FileIORaspHelper.INSTANCE.beforeFileLoaded(pathStr);
+      if (hasWriteOption(options)) {
+        FileIORaspHelper.INSTANCE.beforeFileWritten(pathStr);
+      }
+    }
+  }
+
   @CallSite.Before(
       "java.nio.channels.FileChannel java.nio.channels.FileChannel.open(java.nio.file.Path, java.util.Set, java.nio.file.attribute.FileAttribute[])")
-  public static void beforeOpen(@CallSite.Argument(0) @Nullable final Path path) {
+  public static void beforeOpenSet(
+      @CallSite.Argument(0) @Nullable final Path path,
+      @CallSite.Argument(1) @Nullable final Set<? extends OpenOption> options) {
     if (path != null) {
-      // Fire both read and write callbacks: the WAF determines whether to block
-      // based on the full request context (e.g. zipslip requires body.filenames too).
-      FileIORaspHelper.INSTANCE.beforeFileLoaded(path.toString());
-      FileIORaspHelper.INSTANCE.beforeFileWritten(path.toString());
+      String pathStr = path.toString();
+      FileIORaspHelper.INSTANCE.beforeFileLoaded(pathStr);
+      if (hasWriteOption(options)) {
+        FileIORaspHelper.INSTANCE.beforeFileWritten(pathStr);
+      }
+    }
+  }
+
+  private static boolean hasWriteOption(@Nullable final OpenOption[] options) {
+    if (options == null) {
+      return false;
     }
+    for (OpenOption opt : options) {
+      if (isWriteOption(opt)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean hasWriteOption(@Nullable final Set<? extends OpenOption> options) {
+    if (options == null) {
+      return false;
+    }
+    for (OpenOption opt : options) {
+      if (isWriteOption(opt)) {
+        return true;
+      }
+    }
+    return false;
+  }
+
+  private static boolean isWriteOption(final OpenOption opt) {
+    return opt == StandardOpenOption.WRITE
+        || opt == StandardOpenOption.APPEND
+        || opt == StandardOpenOption.CREATE
+        || opt == StandardOpenOption.CREATE_NEW
+        || opt == StandardOpenOption.TRUNCATE_EXISTING;
   }
 }

dd-java-agent/instrumentation/java/java-io-1.8/src/test/groovy/datadog/trace/instrumentation/java/io/FileChannelCallSiteTest.groovy

@@ -7,7 +7,7 @@ import java.nio.file.StandardOpenOption
 
 class FileChannelCallSiteTest extends BaseIoRaspCallSiteTest {
 
-  void 'test RASP FileChannel.open read-only fires beforeFileLoaded'() {
+  void 'test RASP FileChannel.open read-only fires only beforeFileLoaded'() {
     setup:
     final helper = Mock(FileIORaspHelper)
     FileIORaspHelper.INSTANCE = helper
@@ -18,10 +18,10 @@ class FileChannelCallSiteTest extends BaseIoRaspCallSiteTest {
 
     then:
     1 * helper.beforeFileLoaded(path.toString())
-    1 * helper.beforeFileWritten(path.toString())
+    0 * helper.beforeFileWritten(_)
   }
 
-  void 'test RASP FileChannel.open write fires beforeFileWritten'() {
+  void 'test RASP FileChannel.open write fires both beforeFileLoaded and beforeFileWritten'() {
     setup:
     final helper = Mock(FileIORaspHelper)
     FileIORaspHelper.INSTANCE = helper
@@ -35,7 +35,7 @@ class FileChannelCallSiteTest extends BaseIoRaspCallSiteTest {
     1 * helper.beforeFileWritten(path.toString())
   }
 
-  void 'test RASP FileChannel.open with Set of options'() {
+  void 'test RASP FileChannel.open with Set READ-only fires only beforeFileLoaded'() {
     setup:
     final helper = Mock(FileIORaspHelper)
     FileIORaspHelper.INSTANCE = helper
@@ -45,8 +45,37 @@ class FileChannelCallSiteTest extends BaseIoRaspCallSiteTest {
     when:
     TestFileChannelSuite.openWithSet(path, options).close()
 
+    then:
+    1 * helper.beforeFileLoaded(path.toString())
+    0 * helper.beforeFileWritten(_)
+  }
+
+  void 'test RASP FileChannel.open with Set WRITE fires both callbacks'() {
+    setup:
+    final helper = Mock(FileIORaspHelper)
+    FileIORaspHelper.INSTANCE = helper
+    final path = temporaryFolder.resolve('test_rasp_fc_set_write.txt')
+    final options = EnumSet.of(StandardOpenOption.WRITE, StandardOpenOption.CREATE)
+
+    when:
+    TestFileChannelSuite.openWithSet(path, options).close()
+
     then:
     1 * helper.beforeFileLoaded(path.toString())
     1 * helper.beforeFileWritten(path.toString())
   }
+
+  void 'test RASP FileChannel.open with no options fires only beforeFileLoaded'() {
+    setup:
+    final helper = Mock(FileIORaspHelper)
+    FileIORaspHelper.INSTANCE = helper
+    final path = newFile('test_rasp_fc_default.txt').toPath()
+
+    when:
+    TestFileChannelSuite.openWithOptions(path).close()
+
+    then:
+    1 * helper.beforeFileLoaded(path.toString())
+    0 * helper.beforeFileWritten(_)
+  }
 }