CASSANDRA-21134: Enable background-write Direct I/O in cassandra_latest config

Add background_write_disk_access_mode: direct to cassandra_latest.yaml so
new-install/latest configurations exercise Direct I/O by default, and keep
the test latest config in sync (test/conf/latest_diff.yaml and the
DTEST_JVM_DTESTS_USE_LATEST block in InstanceConfig.java). The cassandra.yaml
default remains standard.

CQLTester.InMemory installs a global jimfs filesystem, which cannot do
Direct I/O (no FileStore.getBlockSize(), rejects ExtendedOpenOption.DIRECT),
so pin in-memory tests to buffered writes to keep the latest config green.

Author: samueldlightfoot

conf/cassandra_latest.yaml

@@ -700,6 +700,21 @@ commitlog_disk_access_mode: auto
 # - direct: use direct I/O for compaction reads, bypassing the OS page cache
 # compaction_read_disk_access_mode: auto
 
+# Set the disk access mode for writing compressed SSTables during background operations
+# (compaction, streaming, cleanup, repair, etc.). The allowed values are:
+# - standard: use buffered I/O (default)
+# - direct: use direct I/O, bypassing the OS page cache
+# Note: Only applies to compressed tables. Uncompressed tables always use buffered I/O.
+# Note: Memtable flushes always use buffered I/O regardless of this setting, as flushed
+# data benefits from page cache for subsequent reads.
+background_write_disk_access_mode: direct
+
+# Size of the in-memory staging buffer for Direct IO background writes. Trades off syscall
+# frequency against per-flush blocking latency on the compaction thread.
+# Aligned up to filesystem block size; auto-expands to fit a single compressed chunk + CRC
+# + one block when chunk_length exceeds this value.
+# direct_write_buffer_size: 1MiB
+
 # Compression to apply to SSTables as they flush for compressed tables.
 # Note that tables without compression enabled do not respect this flag.
 #

src/java/org/apache/cassandra/config/Config.java

@@ -1287,8 +1287,7 @@ public enum DiskAccessMode
         legacy,
 
         /**
-         * Direct-I/O is enabled for commitlog disk only.
-         * When adding support for direct IO, update {@link org.apache.cassandra.service.StartupChecks#checkKernelBug1057843}
+         * When adding support for Direct I/O, update {@link org.apache.cassandra.service.StartupChecks#checkKernelBug1057843}
          */
         direct
     }

test/conf/latest_diff.yaml

@@ -46,6 +46,8 @@ memtable_allocation_type: offheap_objects
 
 commitlog_disk_access_mode: auto
 
+background_write_disk_access_mode: direct
+
 trickle_fsync: true
 
 sstable:

test/distributed/org/apache/cassandra/distributed/impl/InstanceConfig.java

@@ -151,6 +151,8 @@ private InstanceConfig(int num,
 
                 .set("commitlog_disk_access_mode", "auto")
 
+                .set("background_write_disk_access_mode", "direct")
+
                 .set("trickle_fsync", "true")
 
                 .set("sstable", Map.of(

test/unit/org/apache/cassandra/cql3/CQLTester.java

@@ -3758,6 +3758,11 @@ protected static void prePrepareServer()
             setupFileSystem();
 
             CQLTester.prePrepareServer();
+
+            // The in-memory (jimfs) filesystem installed above cannot do Direct I/O: its FileStore has no
+            // getBlockSize() and it rejects ExtendedOpenOption.DIRECT. cassandra_latest.yaml enables
+            // background_write_disk_access_mode: direct, which would otherwise crash every flush+compact here.
+            DatabaseDescriptor.setBackgroundWriteDiskAccessMode(Config.DiskAccessMode.standard);
         }
 
         protected static void setupFileSystem()