Skip to content

Commit 24da4c5

Browse files
fslongjinfslongjin
authored
perf(virtiofs): optimize non-DAX cached reads (#2117)
* perf(virtiofs): optimize non-DAX cached reads Signed-off-by: longjin <longjin@dragonos.org> * fix(virtiofs): complete non-DAX read fault handling Signed-off-by: longjin <longjin@dragonos.org> * fix(vfs): preserve selected mounts across file operations Signed-off-by: longjin <longjin@dragonos.org> * test(virtiofs): add cross-environment performance evidence Signed-off-by: longjin <longjin@dragonos.org> * docs(virtiofs): refresh final validation revisions Signed-off-by: longjin <longjin@dragonos.org> * test(vfs): use mmap-capable fs for mount identity coverage Signed-off-by: longjin <longjin@dragonos.org> * docs(virtiofs): record direct P0 P1 acceptance Signed-off-by: longjin <longjin@dragonos.org> * chore(virtiofs): keep planning documents local Remove documentation-only changes from the pull request while retaining the implementation, tests, and benchmark tooling. Signed-off-by: longjin <longjin@dragonos.org> * fix(virtiofs): validate Linux trace inputs independently Reject an invalid run ID, case ID, file size, or block size even when the paired value is valid. Add negative coverage for every independently validated input. Signed-off-by: longjin <longjin@dragonos.org> --------- Signed-off-by: longjin <longjin@dragonos.org>
1 parent d1d6328 commit 24da4c5

45 files changed

Lines changed: 9095 additions & 367 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎kernel/src/arch/x86_64/mm/fault.rs‎

Lines changed: 21 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
use alloc::sync::Arc;
22
use core::{intrinsics::unlikely, panic};
33
use log::error;
4+
use system_error::SystemError;
45
use x86::{bits64::rflags::RFlags, controlregs::Cr4};
56

67
use crate::{
@@ -499,9 +500,26 @@ impl X86_64MMArch {
499500
let wait = outcome.retry_wait;
500501
drop(space_guard);
501502
if let Some(wait) = wait {
502-
if wait.wait().is_err() {
503-
send_bus_adrerr();
504-
return;
503+
match wait.wait() {
504+
Ok(()) => {}
505+
Err(SystemError::EINTR | SystemError::ERESTARTSYS) => {
506+
if regs.is_from_user() {
507+
<crate::arch::ipc::signal::X86_64SignalArch as SignalArch>::do_signal_or_restart(regs);
508+
} else {
509+
handle_kernel_access_failed(regs);
510+
}
511+
return;
512+
}
513+
Err(SystemError::ENOMEM) => {
514+
fault = VmFaultReason::VM_FAULT_OOM;
515+
space_guard = current_address_space.write();
516+
break;
517+
}
518+
Err(_) => {
519+
fault = VmFaultReason::VM_FAULT_SIGBUS;
520+
space_guard = current_address_space.write();
521+
break;
522+
}
505523
}
506524
}
507525
continue 'fault_retry;

‎kernel/src/debug/fuse.rs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
use alloc::string::ToString;
2+
use core::str;
23

34
use crate::debug::sysfs::debugfs_kobj;
45
use crate::driver::base::kobject::KObject;
@@ -82,6 +83,60 @@ impl KernFSCallback for FuseStatsCallBack {
8283
}
8384
}
8485

86+
#[derive(Debug)]
87+
struct FuseStatsControlCallBack;
88+
89+
impl KernFSCallback for FuseStatsControlCallBack {
90+
fn open(&self, mut data: KernCallbackData) -> Result<(), SystemError> {
91+
let mode = crate::filesystem::fuse::stats::stats_mode();
92+
data.file_private_data_mut()
93+
.replace(KernFilePrivateData::DebugTextSnapshot(alloc::format!(
94+
"{}\n",
95+
mode.as_str()
96+
)));
97+
Ok(())
98+
}
99+
100+
fn read(
101+
&self,
102+
data: KernCallbackData,
103+
buf: &mut [u8],
104+
offset: usize,
105+
) -> Result<usize, SystemError> {
106+
let report = match data.file_private_data() {
107+
Some(KernFilePrivateData::DebugTextSnapshot(report)) => report,
108+
_ => return Err(SystemError::EINVAL),
109+
};
110+
let bytes = report.as_bytes();
111+
if offset >= bytes.len() {
112+
return Ok(0);
113+
}
114+
let len = buf.len().min(bytes.len() - offset);
115+
buf[..len].copy_from_slice(&bytes[offset..offset + len]);
116+
Ok(len)
117+
}
118+
119+
fn write(
120+
&self,
121+
_data: KernCallbackData,
122+
buf: &[u8],
123+
offset: usize,
124+
) -> Result<usize, SystemError> {
125+
if offset != 0 || buf.is_empty() {
126+
return Err(SystemError::EINVAL);
127+
}
128+
let value = str::from_utf8(buf).map_err(|_| SystemError::EINVAL)?;
129+
let mode = crate::filesystem::fuse::stats::FuseStatsMode::parse(value)
130+
.map_err(|_| SystemError::EINVAL)?;
131+
crate::filesystem::fuse::stats::set_stats_mode(mode);
132+
Ok(buf.len())
133+
}
134+
135+
fn poll(&self, _data: KernCallbackData) -> Result<PollStatus, SystemError> {
136+
Ok(PollStatus::READ | PollStatus::WRITE)
137+
}
138+
}
139+
85140
pub fn init_debugfs_fuse() -> Result<(), SystemError> {
86141
let debugfs = debugfs_kobj();
87142
let root_dir = debugfs.inode().ok_or(SystemError::ENOENT)?;
@@ -100,5 +155,13 @@ pub fn init_debugfs_fuse() -> Result<(), SystemError> {
100155
Some(&FuseStatsCallBack),
101156
)?;
102157

158+
fuse_root.add_file(
159+
"stats_mode".to_string(),
160+
InodeMode::S_IRUSR | InodeMode::S_IWUSR,
161+
Some(32),
162+
None,
163+
Some(&FuseStatsControlCallBack),
164+
)?;
165+
103166
Ok(())
104167
}

‎kernel/src/filesystem/fuse/conn.rs‎

Lines changed: 176 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ use alloc::{
66
};
77
use core::{
88
mem::size_of,
9-
sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicUsize, Ordering},
9+
sync::atomic::{AtomicBool, AtomicU32, AtomicU64, AtomicU8, AtomicUsize, Ordering},
1010
};
1111

1212
use system_error::SystemError;
@@ -147,6 +147,10 @@ pub struct FuseRequest {
147147
opcode: u32,
148148
reply_contract: FuseReplyContract,
149149
read_pages_destination: Option<Arc<PageCacheReadDmaReservation>>,
150+
track_direct_read_stats: bool,
151+
/// Tracks which observable pipeline gauge currently owns this request:
152+
/// 0 untracked, 1 connection queue, 2 bridge dispatch, 3 retired/transport.
153+
stats_owner: AtomicU8,
150154
}
151155

152156
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -170,6 +174,47 @@ pub(crate) enum FuseReplyContract {
170174
}
171175

172176
impl FuseRequest {
177+
pub(crate) fn stats_mark_queued(&self) {
178+
if self
179+
.stats_owner
180+
.compare_exchange(0, 1, Ordering::AcqRel, Ordering::Acquire)
181+
.is_ok()
182+
{
183+
stats::on_fuse_queue_owner_acquired();
184+
}
185+
}
186+
187+
pub(crate) fn stats_mark_dispatched(&self) {
188+
if self
189+
.stats_owner
190+
.compare_exchange(1, 2, Ordering::AcqRel, Ordering::Acquire)
191+
.is_ok()
192+
{
193+
stats::on_fuse_dispatch_owner_acquired();
194+
stats::on_fuse_queue_owner_released();
195+
}
196+
}
197+
198+
pub(crate) fn stats_mark_external_dequeued(&self) {
199+
if self
200+
.stats_owner
201+
.compare_exchange(1, 3, Ordering::AcqRel, Ordering::Acquire)
202+
.is_ok()
203+
{
204+
stats::on_fuse_queue_owner_released();
205+
}
206+
}
207+
208+
pub(crate) fn stats_mark_transport_owned(&self) {
209+
if self
210+
.stats_owner
211+
.compare_exchange(2, 3, Ordering::AcqRel, Ordering::Acquire)
212+
.is_ok()
213+
{
214+
stats::on_fuse_dispatch_owner_released();
215+
}
216+
}
217+
173218
pub(crate) fn bytes(&self) -> &[u8] {
174219
&self.bytes
175220
}
@@ -189,6 +234,20 @@ impl FuseRequest {
189234
pub(crate) fn read_pages_destination(&self) -> Option<&Arc<PageCacheReadDmaReservation>> {
190235
self.read_pages_destination.as_ref()
191236
}
237+
238+
pub(crate) fn tracks_direct_read_stats(&self) -> bool {
239+
self.track_direct_read_stats
240+
}
241+
}
242+
243+
impl Drop for FuseRequest {
244+
fn drop(&mut self) {
245+
match self.stats_owner.swap(3, Ordering::AcqRel) {
246+
1 => stats::on_fuse_queue_owner_released(),
247+
2 => stats::on_fuse_dispatch_owner_released(),
248+
_ => {}
249+
}
250+
}
192251
}
193252

194253
#[derive(Debug, Clone, Copy)]
@@ -228,6 +287,7 @@ pub struct FusePendingState {
228287
background_credit: Mutex<Option<FuseBackgroundCredit>>,
229288
read_completion: Option<FuseReadCompletion>,
230289
outcome_unknown: AtomicBool,
290+
processing_owner: AtomicBool,
231291
}
232292

233293
#[derive(Debug)]
@@ -275,6 +335,9 @@ impl FuseReadCompletion {
275335
return Err(SystemError::EIO);
276336
}
277337
self.target.publish_completed(*bytes)?;
338+
if self.target.tracks_direct_read_stats() {
339+
stats::on_virtiofs_direct_read_completed(*bytes);
340+
}
278341
*bytes
279342
}
280343
Err(error) => {
@@ -431,6 +494,13 @@ impl FusePendingState {
431494
background_credit: Mutex::new(background_credit),
432495
read_completion,
433496
outcome_unknown: AtomicBool::new(false),
497+
processing_owner: AtomicBool::new(false),
498+
}
499+
}
500+
501+
pub(crate) fn mark_processing_owner(&self) {
502+
if !self.processing_owner.swap(true, Ordering::AcqRel) {
503+
stats::on_fuse_processing_begin();
434504
}
435505
}
436506

@@ -500,6 +570,9 @@ impl FusePendingState {
500570
let mut guard = self.response.lock();
501571
*guard = PendingCompletion::Ready(v, kind);
502572
drop(guard);
573+
if self.processing_owner.swap(false, Ordering::AcqRel) {
574+
stats::on_fuse_processing_end();
575+
}
503576
// Linux releases a background slot at request completion, not when a
504577
// waiter later consumes the result. Taking the token makes this
505578
// exactly-once across replies, abort and teardown.
@@ -1662,6 +1735,15 @@ impl FuseConn {
16621735
core::cmp::max(1, g.init.max_pages as usize)
16631736
}
16641737

1738+
/// Maximum payload accepted by the ordinary READ request builder after
1739+
/// applying both the negotiated byte cap and the backend/SG page cap.
1740+
pub fn effective_read_payload_limit(&self) -> usize {
1741+
core::cmp::min(
1742+
self.max_read(),
1743+
self.max_pages().saturating_mul(MMArch::PAGE_SIZE),
1744+
)
1745+
}
1746+
16651747
pub fn max_readahead_pages(&self) -> usize {
16661748
let g = self.inner.lock();
16671749
let bytes = if g.init.max_readahead == 0 {
@@ -1672,6 +1754,25 @@ impl FuseConn {
16721754
core::cmp::max(1, bytes >> MMArch::PAGE_SHIFT)
16731755
}
16741756

1757+
/// Snapshot immutable FUSE_INIT read capabilities under one connection
1758+
/// lock acquisition for use by an open file's cached-I/O hot path.
1759+
pub(crate) fn open_io_config(&self) -> super::private_data::FuseOpenIoConfig {
1760+
let g = self.inner.lock();
1761+
let max_readahead = if g.init.max_readahead == 0 {
1762+
Self::DEFAULT_MAX_READAHEAD
1763+
} else {
1764+
g.init.max_readahead as usize
1765+
};
1766+
super::private_data::FuseOpenIoConfig {
1767+
async_read: (g.init.flags & FUSE_ASYNC_READ) != 0,
1768+
auto_inval_data: (g.init.flags & FUSE_AUTO_INVAL_DATA) != 0,
1769+
writeback_cache: (g.init.flags & FUSE_WRITEBACK_CACHE) != 0,
1770+
max_read: core::cmp::max(Self::MIN_MAX_WRITE, g.max_read as usize),
1771+
max_pages: core::cmp::max(1, g.init.max_pages as usize),
1772+
max_readahead_pages: core::cmp::max(1, max_readahead >> MMArch::PAGE_SHIFT),
1773+
}
1774+
}
1775+
16751776
fn acquire_background_credit(
16761777
&self,
16771778
speculative: bool,
@@ -1693,6 +1794,8 @@ mod tests {
16931794

16941795
use system_error::SystemError;
16951796

1797+
use crate::arch::{MMArch, MemoryManagementArch};
1798+
16961799
use super::super::protocol::{
16971800
FuseEntryOut, FuseOpenOut, FuseOutHeader, FuseStatfsOut, FUSE_CREATE, FUSE_DESTROY,
16981801
FUSE_GETATTR, FUSE_HAS_INODE_DAX, FUSE_LOOKUP, FUSE_MAP_ALIGNMENT, FUSE_REMOVEMAPPING,
@@ -1701,14 +1804,85 @@ mod tests {
17011804
use super::super::virtiofs::dax::{DaxMountMode, DAX_RANGE_SIZE};
17021805
use super::{
17031806
daemon, request, stats, FuseCompletionKind, FuseConn, FusePendingState,
1704-
FuseReplyCapacitySource,
1807+
FuseReplyCapacitySource, FuseReplyContract, FuseRequest,
17051808
};
17061809

1810+
#[test]
1811+
fn quiescence_owners_are_tracked_in_off_mode() {
1812+
let previous_mode = stats::stats_mode();
1813+
stats::set_stats_mode(stats::FuseStatsMode::Off);
1814+
let before = stats::fuse_snapshot();
1815+
1816+
let request = FuseRequest {
1817+
bytes: vec![],
1818+
unique: 1,
1819+
opcode: FUSE_DESTROY,
1820+
reply_contract: FuseReplyContract::NoReply,
1821+
read_pages_destination: None,
1822+
track_direct_read_stats: false,
1823+
stats_owner: Default::default(),
1824+
};
1825+
request.stats_mark_queued();
1826+
assert_eq!(request.stats_owner.load(Ordering::Acquire), 1);
1827+
assert_eq!(
1828+
stats::fuse_snapshot().request_queue_current,
1829+
before.request_queue_current + 1
1830+
);
1831+
request.stats_mark_dispatched();
1832+
assert_eq!(request.stats_owner.load(Ordering::Acquire), 2);
1833+
let dispatched = stats::fuse_snapshot();
1834+
assert_eq!(
1835+
dispatched.request_queue_current,
1836+
before.request_queue_current
1837+
);
1838+
assert_eq!(dispatched.dispatch_current, before.dispatch_current + 1);
1839+
request.stats_mark_transport_owned();
1840+
assert_eq!(request.stats_owner.load(Ordering::Acquire), 3);
1841+
assert_eq!(
1842+
stats::fuse_snapshot().dispatch_current,
1843+
before.dispatch_current
1844+
);
1845+
1846+
let pending = FusePendingState::new(2, FUSE_GETATTR);
1847+
pending.mark_processing_owner();
1848+
assert!(pending.processing_owner.load(Ordering::Acquire));
1849+
assert_eq!(
1850+
stats::fuse_snapshot().processing_current,
1851+
before.processing_current + 1
1852+
);
1853+
assert!(pending.complete_never_submitted(SystemError::EIO));
1854+
assert!(!pending.processing_owner.load(Ordering::Acquire));
1855+
assert_eq!(
1856+
stats::fuse_snapshot().processing_current,
1857+
before.processing_current
1858+
);
1859+
1860+
stats::set_stats_mode(previous_mode);
1861+
}
1862+
17071863
fn set_minor(conn: &FuseConn, minor: u32) {
17081864
conn.inner.lock().init.minor = minor;
17091865
conn.reply_layout_minor.store(minor, Ordering::Release);
17101866
}
17111867

1868+
#[test]
1869+
fn effective_read_payload_limit_applies_byte_and_page_caps() {
1870+
let conn = FuseConn::new_for_virtiofs(256 * 1024, 256 * 1024);
1871+
{
1872+
let mut inner = conn.inner.lock();
1873+
inner.max_read = 64 * 1024;
1874+
inner.init.max_pages = 4;
1875+
}
1876+
assert_eq!(conn.effective_read_payload_limit(), 4 * MMArch::PAGE_SIZE);
1877+
1878+
{
1879+
let mut inner = conn.inner.lock();
1880+
inner.max_read = 8 * 1024;
1881+
inner.init.max_pages = 64;
1882+
}
1883+
assert_eq!(conn.effective_read_payload_limit(), 8 * 1024);
1884+
}
1885+
17121886
fn capacity(conn: &FuseConn, opcode: u32, payload: &[u8]) -> (usize, FuseReplyCapacitySource) {
17131887
let capacity = request::reply_capacity_for_test(conn, opcode, payload)
17141888
.unwrap()

0 commit comments

Comments
 (0)