Skip to content

Commit fc230cf

Browse files
chuanbao666chuanbao666
authored
fix(agfs): enable agfs s3 plugin default (#1408)
1 parent bdba0ef commit fc230cf

5 files changed

Lines changed: 102 additions & 62 deletions

File tree

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ build: check-deps check-pip
100100
fi; \
101101
if [ -n "$$MATURIN_CMD" ]; then \
102102
TMPDIR=$$(mktemp -d); \
103-
cd crates/ragfs-python && $$MATURIN_CMD build --release --out "$$TMPDIR" 2>&1; \
103+
cd crates/ragfs-python && $$MATURIN_CMD build --release --features s3 --out "$$TMPDIR" 2>&1; \
104104
cd ../..; \
105105
mkdir -p openviking/lib; \
106106
echo "import zipfile, glob, shutil, os, sys" > /tmp/extract_ragfs.py; \
@@ -136,4 +136,4 @@ clean:
136136
done
137137
@find . -name "*.pyc" -delete
138138
@find . -name "__pycache__" -type d -exec rm -rf {} +
139-
@echo "Cleanup completed."
139+
@echo "Cleanup completed."

‎crates/ragfs-python/Cargo.toml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,10 @@ publish = false
99
name = "ragfs_python"
1010
crate-type = ["cdylib"]
1111

12+
[features]
13+
default = ["s3"]
14+
s3 = ["ragfs/s3"]
15+
1216
[dependencies]
1317
ragfs = { path = "../ragfs" }
1418
pyo3 = { version = "0.27", features = ["extension-module"] }

‎crates/ragfs-python/src/lib.rs‎

Lines changed: 93 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,11 @@ use std::sync::Arc;
1212
use std::time::UNIX_EPOCH;
1313

1414
use ragfs::core::{ConfigValue, FileInfo, FileSystem, MountableFS, PluginConfig, WriteFlag};
15-
use ragfs::plugins::{KVFSPlugin, LocalFSPlugin, MemFSPlugin, QueueFSPlugin, ServerInfoFSPlugin, SQLFSPlugin};
15+
#[cfg(feature = "s3")]
16+
use ragfs::plugins::S3FSPlugin;
17+
use ragfs::plugins::{
18+
KVFSPlugin, LocalFSPlugin, MemFSPlugin, QueueFSPlugin, SQLFSPlugin, ServerInfoFSPlugin,
19+
};
1620

1721
/// Convert a ragfs error into a Python RuntimeError
1822
fn to_py_err(e: ragfs::core::Error) -> PyErr {
@@ -125,6 +129,8 @@ impl RAGFSBindingClient {
125129
fs.register_plugin(SQLFSPlugin::new()).await;
126130
fs.register_plugin(LocalFSPlugin::new()).await;
127131
fs.register_plugin(ServerInfoFSPlugin::new()).await;
132+
#[cfg(feature = "s3")]
133+
fs.register_plugin(S3FSPlugin::new()).await;
128134
});
129135

130136
Ok(Self { fs, rt })
@@ -141,9 +147,24 @@ impl RAGFSBindingClient {
141147
fn get_capabilities(&self) -> PyResult<HashMap<String, Py<PyAny>>> {
142148
Python::attach(|py| {
143149
let mut m = HashMap::new();
144-
m.insert("version".to_string(), "ragfs-python".into_pyobject(py)?.into_any().unbind());
145-
let features = vec!["memfs", "kvfs", "queuefs", "sqlfs"];
146-
m.insert("features".to_string(), features.into_pyobject(py)?.into_any().unbind());
150+
m.insert(
151+
"version".to_string(),
152+
"ragfs-python".into_pyobject(py)?.into_any().unbind(),
153+
);
154+
let mut features = vec![
155+
"memfs",
156+
"kvfs",
157+
"queuefs",
158+
"sqlfs",
159+
"localfs",
160+
"serverinfofs",
161+
];
162+
#[cfg(feature = "s3")]
163+
features.push("s3fs");
164+
m.insert(
165+
"features".to_string(),
166+
features.into_pyobject(py)?.into_any().unbind(),
167+
);
147168
Ok(m)
148169
})
149170
}
@@ -154,9 +175,10 @@ impl RAGFSBindingClient {
154175
/// name, size, mode, modTime, isDir
155176
fn ls(&self, path: String) -> PyResult<Py<PyAny>> {
156177
let fs = self.fs.clone();
157-
let entries = self.rt.block_on(async move {
158-
fs.read_dir(&path).await
159-
}).map_err(to_py_err)?;
178+
let entries = self
179+
.rt
180+
.block_on(async move { fs.read_dir(&path).await })
181+
.map_err(to_py_err)?;
160182

161183
Python::attach(|py| {
162184
let list = PyList::empty(py);
@@ -187,13 +209,12 @@ impl RAGFSBindingClient {
187209
let off = if offset < 0 { 0u64 } else { offset as u64 };
188210
let sz = if size < 0 { 0u64 } else { size as u64 };
189211

190-
let data = self.rt.block_on(async move {
191-
fs.read(&path, off, sz).await
192-
}).map_err(to_py_err)?;
212+
let data = self
213+
.rt
214+
.block_on(async move { fs.read(&path, off, sz).await })
215+
.map_err(to_py_err)?;
193216

194-
Python::attach(|py| {
195-
Ok(PyBytes::new(py, &data).into())
196-
})
217+
Python::attach(|py| Ok(PyBytes::new(py, &data).into()))
197218
}
198219

199220
/// Read file content (alias for read).
@@ -212,19 +233,19 @@ impl RAGFSBindingClient {
212233
let _ = max_retries; // not applicable for local binding
213234
let fs = self.fs.clone();
214235
let len = data.len();
215-
self.rt.block_on(async move {
216-
fs.write(&path, &data, 0, WriteFlag::Create).await
217-
}).map_err(to_py_err)?;
236+
self.rt
237+
.block_on(async move { fs.write(&path, &data, 0, WriteFlag::Create).await })
238+
.map_err(to_py_err)?;
218239

219240
Ok(format!("Written {} bytes", len))
220241
}
221242

222243
/// Create a new empty file.
223244
fn create(&self, path: String) -> PyResult<HashMap<String, String>> {
224245
let fs = self.fs.clone();
225-
self.rt.block_on(async move {
226-
fs.create(&path).await
227-
}).map_err(to_py_err)?;
246+
self.rt
247+
.block_on(async move { fs.create(&path).await })
248+
.map_err(to_py_err)?;
228249

229250
let mut m = HashMap::new();
230251
m.insert("message".to_string(), "created".to_string());
@@ -238,9 +259,9 @@ impl RAGFSBindingClient {
238259
.map_err(|e| PyRuntimeError::new_err(format!("Invalid mode '{}': {}", mode, e)))?;
239260

240261
let fs = self.fs.clone();
241-
self.rt.block_on(async move {
242-
fs.mkdir(&path, mode_int).await
243-
}).map_err(to_py_err)?;
262+
self.rt
263+
.block_on(async move { fs.mkdir(&path, mode_int).await })
264+
.map_err(to_py_err)?;
244265

245266
let mut m = HashMap::new();
246267
m.insert("message".to_string(), "created".to_string());
@@ -251,13 +272,15 @@ impl RAGFSBindingClient {
251272
#[pyo3(signature = (path, recursive=false))]
252273
fn rm(&self, path: String, recursive: bool) -> PyResult<HashMap<String, String>> {
253274
let fs = self.fs.clone();
254-
self.rt.block_on(async move {
255-
if recursive {
256-
fs.remove_all(&path).await
257-
} else {
258-
fs.remove(&path).await
259-
}
260-
}).map_err(to_py_err)?;
275+
self.rt
276+
.block_on(async move {
277+
if recursive {
278+
fs.remove_all(&path).await
279+
} else {
280+
fs.remove(&path).await
281+
}
282+
})
283+
.map_err(to_py_err)?;
261284

262285
let mut m = HashMap::new();
263286
m.insert("message".to_string(), "deleted".to_string());
@@ -267,9 +290,10 @@ impl RAGFSBindingClient {
267290
/// Get file/directory information.
268291
fn stat(&self, path: String) -> PyResult<Py<PyAny>> {
269292
let fs = self.fs.clone();
270-
let info = self.rt.block_on(async move {
271-
fs.stat(&path).await
272-
}).map_err(to_py_err)?;
293+
let info = self
294+
.rt
295+
.block_on(async move { fs.stat(&path).await })
296+
.map_err(to_py_err)?;
273297

274298
Python::attach(|py| {
275299
let dict = file_info_to_py_dict(py, &info)?;
@@ -280,9 +304,9 @@ impl RAGFSBindingClient {
280304
/// Rename/move a file or directory.
281305
fn mv(&self, old_path: String, new_path: String) -> PyResult<HashMap<String, String>> {
282306
let fs = self.fs.clone();
283-
self.rt.block_on(async move {
284-
fs.rename(&old_path, &new_path).await
285-
}).map_err(to_py_err)?;
307+
self.rt
308+
.block_on(async move { fs.rename(&old_path, &new_path).await })
309+
.map_err(to_py_err)?;
286310

287311
let mut m = HashMap::new();
288312
m.insert("message".to_string(), "renamed".to_string());
@@ -292,9 +316,9 @@ impl RAGFSBindingClient {
292316
/// Change file permissions.
293317
fn chmod(&self, path: String, mode: u32) -> PyResult<HashMap<String, String>> {
294318
let fs = self.fs.clone();
295-
self.rt.block_on(async move {
296-
fs.chmod(&path, mode).await
297-
}).map_err(to_py_err)?;
319+
self.rt
320+
.block_on(async move { fs.chmod(&path, mode).await })
321+
.map_err(to_py_err)?;
298322

299323
let mut m = HashMap::new();
300324
m.insert("message".to_string(), "chmod ok".to_string());
@@ -304,16 +328,18 @@ impl RAGFSBindingClient {
304328
/// Touch a file (create if not exists, or update timestamp).
305329
fn touch(&self, path: String) -> PyResult<HashMap<String, String>> {
306330
let fs = self.fs.clone();
307-
self.rt.block_on(async move {
308-
// Try create; if already exists, write empty to update mtime
309-
match fs.create(&path).await {
310-
Ok(_) => Ok(()),
311-
Err(_) => {
312-
// File exists, write empty bytes to update timestamp
313-
fs.write(&path, &[], 0, WriteFlag::None).await.map(|_| ())
331+
self.rt
332+
.block_on(async move {
333+
// Try create; if already exists, write empty to update mtime
334+
match fs.create(&path).await {
335+
Ok(_) => Ok(()),
336+
Err(_) => {
337+
// File exists, write empty bytes to update timestamp
338+
fs.write(&path, &[], 0, WriteFlag::None).await.map(|_| ())
339+
}
314340
}
315-
}
316-
}).map_err(to_py_err)?;
341+
})
342+
.map_err(to_py_err)?;
317343

318344
let mut m = HashMap::new();
319345
m.insert("message".to_string(), "touched".to_string());
@@ -323,9 +349,7 @@ impl RAGFSBindingClient {
323349
/// List all mounted plugins.
324350
fn mounts(&self) -> PyResult<Vec<HashMap<String, String>>> {
325351
let fs = self.fs.clone();
326-
let mount_list = self.rt.block_on(async move {
327-
fs.list_mounts().await
328-
});
352+
let mount_list = self.rt.block_on(async move { fs.list_mounts().await });
329353

330354
let result: Vec<HashMap<String, String>> = mount_list
331355
.into_iter()
@@ -365,9 +389,9 @@ impl RAGFSBindingClient {
365389
};
366390

367391
let fs = self.fs.clone();
368-
self.rt.block_on(async move {
369-
fs.mount(plugin_config).await
370-
}).map_err(to_py_err)?;
392+
self.rt
393+
.block_on(async move { fs.mount(plugin_config).await })
394+
.map_err(to_py_err)?;
371395

372396
let mut m = HashMap::new();
373397
m.insert(
@@ -381,9 +405,9 @@ impl RAGFSBindingClient {
381405
fn unmount(&self, path: String) -> PyResult<HashMap<String, String>> {
382406
let fs = self.fs.clone();
383407
let path_clone = path.clone();
384-
self.rt.block_on(async move {
385-
fs.unmount(&path_clone).await
386-
}).map_err(to_py_err)?;
408+
self.rt
409+
.block_on(async move { fs.unmount(&path_clone).await })
410+
.map_err(to_py_err)?;
387411

388412
let mut m = HashMap::new();
389413
m.insert("message".to_string(), format!("unmounted {}", path));
@@ -393,14 +417,17 @@ impl RAGFSBindingClient {
393417
/// List all registered plugin names.
394418
fn list_plugins(&self) -> PyResult<Vec<String>> {
395419
// Return names of built-in plugins
396-
Ok(vec![
420+
let mut plugins = vec![
397421
"memfs".to_string(),
398422
"kvfs".to_string(),
399423
"queuefs".to_string(),
400424
"sqlfs".to_string(),
401425
"localfs".to_string(),
402426
"serverinfofs".to_string(),
403-
])
427+
];
428+
#[cfg(feature = "s3")]
429+
plugins.push("s3fs".to_string());
430+
Ok(plugins)
404431
}
405432

406433
/// Get detailed plugin information.
@@ -433,7 +460,14 @@ impl RAGFSBindingClient {
433460
stream: bool,
434461
node_limit: Option<i32>,
435462
) -> PyResult<Py<PyAny>> {
436-
let _ = (path, pattern, recursive, case_insensitive, stream, node_limit);
463+
let _ = (
464+
path,
465+
pattern,
466+
recursive,
467+
case_insensitive,
468+
stream,
469+
node_limit,
470+
);
437471
Err(PyRuntimeError::new_err(
438472
"grep not yet implemented in ragfs-python",
439473
))

‎setup.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,8 @@ def build_ragfs_python_artifact(self):
259259
"maturin",
260260
"build",
261261
"--release",
262+
"--features",
263+
"s3",
262264
"--out",
263265
tmpdir,
264266
]

‎tests/agfs/test_fs_binding.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from openviking_cli.utils.config.agfs_config import AGFSConfig
1919

2020
# Direct configuration for testing
21-
AGFS_CONF = AGFSConfig(path="/tmp/ov-test", backend="local", mode="binding-client")
21+
AGFS_CONF = AGFSConfig(path="/tmp/ov-test", backend="local")
2222

2323
# clean up test directory if it exists
2424
if os.path.exists(AGFS_CONF.path):

0 commit comments

Comments
 (0)