Skip to content

Commit 0c31647

Browse files
committed
Used kv interface(Higher API)
Signed-off-by: dpj135 <958208521@qq.com>
1 parent e353874 commit 0c31647

1 file changed

Lines changed: 8 additions & 27 deletions

File tree

docs/storage_backends/openyuanrong_datasystem.md

Lines changed: 8 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,7 @@ pip install torch-npu==2.8.0
8686

8787
After installation, you can run TransferQueue with Yuanrong backend.
8888

89-
Firstly, start ray cluster on localhost:
89+
First, start a local Ray cluster. Yuanrong backend relies on Ray for distributed management:
9090
```bash
9191
ray start --head
9292
```
@@ -106,23 +106,12 @@ conf = OmegaConf.create({"backend": {"storage_backend": "Yuanrong"}})
106106
# Initialize TransferQueue + Yuanrong
107107
tq.init(conf)
108108

109-
# Get TransferQueue client
110-
client = tq.get_client()
111-
112-
# Put data
109+
# Put data using kv_put
113110
data = TensorDict({"input": torch.randn(2, 10)}, batch_size=[2])
114-
batch_meta = client.put(data=data, partition_id="any_id")
115-
116-
# Get metadata before getting data
117-
meta = client.get_meta(
118-
data_fields=list(data.keys()),
119-
batch_size=data.size(0),
120-
partition_id="any_id",
121-
task_name="any_name",
122-
)
123-
124-
# Get data
125-
result = client.get_data(meta)
111+
tq.kv_batch_put(keys=["sample_0", "sample_1"], partition_id="train", fields=data)
112+
113+
# Get data using kv_batch_get
114+
result = tq.kv_batch_get(keys=["sample_0", "sample_1"], partition_id="train")
126115
print("output:", result)
127116

128117
# Cleanup
@@ -240,24 +229,16 @@ class DataActor:
240229
# Each process must call tq.init() to get a client
241230
tq.init(config)
242231
torch.npu.set_device(0)
243-
self.client = tq.get_client()
244232
245233
def put_data(self):
246234
"""Put data on this node."""
247235
data = TensorDict({"input": torch.ones((3, 512), device="npu")}, batch_size=[3])
248-
batch_meta = self.client.put(data=data, partition_id="train_0")
236+
tq.kv_batch_put(keys=["s0", "s1", "s2"], partition_id="train", fields=data)
249237
print(f"[put] Data put completed")
250-
return batch_meta
251238
252239
def get_data(self):
253240
"""Get data on this node."""
254-
meta = self.client.get_meta(
255-
data_fields=["input"],
256-
batch_size=3,
257-
partition_id="train_0",
258-
task_name="generate_sequence",
259-
)
260-
result = self.client.get_data(meta)
241+
result = tq.kv_batch_get(keys=["s0", "s1", "s2"], partition_id="train")
261242
print(f"[get] Data get completed: {result['input']}")
262243
return result
263244

0 commit comments

Comments
 (0)