[Feature] Add async policy-server collector benchmark prototype#3939
Merged
Conversation
🔗 Helpful Links🧪 See artifacts and rendered test results at hud.pytorch.org/pr/pytorch/rl/3939
Note: Links to docs will display an error until the docs builds have been completed. This comment was automatically generated by Dr. CI and updates every 15 minutes. |
github-actions
Bot
added
Documentation
meta-cla
Bot
added
the
CLA Signed
This was referenced Jul 6, 2026
Contributor
Benchmark Results: PR
|
| Benchmark | main ops | PR ops | Change |
|---|---|---|---|
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[False-None] |
61.26 | 89.00 | +45.28% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-ListStorage-RandomSampler-400] |
35.92 | 50.28 | +39.99% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000] |
3,630 | 2,782 | -23.35% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
3,425 | 2,675 | -21.90% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-16] |
36.24 | 28.75 | -20.68% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[True-backward] |
822.58 | 985.45 | +19.80% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
2,848 | 3,384 | +18.84% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-400] |
760.57 | 900.06 | +18.34% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[4-same] |
28.17 | 23.98 | -14.86% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-True-32-512] |
33.89 | 29.08 | -14.20% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[16-same] |
19.74 | 22.54 | +14.19% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-64] |
9.5636 | 8.2290 | -13.96% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
1,947 | 2,217 | +13.90% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-400] |
476.12 | 537.54 | +12.90% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-64] |
12.21 | 10.67 | -12.60% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
2,300 | 2,014 | -12.41% |
benchmarks/test_objectives_benchmarks.py::test_values[vec_generalized_advantage_estimate-True-True] |
49.93 | 55.37 | +10.91% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[True-backward] |
380.58 | 418.94 | +10.08% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[True-backward] |
103.33 | 113.58 | +9.92% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-False-True] |
39,424 | 35,585 | -9.74% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
2,945 | 2,714 | -7.84% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[True-backward] |
116.88 | 125.69 | +7.54% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000] |
3,103 | 2,870 | -7.50% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[True-backward] |
126.41 | 135.35 | +7.08% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
2,128 | 1,978 | -7.05% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-64] |
6.1858 | 5.8600 | -5.27% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-True-True] |
23,723 | 22,539 | -4.99% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[True-None] |
437.01 | 457.87 | +4.77% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[safetensors] |
22,785 | 23,847 | +4.66% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[reduce-overhead-None] |
448.22 | 468.55 | +4.54% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-True-False] |
28,107 | 26,885 | -4.35% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_with_rb[200-img_shape1-large_batch] |
10.22 | 9.7804 | -4.35% |
benchmarks/test_objectives_benchmarks.py::test_values[vec_td1_return_estimate-False-False] |
57.47 | 55.21 | -3.93% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-64] |
1.2222 | 1.2687 | +3.80% |
benchmarks/test_envs_benchmark.py::test_simple |
1.7806 | 1.7136 | -3.77% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-1] |
264.42 | 274.26 | +3.72% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-16] |
4.9007 | 5.0812 | +3.68% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[reduce-overhead-None] |
1,793 | 1,859 | +3.64% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
2,184 | 2,106 | -3.55% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[50-img_shape0-small] |
4,211 | 4,356 | +3.45% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[False-None] |
172.48 | 178.30 | +3.37% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-gru] |
2.9381 | 3.0354 | +3.31% |
benchmarks/test_objectives_benchmarks.py::test_values[vec_td_lambda_return_estimate-True-False] |
57.09 | 55.21 | -3.30% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[200-img_shape3-large_batch] |
326.92 | 337.69 | +3.30% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-True-1-512] |
640.77 | 661.64 | +3.26% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[numpy] |
346,876 | 335,736 | -3.21% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-400] |
492.01 | 507.67 | +3.18% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-True-True] |
19,524 | 18,904 | -3.18% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-True-True] |
20,567 | 19,916 | -3.17% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_without_rb[200-img_shape1-large_batch] |
14.75 | 15.22 | +3.13% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-10000-10000-100-False] |
53.44 | 51.77 | -3.12% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[100-img_shape1-atari] |
687.29 | 708.36 | +3.07% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-1000000-10000-100-False] |
47.89 | 46.45 | -3.01% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-64] |
4.4205 | 4.5525 | +2.99% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[50-img_shape0-small] |
3,452 | 3,554 | +2.97% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000] |
3,253 | 3,348 | +2.92% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-False-False] |
76,759 | 74,529 | -2.91% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[200-img_shape3-large_batch] |
306.97 | 315.67 | +2.84% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-parallel-buffers-False] |
0.5951 | 0.5786 | -2.77% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[True-None] |
254.41 | 261.45 | +2.77% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-True-False] |
35,240 | 34,271 | -2.75% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-400] |
1,094 | 1,064 | -2.75% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-4] |
142.50 | 146.37 | +2.72% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[reduce-overhead-None] |
270.17 | 277.30 | +2.64% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape1-atari] |
638.85 | 655.67 | +2.63% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sample_mixed_devices[1000000-memmap_cpu_storage_cpu... |
81.38 | 83.50 | +2.60% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[False-backward] |
64.04 | 62.38 | -2.59% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[False-backward] |
88.96 | 91.25 | +2.58% |
benchmarks/test_collectors_benchmark.py::test_single |
8.9222 | 8.6939 | -2.56% |
benchmarks/test_envs_benchmark.py::test_serial |
0.5735 | 0.5882 | +2.55% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-False-True-True] |
21,848 | 21,304 | -2.49% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[untyped_storage] |
8.0454 | 8.2434 | +2.46% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape2-large_img] |
406.09 | 416.02 | +2.44% |
benchmarks/test_objectives_benchmarks.py::test_redq_speed[True-None] |
215.97 | 221.17 | +2.40% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-16] |
17.70 | 18.12 | +2.38% |
benchmarks/test_objectives_benchmarks.py::test_redq_speed[False-backward] |
56.08 | 54.77 | -2.35% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[False-backward] |
128.85 | 131.86 | +2.33% |
benchmarks/test_envs_benchmark.py::test_parallel |
0.9525 | 0.9304 | -2.32% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-sampler6-10000] |
771.42 | 788.75 | +2.25% |
benchmarks/test_objectives_benchmarks.py::test_values[generalized_advantage_estimate-True-True] |
91.02 | 93.06 | +2.24% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[True-None] |
278.43 | 284.58 | +2.21% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-400] |
496.38 | 485.47 | -2.20% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-True-False] |
31,874 | 31,220 | -2.05% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-True-True-False] |
34,606 | 33,901 | -2.04% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-4] |
70.37 | 71.78 | +2.01% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-parallel-no-buffers-True] |
0.2098 | 0.2139 | +1.99% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-True-False] |
29,181 | 28,603 | -1.98% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-1] |
638.43 | 625.88 | -1.97% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-False-False] |
64,209 | 62,956 | -1.95% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-True-False] |
31,696 | 31,079 | -1.95% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-True-0-gru] |
4.1953 | 4.2710 | +1.80% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-64] |
3.0207 | 3.0748 | +1.79% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-16] |
12.07 | 12.28 | +1.73% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_with_rb[100-img_shape0-atari] |
20.16 | 20.51 | +1.71% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-10000-10000-100-True] |
25.32 | 24.89 | -1.70% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[True-backward] |
60.49 | 59.50 | -1.63% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-4] |
48.29 | 49.05 | +1.58% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-True-False] |
42,481 | 41,811 | -1.58% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-False-1-512] |
2,199 | 2,233 | +1.56% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-ListStorage-None-400] |
184.23 | 187.10 | +1.56% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000] |
163.08 | 165.61 | +1.55% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-1] |
480.41 | 473.04 | -1.53% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_without_rb[100-img_shape0-atari] |
29.30 | 29.73 | +1.48% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[reduce-overhead-None] |
282.39 | 286.56 | +1.48% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[False-None] |
693.54 | 703.74 | +1.47% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-4] |
19.98 | 20.27 | +1.46% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-1] |
79.28 | 80.42 | +1.44% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[False-None] |
210.95 | 213.98 | +1.43% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_stack_then_write[100-img_shape2-large_img] |
171.18 | 168.76 | -1.42% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[torch.save] |
7,124 | 7,024 | -1.40% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[4-constant] |
4,085 | 4,142 | +1.40% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[reduce-overhead-None] |
707.06 | 716.74 | +1.37% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-lstm] |
2.0035 | 2.0307 | +1.36% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[True-None] |
698.60 | 708.05 | +1.35% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[50-img_shape0-small] |
7,102 | 7,198 | +1.35% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[False-backward] |
33.16 | 32.72 | -1.34% |
benchmarks/test_objectives_benchmarks.py::test_values[td1_return_estimate-False-False] |
36.37 | 36.84 | +1.31% |
benchmarks/test_objectives_benchmarks.py::test_values[td_lambda_return_estimate-True-False] |
24.66 | 24.98 | +1.31% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-single-True] |
1.3078 | 1.2908 | -1.30% |
benchmarks/test_objectives_benchmarks.py::test_cql_speed[True-backward] |
58.07 | 57.32 | -1.29% |
| ... | ... | ... | Showing 120 of 216 comparisons, sorted by absolute change. |
GPU
Compared 226 benchmarks. Regressions over 5%: 10. Improvements over 5%: 18.
| Benchmark | main ops | PR ops | Change |
|---|---|---|---|
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-ListStorage-RandomSampler-400] |
44.56 | 200.70 | +350.43% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-ListStorage-None-400] |
188.71 | 49.32 | -73.87% |
benchmarks/test_objectives_benchmarks.py::test_cql_speed[reduce-overhead-None] |
61.26 | 89.34 | +45.84% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[reduce-overhead-None] |
78.07 | 103.77 | +32.92% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000] |
2,612 | 3,283 | +25.69% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000] |
3,596 | 2,679 | -25.49% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
2,461 | 3,073 | +24.83% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
3,373 | 2,550 | -24.41% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[reduce-overhead-None] |
86.52 | 107.24 | +23.95% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
1,901 | 2,328 | +22.45% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000] |
3,621 | 2,881 | -20.45% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
3,240 | 2,763 | -14.72% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-False-True] |
36,704 | 41,248 | +12.38% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[True-backward] |
901.22 | 1,007 | +11.75% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
3,110 | 3,440 | +10.61% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-400] |
486.50 | 530.76 | +9.10% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000] |
3,410 | 3,692 | +8.26% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
2,054 | 2,223 | +8.19% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-parallel-buffers-True] |
0.5323 | 0.4914 | -7.68% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-1] |
594.98 | 637.78 | +7.19% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[numpy] |
378,701 | 352,729 | -6.86% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-400] |
497.13 | 529.40 | +6.49% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-400] |
995.01 | 931.38 | -6.39% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[True-backward] |
454.48 | 483.52 | +6.39% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[True-backward] |
319.32 | 337.85 | +5.80% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-400] |
867.82 | 821.20 | -5.37% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-True-False] |
27,972 | 29,446 | +5.27% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[True-None] |
758.36 | 718.93 | -5.20% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_stack_then_write[200-img_shape3-large_batch] |
135.02 | 141.48 | +4.78% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-gru] |
22.93 | 21.83 | -4.77% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-400] |
501.52 | 477.85 | -4.72% |
benchmarks/test_objectives_benchmarks.py::test_values[td1_return_estimate-False-False] |
19.77 | 18.84 | -4.67% |
benchmarks/test_envs_benchmark.py::test_simple |
1.2737 | 1.2153 | -4.59% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[True-backward] |
266.52 | 277.88 | +4.26% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[generalized_advantage_estimate-False-1-512] |
49.92 | 47.91 | -4.01% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[False-backward] |
237.97 | 228.56 | -3.96% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
2,289 | 2,200 | -3.88% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sample_mixed_devices[1000000-cuda_storage_cuda_samp... |
1,452 | 1,506 | +3.74% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-single-True] |
1.2860 | 1.3318 | +3.57% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[4-constant] |
4,619 | 4,783 | +3.54% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-True-True] |
22,984 | 23,795 | +3.53% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[safetensors] |
22,977 | 23,749 | +3.36% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-4] |
20.17 | 19.51 | -3.25% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[pickle] |
12,136 | 12,526 | +3.21% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[untyped_storage] |
8.8531 | 8.5714 | -3.18% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-True-False] |
32,152 | 33,167 | +3.15% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-16] |
5.0356 | 4.8773 | -3.14% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[100-img_shape1-atari] |
4,270 | 4,400 | +3.04% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[4-same] |
6.5418 | 6.7405 | +3.04% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_with_rb[100-img_shape0-atari] |
20.02 | 20.61 | +2.97% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-100000-10000-100-False] |
53.37 | 51.79 | -2.96% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-False-32-512] |
1,345 | 1,305 | -2.91% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-False-False] |
50,466 | 51,909 | +2.86% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[reduce-overhead-None] |
123.45 | 120.14 | -2.69% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-16] |
47.35 | 48.62 | +2.68% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[False-None] |
350.78 | 341.38 | -2.68% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-True-False] |
29,889 | 30,689 | +2.68% |
benchmarks/test_collectors_benchmark.py::test_sync_pixels |
10.08 | 10.34 | +2.60% |
benchmarks/test_objectives_benchmarks.py::test_cql_speed[True-backward] |
222.95 | 228.70 | +2.58% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-lstm] |
21.76 | 21.21 | -2.55% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-False-True] |
41,326 | 42,370 | +2.53% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-64] |
12.10 | 12.40 | +2.49% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_without_rb_cuda[100-img_shape0-atari] |
17.21 | 17.62 | +2.38% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-False-False] |
46,212 | 47,296 | +2.34% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[True-None] |
439.64 | 429.33 | -2.34% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-False-False] |
50,983 | 52,157 | +2.30% |
benchmarks/test_objectives_benchmarks.py::test_cql_speed[False-backward] |
40.19 | 39.27 | -2.29% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[True-None] |
742.14 | 758.90 | +2.26% |
benchmarks/test_envs_benchmark.py::test_transformed |
0.7217 | 0.7054 | -2.25% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-True-False] |
35,063 | 35,831 | +2.19% |
benchmarks/test_objectives_benchmarks.py::test_cql_speed[False-None] |
55.03 | 53.82 | -2.19% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-True-True] |
20,724 | 21,169 | +2.15% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-1] |
192.77 | 188.69 | -2.11% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-64] |
4.3766 | 4.4690 | +2.11% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[False-None] |
656.77 | 643.31 | -2.05% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-True-0-gru] |
50.33 | 49.31 | -2.02% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-True-True] |
19,690 | 20,087 | +2.02% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-16] |
35.46 | 36.16 | +1.98% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-False-False] |
78,529 | 80,080 | +1.97% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-serial-buffers-True] |
0.5173 | 0.5274 | +1.97% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[reduce-overhead-None] |
1,918 | 1,881 | -1.93% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-64] |
10.54 | 10.74 | +1.92% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[False-None] |
282.33 | 276.93 | -1.91% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape1-atari] |
624.94 | 636.77 | +1.89% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-False-True] |
31,072 | 31,655 | +1.87% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sampler_sample_scale[1000000-cuda] |
2,186 | 2,226 | +1.87% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-False-False] |
55,862 | 56,904 | +1.86% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[False-backward] |
463.12 | 454.53 | -1.85% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sample_mixed_devices[1000000-cuda_storage_cpu_sampler] |
87.33 | 88.92 | +1.81% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[True-backward] |
351.47 | 357.81 | +1.80% |
benchmarks/test_collectors_benchmark.py::test_single_pixels |
6.3885 | 6.2748 | -1.78% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[False-None] |
114.55 | 112.52 | -1.78% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-4] |
139.50 | 141.92 | +1.74% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000] |
167.33 | 170.23 | +1.73% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_with_rb_cuda[200-img_shape1-large_batch] |
6.8297 | 6.7140 | -1.69% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-1000000-10000-100-False] |
49.01 | 48.18 | -1.68% |
benchmarks/test_objectives_benchmarks.py::test_values[td0_return_estimate-False-False] |
11,290 | 11,477 | +1.65% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-True-True] |
20,326 | 19,990 | -1.65% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-False-1-512] |
1,336 | 1,357 | +1.59% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_with_rb_cuda[100-img_shape0-atari] |
13.62 | 13.40 | -1.58% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[False-None] |
231.88 | 228.23 | -1.57% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape2-large_img] |
399.91 | 393.71 | -1.55% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-True-1-512] |
1,262 | 1,243 | -1.47% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-False-False-True] |
39,074 | 38,501 | -1.47% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[reduce-overhead-None] |
100.22 | 101.68 | +1.45% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-64] |
7.0073 | 6.9073 | -1.43% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[50-img_shape0-small] |
4,266 | 4,206 | -1.41% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[reduce-overhead-None] |
848.61 | 836.86 | -1.39% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-True-False] |
32,695 | 33,147 | +1.38% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-True-0-lstm] |
77.74 | 76.69 | -1.35% |
benchmarks/test_objectives_benchmarks.py::test_values[vec_td1_return_estimate-False-False] |
874.70 | 862.92 | -1.35% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[50-img_shape0-small] |
3,532 | 3,580 | +1.35% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[200-img_shape3-large_batch] |
723.10 | 713.47 | -1.33% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-1] |
278.05 | 274.56 | -1.26% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-sampler7-10000] |
821.65 | 831.87 | +1.24% |
benchmarks/test_collectors_benchmark.py::test_async_pixels |
10.87 | 10.74 | -1.23% |
benchmarks/test_collectors_benchmark.py::test_single |
6.7295 | 6.8123 | +1.23% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-serial-buffers-False] |
0.6008 | 0.6081 | +1.21% |
benchmarks/test_objectives_benchmarks.py::test_values[td_lambda_return_estimate-True-False] |
11.98 | 11.83 | -1.21% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[False-backward] |
76.26 | 77.16 | +1.18% |
| ... | ... | ... | Showing 120 of 226 comparisons, sorted by absolute change. |
This was referenced Jul 6, 2026
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Stack from ghstack (oldest at bottom):