[Feature] Add async policy-server collector benchmark prototype#3938
Closed
vmoens wants to merge 1 commit into
Closed
[Feature] Add async policy-server collector benchmark prototype#3938vmoens wants to merge 1 commit into
vmoens wants to merge 1 commit into
Conversation
🔗 Helpful Links🧪 See artifacts and rendered test results at hud.pytorch.org/pr/pytorch/rl/3938
Note: Links to docs will display an error until the docs builds have been completed. This comment was automatically generated by Dr. CI and updates every 15 minutes. |
meta-cla
Bot
added
the
CLA Signed
github-actions
Bot
added
Feature
Collaborator
Author
|
Superseded by #3939 after resubmitting the updated stack. |
Contributor
Benchmark Results: PR
|
| Benchmark | main ops | PR ops | Change |
|---|---|---|---|
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-ListStorage-RandomSampler-400] |
36.11 | 197.68 | +447.48% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
2,038 | 443.70 | -78.23% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-ListStorage-None-400] |
188.42 | 54.02 | -71.33% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-64] |
10.53 | 8.4004 | -20.25% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
2,638 | 3,166 | +19.99% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[True-backward] |
825.38 | 989.12 | +19.84% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
1,894 | 2,243 | +18.42% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000] |
2,789 | 3,244 | +16.32% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[4-same] |
29.54 | 24.93 | -15.61% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-400] |
910.15 | 780.28 | -14.27% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[16-same] |
20.62 | 23.41 | +13.53% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
2,694 | 3,030 | +12.48% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-64] |
6.4347 | 5.6405 | -12.34% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
3,413 | 2,993 | -12.30% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-64] |
12.14 | 10.72 | -11.65% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
1,971 | 2,196 | +11.41% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000] |
3,093 | 3,430 | +10.88% |
benchmarks/test_collectors_benchmark.py::test_single_with_rb |
7.3529 | 6.5791 | -10.52% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-10000] |
2,775 | 3,060 | +10.27% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
3,322 | 3,016 | -9.22% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-400] |
504.51 | 543.75 | +7.78% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-400] |
444.45 | 474.87 | +6.84% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[False-None] |
174.50 | 162.62 | -6.81% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[safetensors] |
23,098 | 24,626 | +6.62% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[100-img_shape1-atari] |
5,159 | 4,826 | -6.45% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-400] |
549.04 | 516.18 | -5.98% |
benchmarks/test_objectives_benchmarks.py::test_redq_speed[True-None] |
209.24 | 221.08 | +5.66% |
benchmarks/test_envs_benchmark.py::test_simple |
1.8245 | 1.7282 | -5.28% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
1,892 | 1,988 | +5.04% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[True-backward] |
267.32 | 280.28 | +4.85% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[4-constant] |
4,301 | 4,093 | -4.84% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[200-img_shape3-large_batch] |
778.41 | 815.52 | +4.77% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-True-False] |
33,920 | 35,522 | +4.72% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[False-backward] |
87.53 | 91.56 | +4.60% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-10000-10000-100-False] |
52.27 | 54.54 | +4.34% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_stack_then_write[100-img_shape2-large_img] |
174.99 | 167.48 | -4.29% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-10000-10000-100-True] |
24.73 | 25.79 | +4.28% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-1000000-10000-100-True] |
22.83 | 23.80 | +4.27% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-400] |
1,059 | 1,104 | +4.18% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-cudnn-False-0-gru] |
1.3856 | 1.3310 | -3.94% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[True-None] |
274.45 | 285.08 | +3.87% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-gru] |
2.8502 | 2.9604 | +3.87% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[pickle] |
12,433 | 11,965 | -3.77% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[reduce-overhead-None] |
264.66 | 274.53 | +3.73% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-True-True-False] |
34,016 | 35,221 | +3.54% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[untyped_storage] |
8.9036 | 8.5963 | -3.45% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[50-img_shape0-small] |
7,032 | 7,272 | +3.40% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-4] |
162.26 | 156.91 | -3.30% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[False-None] |
208.68 | 215.41 | +3.22% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-100000-10000-100-True] |
24.15 | 24.92 | +3.17% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[100-img_shape2-large_img] |
557.90 | 575.46 | +3.15% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-1000000-10000-100-False] |
46.89 | 48.35 | +3.12% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[torch.save] |
7,247 | 7,024 | -3.08% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[True-backward] |
116.62 | 120.14 | +3.01% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-64] |
4.5983 | 4.4608 | -2.99% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-sampler7-10000] |
791.79 | 768.31 | -2.97% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[False-None] |
158.11 | 162.74 | +2.93% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-False-True] |
29,815 | 30,679 | +2.90% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[100-img_shape2-large_img] |
422.81 | 410.71 | -2.86% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[reduce-overhead-None] |
449.20 | 461.94 | +2.84% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape1-atari] |
639.64 | 657.76 | +2.83% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-cudnn-False-0-lstm] |
0.8801 | 0.8552 | -2.83% |
benchmarks/test_objectives_benchmarks.py::test_values[td1_return_estimate-False-False] |
38.33 | 37.25 | -2.82% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[200-img_shape3-large_batch] |
331.56 | 340.79 | +2.78% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[True-None] |
277.58 | 269.87 | -2.78% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-True-True-True] |
20,155 | 20,706 | +2.73% |
benchmarks/test_envs_benchmark.py::test_serial |
0.5786 | 0.5943 | +2.72% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-16] |
41.77 | 42.87 | +2.62% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sampler_sample_scale[10000000-cpu] |
53.57 | 52.17 | -2.61% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[False-None] |
49.05 | 47.77 | -2.61% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-parallel-buffers-True] |
0.5073 | 0.4942 | -2.59% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-100000-10000-100-False] |
52.08 | 53.41 | +2.56% |
benchmarks/test_objectives_benchmarks.py::test_cql_speed[False-backward] |
28.44 | 27.72 | -2.53% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-False-False] |
44,345 | 45,460 | +2.51% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[50-img_shape0-small] |
3,457 | 3,543 | +2.49% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-False-False-False] |
63,085 | 64,614 | +2.43% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-False-False-True] |
38,727 | 37,793 | -2.41% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-False-True] |
37,589 | 38,494 | +2.41% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-True-True] |
23,849 | 23,276 | -2.40% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-False-True] |
29,647 | 30,347 | +2.36% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-64] |
1.2718 | 1.2429 | -2.27% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-False-True] |
41,689 | 42,611 | +2.21% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-16] |
18.30 | 17.89 | -2.20% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-4] |
148.14 | 144.90 | -2.19% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape2-large_img] |
400.80 | 392.29 | -2.12% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-4] |
20.33 | 19.90 | -2.12% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-400] |
193.54 | 197.60 | +2.10% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[True-None] |
116.57 | 118.98 | +2.07% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-4] |
72.53 | 71.03 | -2.06% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-single-True] |
1.3048 | 1.2783 | -2.03% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[100-img_shape1-atari] |
692.74 | 706.61 | +2.00% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-parallel-buffers-False] |
0.6037 | 0.5918 | -1.98% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[True-backward] |
124.57 | 127.01 | +1.96% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sampler_sample_scale[1000000-cpu] |
97.93 | 99.83 | +1.93% |
benchmarks/test_non_tensor_env_benchmark.py::test_non_tensor_env_rollout_speed[1000-single-False] |
1.5668 | 1.5964 | +1.89% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[True-None] |
1,802 | 1,768 | -1.87% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-False-True] |
34,250 | 34,892 | +1.87% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-1] |
504.07 | 513.50 | +1.87% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-16] |
12.38 | 12.14 | -1.87% |
benchmarks/test_objectives_benchmarks.py::test_values[td_lambda_return_estimate-True-False] |
26.09 | 25.61 | -1.85% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-False-True] |
27,678 | 28,181 | +1.82% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sample_mixed_devices[1000000-memmap_cpu_storage_cpu... |
81.95 | 80.49 | -1.79% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[generalized_advantage_estimate-False-1-512] |
114.84 | 112.81 | -1.77% |
benchmarks/test_envs_benchmark.py::test_transformed |
0.9069 | 0.8910 | -1.76% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-True-0-gru] |
4.2668 | 4.1931 | -1.73% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[True-backward] |
135.84 | 138.18 | +1.72% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[reduce-overhead-None] |
333.54 | 339.25 | +1.71% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-False-32-512] |
557.93 | 567.48 | +1.71% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-True-False-False] |
62,773 | 63,785 | +1.61% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[numpy] |
364,103 | 369,935 | +1.60% |
benchmarks/test_collectors_benchmark.py::test_sync |
16.65 | 16.39 | -1.56% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[reduce-overhead-None] |
116.53 | 118.34 | +1.55% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[50-img_shape0-small] |
4,436 | 4,368 | -1.55% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-False-True-True] |
21,883 | 22,212 | +1.51% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-True-False] |
31,382 | 31,853 | +1.50% |
benchmarks/test_envs_benchmark.py::test_parallel |
0.9480 | 0.9614 | +1.42% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-16] |
5.0841 | 5.0126 | -1.41% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-1] |
472.00 | 478.61 | +1.40% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-True-1-512] |
642.11 | 633.23 | -1.38% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-True-True] |
18,564 | 18,821 | +1.38% |
| ... | ... | ... | Showing 120 of 216 comparisons, sorted by absolute change. |
GPU
Compared 226 benchmarks. Regressions over 5%: 4. Improvements over 5%: 27.
| Benchmark | main ops | PR ops | Change |
|---|---|---|---|
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-ListStorage-RandomSampler-400] |
43.98 | 195.91 | +345.44% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
2,050 | 507.08 | -75.27% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[reduce-overhead-None] |
76.21 | 102.95 | +35.09% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000] |
2,828 | 3,720 | +31.53% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
2,726 | 3,515 | +28.94% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-10000] |
2,993 | 3,738 | +24.89% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[reduce-overhead-None] |
84.80 | 104.78 | +23.55% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-10000] |
3,479 | 2,848 | -18.12% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
1,984 | 2,253 | +13.51% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyMemmapStorage-None-10000] |
2,214 | 1,954 | -11.73% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-sampler6-10000] |
708.62 | 787.02 | +11.06% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyTensorStorage-sampler7-10000] |
700.35 | 774.40 | +10.57% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[100-img_shape1-atari] |
3,539 | 3,900 | +10.18% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-10000] |
2,265 | 2,056 | -9.21% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[True-backward] |
343.62 | 373.45 | +8.68% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
2,602 | 2,818 | +8.33% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[True-backward] |
333.10 | 358.29 | +7.56% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[True-backward] |
340.72 | 365.23 | +7.20% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[reduce-overhead-None] |
804.38 | 861.40 | +7.09% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[False-backward] |
265.93 | 284.48 | +6.98% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-1] |
587.62 | 624.09 | +6.21% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-16] |
40.47 | 42.92 | +6.08% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[False-backward] |
142.43 | 150.92 | +5.97% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[False-backward] |
124.87 | 132.26 | +5.92% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[True-backward] |
377.02 | 399.03 | +5.84% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[100-img_shape2-large_img] |
392.76 | 415.54 | +5.80% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[True-backward] |
236.40 | 249.72 | +5.64% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-gru] |
21.43 | 22.59 | +5.43% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_stack_then_write[200-img_shape3-large_batch] |
134.65 | 141.68 | +5.22% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-400] |
487.68 | 512.25 | +5.04% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-LazyMemmapStorage-RandomSampler-10000] |
2,728 | 2,865 | +5.01% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sampler_sample_scale[1000000-cuda] |
2,141 | 2,248 | +4.97% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[100-img_shape2-large_img] |
568.52 | 542.04 | -4.66% |
benchmarks/test_rnn_reset_backends_benchmark.py::test_rnn_rollout_with_intermediate_resets[b256-t128-i32-h512-scan-False-0-lstm] |
20.40 | 21.29 | +4.37% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[True-None] |
715.94 | 746.96 | +4.33% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-16] |
17.50 | 18.25 | +4.26% |
benchmarks/test_envs_benchmark.py::test_simple |
1.2317 | 1.1800 | -4.19% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-64] |
12.09 | 12.58 | +4.08% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-4] |
69.69 | 72.48 | +4.01% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[pickle] |
11,923 | 12,401 | +4.00% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[True-backward] |
323.33 | 335.99 | +3.92% |
benchmarks/test_objectives_benchmarks.py::test_values[td_lambda_return_estimate-True-False] |
11.64 | 12.09 | +3.90% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape1-atari] |
643.73 | 668.08 | +3.78% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[True-None] |
709.82 | 736.05 | +3.70% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-224-224-64] |
4.4215 | 4.5838 | +3.67% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-False-False] |
51,391 | 49,602 | -3.48% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[True-backward] |
891.52 | 860.60 | -3.47% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-4] |
19.66 | 20.33 | +3.40% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[False-backward] |
66.64 | 68.81 | +3.27% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[False-backward] |
67.65 | 69.84 | +3.24% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-False-1-512] |
1,360 | 1,316 | -3.22% |
benchmarks/test_envs_benchmark.py::test_transformed |
0.7083 | 0.6858 | -3.19% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-True-True] |
19,100 | 18,497 | -3.16% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-False-False-True] |
38,165 | 39,357 | +3.12% |
benchmarks/test_envs_benchmark.py::test_cat_frames_functional[16-constant] |
4,722 | 4,575 | -3.11% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-100000-10000-100-True] |
22.28 | 22.97 | +3.05% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-True-True-False] |
30,040 | 29,128 | -3.04% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[reduce-overhead-None] |
1,893 | 1,836 | -3.03% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyTensorStorage-RandomSampler-400] |
1,001 | 1,031 | +3.02% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[True-backward] |
270.07 | 278.14 | +2.99% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-True-True] |
18,395 | 17,853 | -2.95% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-False-True] |
31,104 | 30,192 | -2.93% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-16] |
4.9434 | 5.0848 | +2.86% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-False-False] |
44,497 | 45,757 | +2.83% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-True-False-False] |
59,084 | 57,412 | -2.83% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-256-256-64] |
10.55 | 10.85 | +2.81% |
benchmarks/test_compressed_storage_benchmark.py::TestCompressedStorageBenchmark::test_tensor_to_bytestream_speed[safetensors] |
22,750 | 23,385 | +2.79% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-64] |
1.2429 | 1.2775 | +2.79% |
benchmarks/test_collectors_benchmark.py::test_single_with_rb_pixels |
5.3419 | 5.1974 | -2.71% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_stack_then_write[100-img_shape2-large_img] |
170.01 | 174.57 | +2.68% |
benchmarks/test_collectors_benchmark.py::test_sync |
10.28 | 10.55 | +2.68% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[False-None] |
95.81 | 98.33 | +2.63% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-ListStorage-RandomSampler-4000] |
163.81 | 168.08 | +2.61% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-224-224-16] |
48.19 | 49.44 | +2.60% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-64] |
3.0182 | 3.0961 | +2.58% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyMemmapStorage-SamplerWithoutReplacement-400] |
520.02 | 533.20 | +2.53% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-480-640-1] |
79.00 | 81.00 | +2.53% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_without_rb_cuda[100-img_shape0-atari] |
17.14 | 17.55 | +2.41% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000] |
159.42 | 163.27 | +2.41% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sample_mixed_devices[1000000-cuda_storage_cuda_samp... |
1,463 | 1,498 | +2.40% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictPrioritizedReplayBuffer-ListStorage-None-4000] |
160.09 | 163.90 | +2.38% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-True-False] |
32,486 | 31,728 | -2.33% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_sample[TensorDictReplayBuffer-ListStorage-RandomSampler-4000] |
159.41 | 163.12 | +2.33% |
benchmarks/test_replaybuffer_benchmark.py::TestPrioritizedReplayBufferBenchmark::test_sample_mixed_devices[1000000-cuda_storage_cpu_sampler] |
86.85 | 88.78 | +2.22% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[False-None] |
224.24 | 229.22 | +2.22% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_iterate[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-4000] |
167.13 | 170.81 | +2.20% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictPrioritizedReplayBuffer-LazyTensorStorage-None-400] |
846.84 | 865.42 | +2.19% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-16] |
12.06 | 12.33 | +2.19% |
benchmarks/test_storage_write_benchmark.py::TestCollectorIntegrationBenchmark::test_collector_without_rb[100-img_shape0-atari] |
30.29 | 29.64 | -2.16% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_lazystack[100-img_shape1-atari] |
700.32 | 685.33 | -2.14% |
benchmarks/test_objectives_benchmarks.py::test_ppo_speed[reduce-overhead-None] |
832.89 | 850.25 | +2.08% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-False-False-False] |
50,742 | 49,688 | -2.08% |
benchmarks/test_envs_benchmark.py::test_serial |
0.4234 | 0.4320 | +2.03% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-64] |
7.0421 | 7.1851 | +2.03% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-LazyTensorStorage-SamplerWithoutReplacement-400] |
1,014 | 1,035 | +2.00% |
benchmarks/test_objectives_benchmarks.py::test_iql_speed[True-None] |
503.72 | 513.53 | +1.95% |
benchmarks/test_objectives_benchmarks.py::test_redq_deprec_speed[False-None] |
96.70 | 98.58 | +1.95% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-False-False] |
55,457 | 54,383 | -1.94% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-False-False-False-True] |
34,378 | 33,717 | -1.92% |
benchmarks/test_objectives_benchmarks.py::test_values[vec_generalized_advantage_estimate-True-True] |
477.08 | 486.17 | +1.91% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_storage_write_contiguous[50-img_shape0-small] |
5,934 | 5,822 | -1.88% |
benchmarks/test_objectives_benchmarks.py::test_reinforce_speed[reduce-overhead-None] |
121.41 | 119.14 | -1.87% |
benchmarks/test_objectives_benchmarks.py::test_ddpg_speed[False-None] |
338.62 | 332.32 | -1.86% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[False-backward] |
74.57 | 75.95 | +1.85% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-False-32-512] |
1,316 | 1,292 | -1.82% |
benchmarks/test_objectives_benchmarks.py::test_a2c_speed[False-None] |
272.28 | 277.23 | +1.82% |
benchmarks/test_objectives_benchmarks.py::test_gae_speed[vec_generalized_advantage_estimate-True-1-512] |
1,232 | 1,255 | +1.81% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[torchvision-480-640-1] |
465.72 | 474.14 | +1.81% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-False-False-True-False] |
27,684 | 27,184 | -1.81% |
benchmarks/test_objectives_benchmarks.py::test_dqn_speed[False-backward] |
447.85 | 455.92 | +1.80% |
benchmarks/test_objectives_benchmarks.py::test_values[td0_return_estimate-False-False] |
11,460 | 11,663 | +1.77% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-4] |
48.03 | 48.87 | +1.75% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_extend_sample[ReplayBuffer-LazyTensorStorage-RandomSampler-1000000-10000-100-False] |
47.59 | 46.77 | -1.72% |
benchmarks/test_storage_write_benchmark.py::TestStorageWriteBenchmark::test_collector_lazystack_then_write[100-img_shape2-large_img] |
392.77 | 399.40 | +1.69% |
benchmarks/test_replaybuffer_benchmark.py::test_rb_populate[TensorDictReplayBuffer-ListStorage-SamplerWithoutReplacement-400] |
190.31 | 193.51 | +1.68% |
benchmarks/test_objectives_benchmarks.py::test_sac_speed[reduce-overhead-None] |
99.88 | 98.21 | -1.67% |
benchmarks/test_vla_preprocessing_benchmark.py::test_openvla_preprocessing_throughput[pil-256-256-1] |
189.54 | 192.69 | +1.66% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[False-True-True-True-False] |
34,851 | 34,277 | -1.65% |
benchmarks/test_envs_benchmark.py::test_step_mdp_speed[True-True-True-True-True] |
23,758 | 24,145 | +1.63% |
benchmarks/test_objectives_benchmarks.py::test_td3_speed[reduce-overhead-None] |
44.94 | 44.21 | -1.61% |
| ... | ... | ... | Showing 120 of 226 comparisons, sorted by absolute change. |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
1 participant
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
Stack from ghstack (oldest at bottom):