Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions docs/running_tests/releases.md
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ follow the same layout.

When filling with `--gas-benchmark-values`, benchmark tests additionally
include the gas limit in the subdirectory name (`for_{fork}_at_{gas}M`,
where `{gas}` is in millions, zero-padded to four digits), with one
subdirectory per gas value:
where `{gas}` is in millions, zero-padded to at least four digits), with
one subdirectory per gas value:

```text
fixtures/
Expand Down
2 changes: 1 addition & 1 deletion docs/writing_tests/benchmarks.md
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ This mode is designed for gas limit testing, and gas repricing, where it enables

**Note:** For both benchmark modes, users may supply multiple values in a single invocation. For example:

- `--gas-benchmark-values 1,2,3` runs the test with 1M, 2M, and 3M block gas limits
- `--gas-benchmark-values 0.5,1,10` runs the test with 0.5M, 1M, and 10M block gas limits
- `--fixed-opcode-count 4,5` runs the test with approximately 4K and 5K opcode executions

**Output layout with gas benchmark values:** When `--gas-benchmark-values` is provided and benchmark tests are filled, fixtures are written into per‑fork, per‑gas‑limit subdirectories under each format directory:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
FORK_SUBDIR_PREFIX,
SUBFOLDER_LEVEL_SEPARATOR,
format_fork_subdir,
format_gas_limit_prefix,
)

# EVM binary for fill tests; defaults to geth evm
Expand Down Expand Up @@ -200,6 +201,33 @@ def test_benchmarking_mode_configured_with_option(
assert any("benchmark-gas-value_30M" in line for line in result.outlines)


def test_benchmarking_mode_with_float_gas_values(
pytester: pytest.Pytester,
) -> None:
"""Test that float --gas-benchmark-values are accepted and formatted."""
setup_test_directory_structure(
pytester, test_module_dummy, "test_dummy_benchmark.py"
)

result = pytester.runpytest(
"-c",
"pytest-fill.ini",
"--fork",
"Prague",
"--gas-benchmark-values",
"0.5,1.5,10",
"tests/benchmark/dummy_test_module/",
"--collect-only",
"-q",
)

assert result.ret == 0
assert any("6 tests collected" in line for line in result.outlines)
assert any("benchmark-gas-value_0.5M" in line for line in result.outlines)
assert any("benchmark-gas-value_1.5M" in line for line in result.outlines)
assert any("benchmark-gas-value_10M" in line for line in result.outlines)


def test_benchmark_gas_values_split_into_subdirs(
pytester: pytest.Pytester, tmp_path: Path
) -> None:
Expand Down Expand Up @@ -1209,3 +1237,20 @@ def test_fork_split_example(state_test, pre) -> None:
assert "fork_Prague" not in key, (
f"Unexpected fork_Prague in key {key} ({file_path})"
)


@pytest.mark.parametrize(
"values,expected",
[
([1.0, 10.0, 60.0], ["0001M", "0010M", "0060M"]),
([0.5, 1.0, 10.0], ["0000.5M", "0001.0M", "0010.0M"]),
([0.25, 0.5, 1.0], ["0000.25M", "0000.50M", "0001.00M"]),
],
ids=["integers", "mixed", "fractional"],
)
def test_format_gas_limit_prefix(
values: List[float], expected: List[str]
) -> None:
"""Test gas limit prefix formatting for int and float values."""
result = [format_gas_limit_prefix(v, values) for v in values]
assert result == expected
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
import re
from abc import ABC, abstractmethod
from pathlib import Path
from typing import ClassVar, Dict, List, Self
from typing import Annotated, ClassVar, Dict, List, Self

import pytest
from annotated_types import Gt
from pydantic import BaseModel, Field, RootModel

from execution_testing.test_types import Environment, EnvironmentDefaults
Expand Down Expand Up @@ -33,7 +34,7 @@ def pytest_addoption(parser: pytest.Parser) -> None:
default=None,
help=(
"Gas limits (in millions) for benchmark tests. "
"Example: '100,500' runs tests with 100M and 500M gas. "
"Example: '0.5,1,10' runs tests with 0.5M, 1M and 10M gas. "
"Benchmark outputs are grouped under "
f"{FORK_SUBDIR_PREFIX}{{fork}}"
f"{SUBFOLDER_LEVEL_SEPARATOR}XXXXM/ subdirectories. "
Expand Down Expand Up @@ -174,7 +175,7 @@ def parametrize(self, metafunc: pytest.Metafunc) -> None:
class GasBenchmarkValues(RootModel, BenchmarkParametrizer):
"""Gas benchmark values configuration object."""

root: List[int]
root: List[Annotated[float, Gt(0)]]

flag: ClassVar[str] = "--gas-benchmark-values"
config_field: ClassVar[str] = "_gas_benchmark_values_config"
Expand All @@ -191,8 +192,8 @@ def get_test_parameters(self, _test_name: str) -> list[ParameterSet]:
"""Get benchmark values. All tests have the same list."""
return [
pytest.param(
gas_value * 1_000_000,
id=f"benchmark-gas-value_{gas_value}M",
int(gas_value * 1_000_000),
id=f"benchmark-gas-value_{gas_value:g}M",
marks=[
pytest.mark.fixture_subfolder(
level=1,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -315,12 +315,22 @@ def from_config(cls, config: pytest.Config) -> "FixtureOutput":


def format_gas_limit_prefix(
gas_value_millions: int, all_values_millions: list[int]
gas_value_millions: float, all_values_millions: list[float]
) -> str:
"""Return a stable, sortable gas-limit prefix for a fixture subfolder."""
max_value = max(all_values_millions) if all_values_millions else 0
width = max(4, len(str(max_value)))
return f"{gas_value_millions:0{width}d}M"
int_width = max(4, len(str(int(max_value))))
# Find max decimal places across all fractional values for
# consistent padding (ensures lexicographic sortability).
max_decimals = 0
for v in all_values_millions:
label = f"{v:g}"
if "." in label:
max_decimals = max(max_decimals, len(label.split(".")[1]))
if gas_value_millions == int(gas_value_millions) and max_decimals == 0:
return f"{int(gas_value_millions):0{int_width}d}M"
total_width = int_width + 1 + max_decimals # digits + dot + decimals
return f"{gas_value_millions:0{total_width}.{max_decimals}f}M"


def format_fork_subdir(
Expand Down
Loading