Benchmarks for large sets with 10000 lines

Author: tisnik

tests/benchmarks/test_token_estimator.py

@@ -123,6 +123,21 @@ def test_lorem_ipsum_times_1000_times(benchmark: BenchmarkFixture) -> None:
     benchmark(estimate_tokens, input_string)
 
 
+def test_lorem_ipsum_times_2000_times(benchmark: BenchmarkFixture) -> None:
+    """Benchmark for tokenizing Lorem Ipsum text repeated 2000x.
+
+    Parameters:
+    ----------
+        benchmark (BenchmarkFixture): pytest-benchmark fixture.
+
+    Returns:
+    -------
+        None
+    """
+    input_string = LOREM_IPSUM * 2000
+    benchmark(estimate_tokens, input_string)
+
+
 def benchmark_file_tokenization(benchmark: BenchmarkFixture, filename: str) -> None:
     """Read the given file and tokenize it as benchmark.
 
@@ -184,6 +199,20 @@ def test_xml_file_1000_lines(benchmark: BenchmarkFixture) -> None:
     benchmark_file_tokenization(benchmark, "xml_1000_lines.xml")
 
 
+def test_xml_file_10000_lines(benchmark: BenchmarkFixture) -> None:
+    """Test tokenizing XML file containing just 10000 lines.
+
+    Parameters:
+    ----------
+        benchmark (BenchmarkFixture): pytest-benchmark fixture.
+
+    Returns:
+    -------
+        None
+    """
+    benchmark_file_tokenization(benchmark, "xml_10000_lines.xml")
+
+
 def test_yaml_file_10_lines(benchmark: BenchmarkFixture) -> None:
     """Test tokenizing YAML file containing just 10 lines.
 
@@ -226,6 +255,20 @@ def test_yaml_file_1000_lines(benchmark: BenchmarkFixture) -> None:
     benchmark_file_tokenization(benchmark, "yaml_1000_lines.yml")
 
 
+def test_yaml_file_10000_lines(benchmark: BenchmarkFixture) -> None:
+    """Test tokenizing YAML file containing just 10000 lines.
+
+    Parameters:
+    ----------
+        benchmark (BenchmarkFixture): pytest-benchmark fixture.
+
+    Returns:
+    -------
+        None
+    """
+    benchmark_file_tokenization(benchmark, "yaml_10000_lines.yml")
+
+
 def test_json_file_10_lines(benchmark: BenchmarkFixture) -> None:
     """Test tokenizing JSON file containing just 10 lines.
 
@@ -268,6 +311,20 @@ def test_json_file_1000_lines(benchmark: BenchmarkFixture) -> None:
     benchmark_file_tokenization(benchmark, "json_1000_lines.json")
 
 
+def test_json_file_10000_lines(benchmark: BenchmarkFixture) -> None:
+    """Test tokenizing JSON file containing just 10000 lines.
+
+    Parameters:
+    ----------
+        benchmark (BenchmarkFixture): pytest-benchmark fixture.
+
+    Returns:
+    -------
+        None
+    """
+    benchmark_file_tokenization(benchmark, "json_10000_lines.json")
+
+
 def test_python_source_10_lines(benchmark: BenchmarkFixture) -> None:
     """Test tokenizing Python script containing just 10 lines.
 
@@ -310,6 +367,20 @@ def test_python_source_1000_lines(benchmark: BenchmarkFixture) -> None:
     benchmark_file_tokenization(benchmark, "python_1000_lines.py")
 
 
+def test_python_source_10000_lines(benchmark: BenchmarkFixture) -> None:
+    """Test tokenizing Python script containing just 10000 lines.
+
+    Parameters:
+    ----------
+        benchmark (BenchmarkFixture): pytest-benchmark fixture.
+
+    Returns:
+    -------
+        None
+    """
+    benchmark_file_tokenization(benchmark, "python_10000_lines.py")
+
+
 def test_javascript_source_10_lines(benchmark: BenchmarkFixture) -> None:
     """Test tokenizing JavaScript script containing just 10 lines.
 
@@ -350,3 +421,17 @@ def test_javascript_source_1000_lines(benchmark: BenchmarkFixture) -> None:
         None
     """
     benchmark_file_tokenization(benchmark, "js_1000_lines.js")
+
+
+def test_javascript_source_10000_lines(benchmark: BenchmarkFixture) -> None:
+    """Test tokenizing JavaScript script containing just 10000 lines.
+
+    Parameters:
+    ----------
+        benchmark (BenchmarkFixture): pytest-benchmark fixture.
+
+    Returns:
+    -------
+        None
+    """
+    benchmark_file_tokenization(benchmark, "js_10000_lines.js")