add three iterations

Author: aadland6

.github/workflows/partition-benchmark.yaml

@@ -13,15 +13,12 @@ permissions:
 env:
   NLTK_DATA: ${{ github.workspace }}/nltk_data
   PYTHON_VERSION: "3.12"
-  # Number of timed iterations per file.
-  # Kept at 1 in CI to minimise runner time; raise locally for more stable averages.
-  NUM_ITERATIONS: "1"
-  # Fraction by which the current total may exceed the stored best before the
-  # job is marked as failed. 0.20 = 20%.
+  # Number of times to run the full benchmark suite.
+  NUM_ITERATIONS: "3"
+  # 20% threshold for now and tune later
   REGRESSION_THRESHOLD: "0.20"
-  # Increment to bust all stored caches (e.g. after a deliberate performance
-  # trade-off is accepted).
-  CACHE_VERSION: "v1"
+  # Increment to change cache key when benchmark-affecting dependencies are updated, to ensure clean slate runs.
+  CACHE_VERSION: "v2"
   # S3 location for metrics – matches core-product convention.
   S3_METRICS_BUCKET_KEY: utic-metrics/ci-metrics
   S3_BENCHMARK_PATH: open-source/partition-benchmark/benchmark_best.json
@@ -41,9 +38,7 @@ jobs:
     needs: [setup]
 
     steps:
-      # ------------------------------------------------------------------ #
-      # 1. Source & Python environment                                      #
-      # ------------------------------------------------------------------ #
+
       - uses: actions/checkout@v4
 
       - uses: ./.github/actions/base-cache
@@ -58,11 +53,7 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y tesseract-ocr tesseract-ocr-kor
 
-      # ------------------------------------------------------------------ #
-      # 2. Cache HuggingFace models                                        #
-      #    hi_res downloads layout-detection models on first use.          #
-      #    Caching keeps benchmark timings stable across runs.             #
-      # ------------------------------------------------------------------ #
+
       - name: Restore HuggingFace model cache
         uses: actions/cache/restore@v4
         with:
@@ -72,10 +63,7 @@ jobs:
             hf-models-${{ runner.os }}-${{ env.CACHE_VERSION }}-
             hf-models-${{ runner.os }}-
 
-      # ------------------------------------------------------------------ #
-      # 3. Run the benchmark                                               #
-      #    Writes per-file timings + __total__ to benchmark_results.json.  #
-      # ------------------------------------------------------------------ #
+
       - name: Run partition benchmark
         env:
           NUM_ITERATIONS: ${{ env.NUM_ITERATIONS }}
@@ -89,10 +77,7 @@ jobs:
           path: ~/.cache/huggingface
           key: hf-models-${{ runner.os }}-${{ env.CACHE_VERSION }}-${{ github.sha }}
 
-      # ------------------------------------------------------------------ #
-      # 4. Download the stored best runtime from S3                        #
-      #    continue-on-error: first run will have nothing stored yet.      #
-      # ------------------------------------------------------------------ #
+
       - name: Download previous best from S3
         continue-on-error: true
         env:
@@ -103,13 +88,7 @@ jobs:
             "s3://${{ env.S3_METRICS_BUCKET_KEY }}/${{ env.S3_BENCHMARK_PATH }}" \
             benchmark_best.json
 
-      # ------------------------------------------------------------------ #
-      # 5. Compare results; fail on regression                             #
-      #    compare_benchmark.py:                                           #
-      #      - Logs a per-file table with deltas.                          #
-      #      - Exits 1 if current > best * (1 + threshold).               #
-      #      - Overwrites benchmark_best.json when current is faster.      #
-      # ------------------------------------------------------------------ #
+
       - name: Compare results against stored best
         id: compare
         run: |
@@ -118,12 +97,7 @@ jobs:
             benchmark_best.json \
             ${{ env.REGRESSION_THRESHOLD }}
 
-      # ------------------------------------------------------------------ #
-      # 6. Upload the (possibly updated) best result back to S3            #
-      #    compare_benchmark.py only overwrites benchmark_best.json when   #
-      #    the current run is strictly faster, so the true minimum is      #
-      #    preserved.                                                      #
-      # ------------------------------------------------------------------ #
+
       - name: Upload best result to S3
         continue-on-error: true
         env:
@@ -134,9 +108,7 @@ jobs:
             benchmark_best.json \
             "s3://${{ env.S3_METRICS_BUCKET_KEY }}/${{ env.S3_BENCHMARK_PATH }}"
 
-      # ------------------------------------------------------------------ #
-      # 7. Upload artifacts – always, so regressions can be inspected      #
-      # ------------------------------------------------------------------ #
+
       - name: Upload benchmark artifacts
         if: always()
         uses: actions/upload-artifact@v4

scripts/performance/benchmark_partition.py

@@ -30,9 +30,10 @@
     ("example-docs/pdf/reliance.pdf", "hi_res"),
     ("example-docs/pdf/pdf-with-ocr-text.pdf", "hi_res"),
     ("example-docs/pdf/layout-parser-paper.pdf", "hi_res"),
-    ("example-docs/pdf/layout-parser-paper-with-table.pdf", "hi_res"),
+    ("example-docs/pdf/list-item-example.pdf", "hi_res"),
     # Images – hi_res
     ("example-docs/embedded-images-tables.jpg", "hi_res"),
+    ("example-docs/double-column-A.jpg", "hi_res"),
     ("example-docs/double-column-B.jpg", "hi_res"),
     ("example-docs/layout-parser-paper-fast.jpg", "hi_res"),
     # Other document types – fast