use Docker to build pisa, update results

Makefile

@@ -9,7 +9,7 @@ COMMANDS ?= TOP_10 TOP_100 TOP_1000 TOP_100_COUNT COUNT
 # ENGINES ?= tantivy-0.16 lucene-8.10.1 pisa-0.8.2 bleve-0.8.0-scorch bluge-0.2.2 rucene-0.1
 # ENGINES ?= tantivy-0.16 tantivy-0.17 tantivy-0.18 tantivy-0.19
 # ENGINES ?= tantivy-0.22 tantivy-0.24 tantivy-0.25 tantivy-main lucene-10.3.0 lucene-10.3.0-bp
-ENGINES ?= tantivy-0.26 lucene-10.3.0 lucene-10.3.0-bp pisa-0.8.2 iresearch-26.03.1
+ENGINES ?= tantivy-0.26 lucene-10.4.0 lucene-10.4.0-bp pisa-0.8.2 iresearch-26.03.1
 PORT ?= 8080
 WARMUP_TIME ?= 60
 NUM_ITER ?= 10

README.md

@@ -55,12 +55,11 @@ Out of 10 runs, we only retain the best score, so Garbage Collection likely does
 
 The results file that is included in this repository was generated using the following benchmark
 environment:
- - AWS c7i.2xlarge instance running on us-east-1
- - Processor Intel(R) Xeon(R) Platinum 8488C
- - Amazon Linux 2023
- - Kernel `6.1.148-173.267.amzn2023.x86_64`
- - Rust 1.89
- - Adoptium JDK `Temurin-21.0.8+9`
+ - Processor AMD Ryzen 7 9800X3D
+ - Manjaro Linux (rolling)
+ - Kernel `7.0.3-1-MANJARO`
+ - Rust 1.96.0
+ - OpenJDK 21.0.11
 
 ## Engine specific detail
 
@@ -126,6 +125,23 @@ make serve
 
 And open the following in your browser: [http://localhost:8080/](http://localhost:8080/)
 
+For a quick text summary from `results.json` without opening the browser:
+
+```
+python tools/analyze.py <engine> <command> [--filter <tag>]
+```
+
+- `<engine>` — e.g. `tantivy-main`, `tantivy-0.26`, `iresearch-26.03.1`
+- `<command>` — `TOP_100`, `TOP_100_COUNT`, or `COUNT`
+- `--filter` — optional tag filter: `union`, `intersection`, or `intersection_union`
+
+Examples:
+```
+python tools/analyze.py tantivy-main TOP_100_COUNT
+python tools/analyze.py tantivy-main TOP_100 --filter union
+```
+
+It prints query count, sample count, and avg/min/max/p50/p95/p99 latencies (in µs).
 
 # Adding another search engine
 

engines/iresearch-26.03.1/Makefile

@@ -1,9 +1,11 @@
 ROOT_DIR := $(abspath $(CURDIR)/../..)
 DOCKER_IMAGE ?= iresearch-26.03.1-builder
 DOCKER_TARGETARCH ?= $(shell uname -m | sed -e 's/^x86_64$$/amd64/' -e 's/^aarch64$$/arm64/')
+HOST_OS := $(shell uname -s)
+RUN_IN_DOCKER ?= $(if $(filter Darwin,$(HOST_OS)),1,0)
 
 submodules:
-	@git -C serenedb submodule update --init --recursive --jobs $$(nproc)
+	@git -C serenedb submodule update --init --recursive --jobs $$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 1)
 
 clean:
 	@echo "--- Cleaning iresearch ---"
@@ -44,10 +46,18 @@ compile_use_pgo:
 index:
 	@echo "--- Indexing iresearch ---"
 	@mkdir -p idx
+ifeq ($(RUN_IN_DOCKER),1)
+	@docker run --rm --user "$$(id -u):$$(id -g)" -v "$(CURDIR):/work" -w /work -i $(DOCKER_IMAGE) ./build/bin/search-benchmark-game-build < ${CORPUS}
+else
 	@./build/bin/search-benchmark-game-build < ${CORPUS}
+endif
 
 serve:
+ifeq ($(RUN_IN_DOCKER),1)
+	@docker run --rm --user "$$(id -u):$$(id -g)" -v "$(CURDIR):/work" -w /work -i $(DOCKER_IMAGE) ./build/bin/search-benchmark-game-query idx
+else
 	@./build/bin/search-benchmark-game-query idx
+endif
 
 serve_perf:
 	@perf record -g -F 997 -o perf.data -- ./build/bin/search-benchmark-game-query idx

engines/lucene-10.4.0-bp/src/main/java/BuildIndex.java

@@ -86,7 +86,7 @@ public static void main(String[] args) throws Exception {
 						final JsonObject parsed_doc = Json.parse(line).asObject();
 						final String id = parsed_doc.get("id").asString();
 						final String text = parsed_doc.get("text").asString();
-						final long sortValue = parsed_doc.get("sort_field").asLong();
+						final long sortValue = parsed_doc.getLong("sort_field", 0L);
 						idField.setStringValue(id);
 						textField.setStringValue(text);
 						sortField.setLongValue(sortValue);

engines/lucene-10.4.0/src/main/java/BuildIndex.java

@@ -79,7 +79,7 @@ public static void main(String[] args) throws Exception {
 						final JsonObject parsed_doc = Json.parse(line).asObject();
 						final String id = parsed_doc.get("id").asString();
 						final String text = parsed_doc.get("text").asString();
-						final long sortValue = parsed_doc.get("sort_field").asLong();
+						final long sortValue = parsed_doc.getLong("sort_field", 0L);
 						idField.setStringValue(id);
 						textField.setStringValue(text);
 						sortField.setLongValue(sortValue);

engines/pisa-0.8.2/.dockerignore

@@ -0,0 +1,2 @@
+*
+!Dockerfile

engines/pisa-0.8.2/Makefile

@@ -1,6 +1,16 @@
+ROOT_DIR := $(abspath $(CURDIR)/../..)
+DOCKER_IMAGE ?= pisa-0.8.2-builder
+HOST_OS := $(shell uname -s)
+RUN_IN_DOCKER ?= $(if $(filter Darwin,$(HOST_OS)),1,0)
+TBB_LIB := $(CURDIR)/build/pisa/external/tbb_cmake_build/tbb_cmake_build_subdir_release
+CONTAINER_TBB_LIB := /home/pisa/src/build/pisa/external/tbb_cmake_build/tbb_cmake_build_subdir_release
+
 help:
 	@grep '^[^#[:space:]].*:' Makefile
 
+submodules:
+	@git -C $(ROOT_DIR) submodule update --init --recursive engines/pisa-0.8.2/pisa
+
 clean:
 	@echo "--- Cleaning PISA 0.8.2 ---"
 	@rm -fr idx
@@ -12,24 +22,32 @@ clean:
 compile-within-container: build/bin/build_index build/bin/do_query
 	echo "compiling"
 
-compile:
-	docker build -t pisa-builder .
-	docker run --rm -it -v "$(shell pwd)":/home/pisa/src/ pisa-builder
+compile: submodules
+	docker build -t $(DOCKER_IMAGE) .
+	@rm -rf build
+	# Run a throwaway builder container with this directory mounted as /home/pisa/src.
+	# --rm removes the stopped container; --user avoids root-owned build outputs.
+	docker run --rm --user "$$(id -u):$$(id -g)" -v "$(CURDIR)":/home/pisa/src/ $(DOCKER_IMAGE)
+	@rm -f pisa/external/tbb/cmake/TBBConfig.cmake pisa/external/tbb/cmake/TBBConfigVersion.cmake
 
 index: idx
 
-TBB_LIB := $(shell pwd)/build/pisa/external/tbb_cmake_build/tbb_cmake_build_subdir_release
-
 serve: build/bin/do_query
+ifeq ($(RUN_IN_DOCKER),1)
+	@docker run --rm --user "$$(id -u):$$(id -g)" -v "$(CURDIR)":/home/pisa/src/ -i --entrypoint bash $(DOCKER_IMAGE) -lc 'LD_LIBRARY_PATH=$(CONTAINER_TBB_LIB) ./build/bin/do_query idx'
+else
 	@LD_LIBRARY_PATH=$(TBB_LIB) ./build/bin/do_query idx
+endif
 
-build/bin/%:
-	@echo "--- Building PISA 0.8.2 ---"
-	@mkdir -p build
-	@cmake -B build/ .
-	@make -C build/
+build/bin/%: submodules
+	@$(MAKE) compile >&2
 
 idx: build/bin/build_index
 	@echo "--- Indexing PISA 0.8.2 ---"
+	@rm -rf idx
 	@mkdir -p idx
+ifeq ($(RUN_IN_DOCKER),1)
+	@docker run --rm --user "$$(id -u):$$(id -g)" -v "$(CURDIR)":/home/pisa/src/ -i --entrypoint bash $(DOCKER_IMAGE) -lc 'LD_LIBRARY_PATH=$(CONTAINER_TBB_LIB) ./build/bin/build_index' < ${CORPUS}
+else
 	@LD_LIBRARY_PATH=$(TBB_LIB) ./build/bin/build_index < ${CORPUS}
+endif

engines/tantivy-main/Cargo.toml

@@ -8,7 +8,7 @@ edition = "2018"
 serde = "1.0"
 serde_derive = "1.0"
 serde_json = "1.0"
-tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "129c40f" }
+tantivy = { git = "https://github.com/quickwit-oss/tantivy.git", rev = "46b3fb9" }
 env_logger = "0.5"
 futures = "*"
 

tools/analyze.py

@@ -0,0 +1,117 @@
+#!/usr/bin/env python3
+"""
+Analyze results.json from the search benchmark game.
+
+Usage:
+    python tools/analyze.py <engine> <command> [--filter <tag>]
+
+Arguments:
+    engine    Engine name (e.g., tantivy-main, tantivy-0.26, iresearch-26.03.1)
+    command   Benchmark command (e.g., TOP_100, TOP_100_COUNT, COUNT)
+    --filter  Optional tag filter: 'union', 'intersection', or 'intersection_union'
+              Filters entries whose tags contain the given string.
+
+Examples:
+    python tools/analyze.py tantivy-main TOP_100_COUNT
+    python tools/analyze.py tantivy-main TOP_100 --filter intersection
+    python tools/analyze.py tantivy-0.26 TOP_100_COUNT --filter union
+"""
+
+import argparse
+import json
+import statistics
+import sys
+from pathlib import Path
+
+
+def load_results(path: Path) -> dict:
+    with open(path) as f:
+        return json.load(f)
+
+
+def summarise(entries: list[dict]) -> dict:
+    """Compute summary statistics over all duration values across entries."""
+    all_durations = []
+    for entry in entries:
+        all_durations.extend(entry["duration"])
+
+    if not all_durations:
+        return {"count": 0, "query_count": 0}
+
+    sorted_d = sorted(all_durations)
+    n = len(sorted_d)
+    return {
+        "query_count": len(entries),
+        "sample_count": n,
+        "avg": statistics.mean(sorted_d),
+        "min": sorted_d[0],
+        "max": sorted_d[-1],
+        "p50": sorted_d[n // 2],
+        "p95": sorted_d[int(n * 0.95)],
+        "p99": sorted_d[int(n * 0.99)],
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="Analyze search benchmark results.json"
+    )
+    parser.add_argument("engine", help="Engine name (e.g., tantivy-main)")
+    parser.add_argument("command", help="Benchmark command (e.g., TOP_100_COUNT)")
+    parser.add_argument(
+        "--filter", "-f",
+        dest="tag_filter",
+        help="Filter entries by tag (e.g., union, intersection, intersection_union)",
+    )
+    args = parser.parse_args()
+
+    results_path = Path("results.json")
+    if not results_path.exists():
+        print("Error: results.json not found. Run from the project root.", file=sys.stderr)
+        sys.exit(1)
+
+    data = load_results(results_path)
+
+    # Validate command
+    if args.command not in data["results"]:
+        print(f"Error: unknown command '{args.command}'.", file=sys.stderr)
+        print(f"  Available: {list(data['results'].keys())}", file=sys.stderr)
+        sys.exit(1)
+
+    results = data["results"][args.command]
+
+    # Validate engine
+    if args.engine not in results:
+        print(f"Error: unknown engine '{args.engine}'.", file=sys.stderr)
+        print(f"  Available: {list(results.keys())}", file=sys.stderr)
+        sys.exit(1)
+
+    entries = results[args.engine]
+
+    # Optional tag filter
+    if args.tag_filter:
+        tag = args.tag_filter
+        entries = [e for e in entries if any(tag in t for t in e["tags"])]
+        if not entries:
+            print(f"No entries matched filter '{tag}'.", file=sys.stderr)
+            sys.exit(1)
+
+    summary = summarise(entries)
+
+    # Display
+    filter_info = f" (filter: {args.tag_filter})" if args.tag_filter else ""
+    print(f"Engine:  {args.engine}")
+    print(f"Command: {args.command}{filter_info}")
+    print(f"Queries: {summary['query_count']}")
+    print(f"Samples: {summary['sample_count']} ({summary['sample_count'] // summary['query_count']} per query)")
+    print(f"---")
+    print(f"Avg:  {summary['avg']:>10.1f} µs")
+    print(f"Min:  {summary['min']:>10.1f} µs")
+    print(f"Max:  {summary['max']:>10.1f} µs")
+    print(f"P50:  {summary['p50']:>10.1f} µs")
+    print(f"P95:  {summary['p95']:>10.1f} µs")
+    print(f"P99:  {summary['p99']:>10.1f} µs")
+
+
+if __name__ == "__main__":
+    main()