Makefile

# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Version parameters (can be overridden from command line)
# Example: make install SPARK_VERSION=3.5 SCALA_VERSION=2.13
SPARK_VERSION ?= 3.5
SCALA_VERSION ?= 2.12

# Derived module names
MODULE := lance-spark-$(SPARK_VERSION)_$(SCALA_VERSION)
BUNDLE_MODULE := lance-spark-bundle-$(SPARK_VERSION)_$(SCALA_VERSION)
BASE_MODULE := lance-spark-base_$(SCALA_VERSION)

# Spark download versions for Docker
include docker/versions.mk
SPARK_DOWNLOAD_VERSION := $(SPARK_DOWNLOAD_VERSION_$(SPARK_VERSION))
PY4J_VERSION := $(PY4J_VERSION_$(SPARK_VERSION))

# Spark 3.x default binaries are Scala 2.12; Scala 2.13 needs explicit suffix.
# Spark 4.x only supports Scala 2.13, so no suffix is needed.
ifeq ($(SCALA_VERSION),2.13)
  ifeq ($(filter 4.%,$(SPARK_VERSION)),)
    SPARK_SCALA_SUFFIX := -scala2.13
  else
    SPARK_SCALA_SUFFIX :=
  endif
else
  SPARK_SCALA_SUFFIX :=
endif

# Optional Docker build cache flags (set in CI for layer caching)
# Example: make docker-build-test-base DOCKER_CACHE_FROM="type=gha" DOCKER_CACHE_TO="type=gha,mode=max"
DOCKER_CACHE_FROM ?=
DOCKER_CACHE_TO ?=
LANCE_NAMESPACE_IMPL_VERSION ?= $(shell sed -n 's:.*<lance-namespace-impl.version>\(.*\)</lance-namespace-impl.version>.*:\1:p' pom.xml | head -n 1)

DOCKER_COMPOSE := $(shell \
	if docker compose version >/dev/null 2>&1; then \
		echo "docker compose"; \
	elif command -v docker-compose >/dev/null 2>&1; then \
		echo "docker-compose"; \
	else \
		echo ""; \
	fi)

# =============================================================================
# Parameterized commands (use SPARK_VERSION and SCALA_VERSION)
# =============================================================================

.PHONY: install
install:
	./mvnw install -pl $(MODULE) -am -DskipTests

.PHONY: test
test:
	./mvnw test -pl $(MODULE)

.PHONY: build
build: lint install

.PHONY: clean-module
clean-module:
	./mvnw clean -pl $(MODULE)

.PHONY: bundle
bundle:
	./mvnw install -pl $(BUNDLE_MODULE) -am -DskipTests

.PHONY: clean-bundle
clean-bundle:
	./mvnw clean install -pl $(BUNDLE_MODULE) -am -DskipTests

.PHONY: install-base
install-base:
	./mvnw install -pl $(BASE_MODULE) -am -DskipTests

# =============================================================================
# Global commands (all modules)
# =============================================================================

.PHONY: lint
lint:
	./mvnw checkstyle:check spotless:check

.PHONY: format
format:
	./mvnw spotless:apply

.PHONY: install-all
install-all:
	./mvnw install -DskipTests

.PHONY: test-all
test-all:
	./mvnw test

.PHONY: build-all
build-all: lint install-all

.PHONY: clean
clean:
	./mvnw clean

# =============================================================================
# Docker commands
# =============================================================================

.PHONY: check-docker-compose
  check-docker-compose:
  ifndef DOCKER_COMPOSE
        $(error Neither 'docker compose' nor 'docker-compose' found. Please install Docker Compose.)
  endif

.PHONY: docker-build
docker-build:
	@ls $(BUNDLE_MODULE)/target/$(BUNDLE_MODULE)-*.jar >/dev/null 2>&1 || \
		(echo "Error: Bundle jar not found. Run 'make bundle' first." && exit 1)
	$(DOCKER_COMPOSE) -f docker/docker-compose.yml build --no-cache \
		--build-arg SPARK_DOWNLOAD_VERSION=$(SPARK_DOWNLOAD_VERSION) \
		--build-arg SPARK_MAJOR_VERSION=$(SPARK_VERSION) \
		--build-arg SCALA_VERSION=$(SCALA_VERSION) \
		--build-arg SPARK_SCALA_SUFFIX=$(SPARK_SCALA_SUFFIX) \
		spark-lance

.PHONY: docker-up
docker-up: check-docker-compose
	${DOCKER_COMPOSE} -f docker/docker-compose.yml up -d

.PHONY: docker-shell
docker-shell:
	cd docker && docker exec -it spark-lance bash

.PHONY: docker-down
docker-down: check-docker-compose
	${DOCKER_COMPOSE} -f docker/docker-compose.yml down

# Print resolved Docker build args for use in CI (e.g. GitHub Actions step outputs).
# This keeps versions.mk as the single source of truth for version mappings.
.PHONY: print-docker-build-args
print-docker-build-args:
	@echo "spark-download-version=$(SPARK_DOWNLOAD_VERSION)"
	@echo "py4j-version=$(PY4J_VERSION)"
	@echo "spark-scala-suffix=$(SPARK_SCALA_SUFFIX)"
	@echo "lance-namespace-impl-version=$(LANCE_NAMESPACE_IMPL_VERSION)"

.PHONY: docker-build-test-base
docker-build-test-base:
	cd docker && docker buildx build \
		--build-arg SPARK_DOWNLOAD_VERSION=$(SPARK_DOWNLOAD_VERSION) \
		--build-arg SPARK_MAJOR_VERSION=$(SPARK_VERSION) \
		--build-arg SCALA_VERSION=$(SCALA_VERSION) \
		--build-arg PY4J_VERSION=$(PY4J_VERSION) \
		--build-arg SPARK_SCALA_SUFFIX=$(SPARK_SCALA_SUFFIX) \
		$(if $(DOCKER_CACHE_FROM),--cache-from $(DOCKER_CACHE_FROM)) \
		$(if $(DOCKER_CACHE_TO),--cache-to $(DOCKER_CACHE_TO)) \
		--load \
		-f Dockerfile.test-base \
		-t lance-spark-test-base:$(SPARK_VERSION)_$(SCALA_VERSION) \
		.

.PHONY: docker-build-test
docker-build-test:
	@ls $(BUNDLE_MODULE)/target/$(BUNDLE_MODULE)-*.jar >/dev/null 2>&1 || \
		(echo "Error: Bundle jar not found. Run 'make bundle' first." && exit 1)
	docker build --no-cache \
		--build-arg SPARK_MAJOR_VERSION=$(SPARK_VERSION) \
		--build-arg SCALA_VERSION=$(SCALA_VERSION) \
		--build-arg LANCE_NAMESPACE_IMPL_VERSION=$(LANCE_NAMESPACE_IMPL_VERSION) \
		-f docker/Dockerfile.test \
		-t lance-spark-test:$(SPARK_VERSION)_$(SCALA_VERSION) \
		.


.PHONY: docker-test
docker-test:
	@docker image inspect lance-spark-test:$(SPARK_VERSION)_$(SCALA_VERSION) >/dev/null 2>&1 || \
		(echo "Error: Docker image 'lance-spark-test:$(SPARK_VERSION)_$(SCALA_VERSION)' not found. Run 'make docker-build-test' first." && exit 1)
	docker run --rm --hostname lance-spark \
		-e SPARK_VERSION=$(SPARK_VERSION) \
		$(if $(LANCEDB_DB),-e LANCEDB_DB=$(LANCEDB_DB)) \
		$(if $(LANCEDB_API_KEY),-e LANCEDB_API_KEY=$(LANCEDB_API_KEY)) \
		$(if $(LANCEDB_HOST_OVERRIDE),-e LANCEDB_HOST_OVERRIDE=$(LANCEDB_HOST_OVERRIDE)) \
		$(if $(LANCEDB_REGION),-e LANCEDB_REGION=$(LANCEDB_REGION)) \
		$(if $(TEST_BACKENDS),-e TEST_BACKENDS=$(TEST_BACKENDS)) \
		$(if $(LANCE_FTS_FORMAT_VERSION),-e LANCE_FTS_FORMAT_VERSION=$(LANCE_FTS_FORMAT_VERSION)) \
		$(if $(AWS_REGION),-e AWS_REGION=$(AWS_REGION)) \
		$(if $(AWS_DEFAULT_REGION),-e AWS_DEFAULT_REGION=$(AWS_DEFAULT_REGION)) \
		$(if $(AWS_S3_BUCKET_NAME),-e AWS_S3_BUCKET_NAME=$(AWS_S3_BUCKET_NAME)) \
		$(if $(AWS_GLUE_ROOT),-e AWS_GLUE_ROOT=$(AWS_GLUE_ROOT)) \
		$(if $(AWS_GLUE_CATALOG_ID),-e AWS_GLUE_CATALOG_ID=$(AWS_GLUE_CATALOG_ID)) \
		$(if $(AWS_GLUE_ENDPOINT),-e AWS_GLUE_ENDPOINT=$(AWS_GLUE_ENDPOINT)) \
		$(if $(AWS_ACCESS_KEY_ID),-e AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID)) \
		$(if $(AWS_SECRET_ACCESS_KEY),-e AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY)) \
		$(if $(AWS_SESSION_TOKEN),-e AWS_SESSION_TOKEN=$(AWS_SESSION_TOKEN)) \
		$(if $(AWS_PROFILE),-e AWS_PROFILE=$(AWS_PROFILE)) \
		$(if $(AWS_PROFILE),-v $(HOME)/.aws:/root/.aws:ro) \
		lance-spark-test:$(SPARK_VERSION)_$(SCALA_VERSION) \
		"pytest /home/lance/tests/ -v --timeout=180"

# =============================================================================
# Benchmark
# =============================================================================

.PHONY: benchmark-build
benchmark-build:
	cd benchmark && ../mvnw package -DskipTests \
		-Dspark.compat.version=$(SPARK_VERSION) \
		-Dscala.compat.version=$(SCALA_VERSION)

.PHONY: benchmark-generate
benchmark-generate:
	cd benchmark && \
		SPARK_VERSION=$(SPARK_VERSION) SCALA_VERSION=$(SCALA_VERSION) \
		./scripts/generate-data.sh $(SF) $(FORMATS) $(SPARK_MASTER)

.PHONY: benchmark-run
benchmark-run:
	cd benchmark && \
		SPARK_VERSION=$(SPARK_VERSION) SCALA_VERSION=$(SCALA_VERSION) \
		./scripts/run-benchmark.sh $(FORMATS) $(SPARK_MASTER) $(ITERATIONS)

.PHONY: benchmark
benchmark: benchmark-generate benchmark-run

.PHONY: benchmark-tpch-generate
benchmark-tpch-generate:
	cd benchmark && \
		SPARK_VERSION=$(SPARK_VERSION) SCALA_VERSION=$(SCALA_VERSION) \
		./scripts/generate-tpch-data.sh $(SF) $(FORMATS) $(SPARK_MASTER)

.PHONY: benchmark-tpch-run
benchmark-tpch-run:
	cd benchmark && \
		SPARK_VERSION=$(SPARK_VERSION) SCALA_VERSION=$(SCALA_VERSION) \
		./scripts/run-tpch-benchmark.sh $(FORMATS) $(SPARK_MASTER) $(ITERATIONS)

.PHONY: benchmark-tpch
benchmark-tpch: benchmark-tpch-generate benchmark-tpch-run

SF ?= 1
FORMATS ?= lance,parquet
SPARK_MASTER ?= local[*]
ITERATIONS ?= 3

# =============================================================================
# Documentation
# =============================================================================

.PHONY: serve-docs
serve-docs:
	cd docs && uv pip install -r requirements.txt && uv run mkdocs serve

# =============================================================================
# Help
# =============================================================================

.PHONY: help
help:
	@echo "Lance Spark Makefile"
	@echo ""
	@echo "Version parameters (defaults: SPARK_VERSION=3.5, SCALA_VERSION=2.12):"
	@echo "  Example: make install SPARK_VERSION=3.4 SCALA_VERSION=2.13"
	@echo ""
	@echo "Parameterized commands (use SPARK_VERSION and SCALA_VERSION):"
	@echo "  install        - Install module without tests"
	@echo "  test           - Run tests for module"
	@echo "  build          - Lint and install module"
	@echo "  clean-module   - Clean module"
	@echo "  bundle         - Build bundle module (incremental)"
	@echo "  clean-bundle   - Clean then build bundle module (use when source changes are not picked up)"
	@echo "  install-base   - Install base module"
	@echo ""
	@echo "Global commands (all modules):"
	@echo "  lint           - Check code style (checkstyle + spotless)"
	@echo "  format         - Apply spotless formatting"
	@echo "  install-all    - Install all modules without tests"
	@echo "  test-all       - Run all tests"
	@echo "  build-all      - Lint and install all modules"
	@echo "  clean          - Clean all modules"
	@echo ""
	@echo "Docker commands:"
	@echo "  docker-build           - Build docker image with Spark 3.5/Scala 2.12 bundle"
	@echo "  docker-up              - Start docker containers"
	@echo "  docker-shell           - Open shell in spark-lance container"
	@echo "  docker-down            - Stop docker containers"
	@echo "  docker-build-test-base - Build test base image (system deps + Spark)"
	@echo "  docker-build-test      - Build test image (base + bundle JAR)"
	@echo "  docker-test            - Run integration tests in lance-spark-test container"
	@echo ""
	@echo "Benchmark:"
	@echo "  benchmark-build         - Build benchmark jar (shared by TPC-DS and TPC-H)"
	@echo "  benchmark-generate      - Generate TPC-DS data via Spark (SF=1 FORMATS=lance,parquet)"
	@echo "  benchmark-run           - Run TPC-DS queries (FORMATS=lance,parquet ITERATIONS=3)"
	@echo "  benchmark               - Generate TPC-DS data + run queries (end-to-end)"
	@echo "  benchmark-tpch-generate - Generate TPC-H data via Spark (SF=1 FORMATS=lance,parquet)"
	@echo "  benchmark-tpch-run      - Run TPC-H queries (FORMATS=lance,parquet ITERATIONS=3)"
	@echo "  benchmark-tpch          - Generate TPC-H data + run queries (end-to-end)"
	@echo ""
	@echo "Documentation:"
	@echo "  serve-docs     - Serve documentation locally"