Add extension upgrade template regression test (#2364)

Note: This PR was created with AI tools and a human.

Add a version-agnostic regression test (age_upgrade) that validates the
upgrade template (age--<VER>--y.y.y.sql) works correctly by simulating a
full extension version upgrade within "make installcheck".

Add full upgrade scripts to the install path (DATA) in the Makefile,
excluding template upgrade files. This enables the install to copy all
version upgrade files into the PG AGE install. This is needed for
ALTER EXTENSION

Adjusted installcheck.yaml to allow git commit history for this test.

Makefile infrastructure:
- Build the install SQL (age--<CURR>.sql) from the initial version-bump
  commit in git history, so CREATE EXTENSION installs "day-one" SQL while
  the .so comes from current HEAD — implicitly testing backward compat.
- Build a synthetic "next" version (age--<NEXT>.sql) from HEAD and stamp
  the upgrade template to produce age--<CURR>--<NEXT>.sql.
- Add an installcheck prerequisite that temporarily installs both synthetic
  files into the PG extension directory; a generated cleanup script removes
  them at the end of the test via \! shell escape. EXTRA_CLEAN catches
  stragglers on "make clean".
- Skip the test automatically when: (a) no git history (tarball builds),
  (b) no upgrade template exists, or (c) a real upgrade script from the
  current version is already committed (detected via git ls-files).o

Regression test (regress/sql/age_upgrade.sql):
- Creates 3 graphs (company, network, routes) with 8 vertex labels,
  8 edge labels, 23 vertices, 28 edges, and 4 GIN indexes.
- Records integrity checksums (agtype sums), vertex/edge counts, and
  label counts before the upgrade; repeats all checks after ALTER
  EXTENSION UPDATE to the synthetic next version.
- Verifies structural queries: VLE management chains, circular follow
  chains, flight distances with edge properties.
- Verifies all 4 GIN indexes survive the upgrade via pg_indexes.
- Uses ORDER BY on all multi-row queries for deterministic output.
- Returns agtype natively (no ::numeric casts) for portability.
- Avoids version-dependent output (checks boolean IS NOT NULL instead
  of printing the version string).
- Uses JOIN-based label counts to avoid NULL comparison bugs with the
  internal _ag_catalog graph.
- Cleans up all 3 graphs and restores the default AGE version.

modified:   Makefile
new file:   regress/expected/age_upgrade.out
new file:   regress/sql/age_upgrade.sql
modified:   .github/workflows/installcheck.yaml

Author: jrgemignani

.github/workflows/installcheck.yaml

@@ -42,6 +42,8 @@ jobs:
           make PG_CONFIG=$HOME/pg18/bin/pg_config install -j$(nproc) > /dev/null
 
       - uses: actions/checkout@v3
+        with:
+          fetch-depth: 75
 
       - name: Build AGE
         id: build

Makefile

@@ -19,6 +19,64 @@ MODULE_big = age
 
 age_sql = age--1.7.0.sql
 
+# --- Extension upgrade regression test support ---
+#
+# Validates the upgrade template (age--<VER>--y.y.y.sql) by simulating an
+# extension version upgrade entirely within "make installcheck". The test:
+#
+#   1. Builds the install SQL from the INITIAL version-bump commit (the "from"
+#      version). This is the age--<CURR>.sql used by CREATE EXTENSION age.
+#   2. Builds the install SQL from current HEAD as a synthetic "next" version
+#      (the "to" version). This is age--<NEXT>.sql where NEXT = CURR.minor+1.
+#   3. Stamps the upgrade template with the synthetic version, producing the
+#      upgrade script age--<CURR>--<NEXT>.sql.
+#   4. Temporarily installs both synthetic files into the PG extension directory
+#      so that ALTER EXTENSION age UPDATE TO '<NEXT>' can find them.
+#   5. The age_upgrade regression test exercises the full upgrade path: install
+#      at CURR, create data, ALTER EXTENSION UPDATE to NEXT, verify data.
+#   6. The test SQL cleans up the synthetic files via a generated shell script.
+#
+# This forces developers to keep the upgrade template in sync: any SQL object
+# added after the version-bump commit must also appear in the template, or the
+# upgrade test will fail (the object will be missing after ALTER EXTENSION UPDATE).
+#
+# The .so (shared library) is always built from current HEAD, so C-level changes
+# in the PR are tested by every regression test, not just the upgrade test.
+#
+# Graceful degradation — the upgrade test is silently skipped when:
+#   - No git history (tarball build): AGE_VER_COMMIT is empty.
+#   - No upgrade template: age--<CURR>--y.y.y.sql does not exist.
+#   - A real (git-tracked) upgrade script from <CURR> already exists
+#     (e.g., age--1.7.0--1.8.0.sql is committed): the synthetic test is
+#     redundant because the real script ships with the extension.
+# Current version from age.control (e.g., "1.7.0")
+AGE_CURR_VER := $(shell awk -F"'" '/default_version/ {print $$2}' age.control 2>/dev/null)
+# Git commit that last changed age.control — the "initial release" commit
+AGE_VER_COMMIT := $(shell git log -1 --format=%H -- age.control 2>/dev/null)
+# Synthetic next version: current minor + 1 (e.g., 1.7.0 -> 1.8.0)
+AGE_NEXT_VER := $(shell echo $(AGE_CURR_VER) | awk -F. '{printf "%s.%s.%s", $$1, $$2+1, $$3}')
+# The upgrade template file (e.g., age--1.7.0--y.y.y.sql); empty if not present
+AGE_UPGRADE_TEMPLATE := $(wildcard age--$(AGE_CURR_VER)--y.y.y.sql)
+
+# Synthetic filenames — these are NOT installed permanently; they are temporarily
+# placed in $(SHAREDIR)/extension/ during installcheck and removed after.
+age_next_sql = $(if $(AGE_NEXT_VER),age--$(AGE_NEXT_VER).sql)
+age_upgrade_test_sql = $(if $(AGE_NEXT_VER),age--$(AGE_CURR_VER)--$(AGE_NEXT_VER).sql)
+
+# Real (git-tracked, non-template) upgrade scripts FROM the current version.
+# If any exist (e.g., age--1.7.0--1.8.0.sql is committed), the synthetic
+# upgrade test is redundant because a real upgrade path already ships.
+# Uses git ls-files so untracked synthetic files are NOT matched.
+AGE_REAL_UPGRADE := $(shell git ls-files 'age--$(AGE_CURR_VER)--*.sql' 2>/dev/null | grep -v 'y\.y\.y')
+
+# Non-empty when ALL of these hold:
+#   1. Git history is available (AGE_VER_COMMIT non-empty)
+#   2. The upgrade template exists (AGE_UPGRADE_TEMPLATE non-empty)
+#   3. No real upgrade script from current version exists (AGE_REAL_UPGRADE empty)
+# When a real upgrade script ships, the test is skipped — the real script
+# supersedes the synthetic one and has its own validation path.
+AGE_HAS_UPGRADE_TEST = $(and $(AGE_VER_COMMIT),$(AGE_UPGRADE_TEMPLATE),$(if $(AGE_REAL_UPGRADE),,yes))
+
 OBJS = src/backend/age.o \
        src/backend/catalog/ag_catalog.o \
        src/backend/catalog/ag_graph.o \
@@ -83,6 +141,10 @@ SQLS := $(addsuffix .sql,$(SQLS))
 
 DATA_built = $(age_sql)
 
+# Git-tracked upgrade scripts shipped with the extension (e.g., age--1.6.0--1.7.0.sql).
+# Excludes the upgrade template (y.y.y) and the synthetic stamped test file.
+DATA = $(filter-out age--%-y.y.y.sql $(age_upgrade_test_sql),$(wildcard age--*--*.sql))
+
 # sorted in dependency order
 REGRESS = scan \
           graphid \
@@ -119,6 +181,13 @@ ifneq ($(EXTRA_TESTS),)
   REGRESS += $(EXTRA_TESTS)
 endif
 
+# Extension upgrade test — included when git history is available, the upgrade
+# template exists, and no real upgrade script from the current version is
+# committed. Runs between "security" and "drop" in test order.
+ifneq ($(AGE_HAS_UPGRADE_TEST),)
+  REGRESS += age_upgrade
+endif
+
 REGRESS += drop
 
 srcdir=`pwd`
@@ -127,7 +196,7 @@ ag_regress_dir = $(srcdir)/regress
 REGRESS_OPTS = --load-extension=age --inputdir=$(ag_regress_dir) --outputdir=$(ag_regress_dir) --temp-instance=$(ag_regress_dir)/instance --port=61958 --encoding=UTF-8 --temp-config $(ag_regress_dir)/age_regression.conf
 
 ag_regress_out = instance/ log/ results/ regression.*
-EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql)
+EXTRA_CLEAN = $(addprefix $(ag_regress_dir)/, $(ag_regress_out)) src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h src/include/parser/cypher_kwlist_d.h $(all_age_sql) $(age_next_sql) $(age_upgrade_test_sql) $(ag_regress_dir)/age_upgrade_cleanup.sh
 
 GEN_KEYWORDLIST = $(PERL) -I ./tools/ ./tools/gen_keywordlist.pl
 GEN_KEYWORDLIST_DEPS = ./tools/gen_keywordlist.pl tools/PerfectHash.pm
@@ -157,15 +226,78 @@ src/backend/parser/cypher_parser.bc: src/backend/parser/cypher_gram.c src/includ
 src/backend/parser/cypher_keywords.o: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h
 src/backend/parser/cypher_keywords.bc: src/backend/parser/cypher_gram.c src/include/parser/cypher_gram_def.h
 
-# Strip PASSEDBYVALUE on 32-bit (SIZEOF_DATUM=4) for graphid pass-by-reference
+# Build the default install SQL (age--<VER>.sql) from the INITIAL version-bump
+# commit in git history. This means CREATE EXTENSION age installs the "day-one
+# release" SQL — the state of sql/ at the moment the version was bumped in
+# age.control. All other regression tests run against this SQL + the current
+# HEAD .so, implicitly validating that the .so is backward-compatible with the
+# initial release SQL.
+#
+# The current HEAD SQL goes into the synthetic next version (age--<NEXT>.sql)
+# which is only reachable via ALTER EXTENSION UPDATE in the upgrade test.
+ifneq ($(AGE_VER_COMMIT),)
+$(age_sql): age.control
+	@echo "Building initial-release install SQL: $@ from commit $(AGE_VER_COMMIT)"
+	@for f in $$(git show $(AGE_VER_COMMIT):sql/sql_files 2>/dev/null); do \
+		git show $(AGE_VER_COMMIT):sql/$${f}.sql 2>/dev/null; \
+	done > $@
+ifeq ($(SIZEOF_DATUM),4)
+	@echo "32-bit build: removing PASSEDBYVALUE from graphid type"
+	@sed 's/^  PASSEDBYVALUE,$$/  -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@
+endif
+else
+# Fallback: no git history (tarball build) — use current HEAD SQL fragments
 $(age_sql): $(SQLS)
 	@cat $(SQLS) > $@
 ifeq ($(SIZEOF_DATUM),4)
 	@echo "32-bit build: removing PASSEDBYVALUE from graphid type"
 	@sed 's/^  PASSEDBYVALUE,$$/  -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@
 	@grep -q 'PASSEDBYVALUE removed for 32-bit' $@ || { echo "Error: PASSEDBYVALUE replacement failed in $@"; exit 1; }
 endif
+endif
+
+# Build synthetic "next" version install SQL from current HEAD's sql/sql_files.
+# This represents what the extension SQL looks like in the PR being tested.
+ifneq ($(AGE_HAS_UPGRADE_TEST),)
+$(age_next_sql): $(SQLS)
+	@echo "Building synthetic next version install SQL: $@ ($(AGE_NEXT_VER))"
+	@cat $(SQLS) > $@
+ifeq ($(SIZEOF_DATUM),4)
+	@sed 's/^  PASSEDBYVALUE,$$/  -- PASSEDBYVALUE removed for 32-bit (see Makefile)/' $@ > $@.tmp && mv $@.tmp $@
+endif
+
+# Stamp upgrade template as upgrade from current to synthetic next version
+$(age_upgrade_test_sql): $(AGE_UPGRADE_TEMPLATE)
+	@echo "Stamping upgrade template: $< -> $@"
+	@sed -e "s/1\.X\.0/$(AGE_NEXT_VER)/g" -e "s/y\.y\.y/$(AGE_NEXT_VER)/g" $< > $@
+endif
 
 src/backend/parser/ag_scanner.c: FLEX_NO_BACKUP=yes
 
+# --- Upgrade test file lifecycle during installcheck ---
+#
+# Problem: The upgrade test needs age--<NEXT>.sql and age--<CURR>--<NEXT>.sql
+# in the PG extension directory for ALTER EXTENSION UPDATE to find them, but
+# we must not leave them installed permanently (they would confuse users).
+#
+# Solution: A Make prerequisite installs them before pg_regress runs, and the
+# test SQL removes them at the end via \! (shell escape in psql). A generated
+# cleanup script (regress/age_upgrade_cleanup.sh) contains the exact absolute
+# paths so the removal works regardless of the working directory. EXTRA_CLEAN
+# also removes them on "make clean" as a safety net.
+#
+# This adds a prerequisite to "installcheck" but does NOT override the PGXS
+# recipe, so there are no Makefile warnings.
+SHAREDIR = $(shell $(PG_CONFIG) --sharedir)
+
 installcheck: export LC_COLLATE=C
+ifneq ($(AGE_HAS_UPGRADE_TEST),)
+.PHONY: _install_upgrade_test_files
+_install_upgrade_test_files: $(age_next_sql) $(age_upgrade_test_sql)  ## Build, install synthetic files, generate cleanup script
+	@echo "Installing upgrade test files to $(SHAREDIR)/extension/"
+	@$(INSTALL_DATA) $(age_next_sql) $(age_upgrade_test_sql) '$(SHAREDIR)/extension/'
+	@printf '#!/bin/sh\nrm -f "$(SHAREDIR)/extension/$(age_next_sql)" "$(SHAREDIR)/extension/$(age_upgrade_test_sql)"\n' > $(ag_regress_dir)/age_upgrade_cleanup.sh
+	@chmod +x $(ag_regress_dir)/age_upgrade_cleanup.sh
+
+installcheck: _install_upgrade_test_files
+endif