Skip to content

Commit 0412a73

Browse files
zachmuzachmu
authored
Merge pull request #2649 from dolthub/zachmu/compat-testing
compatibility testing
2 parents 7a7ae16 + 46715b1 commit 0412a73

9 files changed

Lines changed: 1861 additions & 0 deletions

File tree

‎.github/workflows/ci-compatibility-tests.yaml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
name: Check Compatibility
2+
3+
on:
4+
pull_request:
5+
branches: [ main ]
6+
paths:
7+
- 'go/**'
8+
- 'core/**'
9+
- 'server/**'
10+
- 'integration-tests/**'
11+
12+
concurrency:
13+
group: ci-compatibility-tests-${{ github.event.pull_request.number || github.ref }}
14+
cancel-in-progress: true
15+
16+
jobs:
17+
test:
18+
name: Compatibility Test
19+
runs-on: ubuntu-22.04
20+
steps:
21+
- uses: actions/checkout@v4
22+
23+
- name: Set up Go
24+
uses: actions/setup-go@v5
25+
with:
26+
go-version-file: go.mod
27+
id: go
28+
29+
- uses: actions/setup-node@v4
30+
with:
31+
node-version: ^16
32+
33+
- name: Build SQL Syntax
34+
run: ./build.sh
35+
working-directory: ./postgres/parser
36+
shell: bash
37+
38+
- name: Create CI Bin
39+
run: |
40+
mkdir -p ./.ci_bin
41+
echo "$(pwd)/.ci_bin" >> $GITHUB_PATH
42+
43+
- name: Install Bats
44+
run: |
45+
npm i bats
46+
echo "$(pwd)/node_modules/.bin" >> $GITHUB_PATH
47+
working-directory: ./.ci_bin
48+
49+
- name: Install DoltgreSQL
50+
run: |
51+
go build -mod=readonly -o .ci_bin/doltgres ./cmd/doltgres
52+
53+
- name: Install PSQL
54+
run: |
55+
sudo sh -c 'echo "deb https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
56+
wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
57+
sudo apt-get update
58+
sudo apt-get install --yes --no-install-recommends postgresql-client-15
59+
60+
- name: Make scripts executable
61+
run: chmod +x runner.sh test_files/setup_repo.sh
62+
working-directory: ./integration-tests/compatibility
63+
64+
- name: Run compatibility tests
65+
run: ./runner.sh
66+
working-directory: ./integration-tests/compatibility

‎integration-tests/compatibility/runner.sh‎

Lines changed: 301 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,301 @@
1+
#!/bin/bash
2+
# Doltgresql compatibility test runner.
3+
#
4+
# Downloads specified doltgresql release binaries, creates test repositories
5+
# using them, and runs BATS test suites to verify backward, forward, and
6+
# bidirectional compatibility.
7+
#
8+
# Usage: ./runner.sh
9+
# Run from the integration-tests/compatibility directory.
10+
#
11+
# Environment variables (optional):
12+
# DOLTGRES_SKIP_BACKWARD — skip backward-compatibility tests if set
13+
# DOLTGRES_SKIP_FORWARD — skip forward-compatibility tests if set
14+
# DOLTGRES_SKIP_BIDIR — skip bidirectional-compatibility tests if set
15+
16+
set -eo pipefail
17+
18+
PLATFORM_TUPLE=""
19+
20+
# ---------------------------------------------------------------------------
21+
# Platform detection
22+
# ---------------------------------------------------------------------------
23+
24+
get_platform_tuple() {
25+
local OS ARCH
26+
OS=$(uname)
27+
ARCH=$(uname -m)
28+
29+
if [ "$OS" != Linux ] && [ "$OS" != Darwin ]; then
30+
echo "tests only support linux or macOS." >&2
31+
exit 1
32+
fi
33+
34+
if [ "$OS" = Linux ]; then
35+
PLATFORM_TUPLE=linux
36+
else
37+
PLATFORM_TUPLE=darwin
38+
fi
39+
40+
if [ "$ARCH" = x86_64 ]; then
41+
PLATFORM_TUPLE="${PLATFORM_TUPLE}-amd64"
42+
elif [ "$ARCH" = arm64 ] || [ "$ARCH" = aarch64 ]; then
43+
PLATFORM_TUPLE="${PLATFORM_TUPLE}-arm64"
44+
else
45+
echo "unsupported architecture: $ARCH" >&2
46+
exit 1
47+
fi
48+
49+
echo "$PLATFORM_TUPLE"
50+
}
51+
52+
# ---------------------------------------------------------------------------
53+
# Release download
54+
# ---------------------------------------------------------------------------
55+
56+
# download_release <version>
57+
# Downloads doltgresql-<platform>.tar.gz for the given version tag, extracts it
58+
# into binaries/<version>/, and prints the path to the bin directory.
59+
download_release() {
60+
local ver="$1"
61+
local dirname="binaries/$ver"
62+
mkdir -p "$dirname"
63+
64+
local basename="doltgresql-${PLATFORM_TUPLE}"
65+
local filename="${basename}.tar.gz"
66+
local filepath="${dirname}/${filename}"
67+
local url="https://github.com/dolthub/doltgresql/releases/download/${ver}/${filename}"
68+
69+
echo "Downloading doltgresql ${ver} for ${PLATFORM_TUPLE} ..." >&2
70+
curl -L -o "$filepath" "$url"
71+
tar -zxf "$filepath" -C "$dirname"
72+
73+
# Binary lives at doltgresql-<os>-<arch>/bin/doltgres
74+
echo "${dirname}/${basename}/bin"
75+
}
76+
77+
# ---------------------------------------------------------------------------
78+
# Server management helpers used by the runner (not BATS)
79+
# ---------------------------------------------------------------------------
80+
81+
_pick_port() {
82+
for i in {0..99}; do
83+
local port=$((RANDOM % 4096 + 2048))
84+
if ! nc -z localhost "$port" 2>/dev/null; then
85+
echo "$port"
86+
return 0
87+
fi
88+
done
89+
echo "ERROR: could not find a free port" >&2
90+
return 1
91+
}
92+
93+
_write_config() {
94+
local dir="$1" port="$2"
95+
cat > "$dir/runner-config.yaml" <<EOF
96+
log_level: warning
97+
behavior:
98+
read_only: false
99+
disable_client_multi_statements: false
100+
listener:
101+
host: localhost
102+
port: $port
103+
EOF
104+
}
105+
106+
# start_server <binary> <datadir> <logfile> → sets RUNNER_SERVER_PID and RUNNER_SERVER_PORT
107+
start_server() {
108+
local binary="$1" datadir="$2" logfile="$3"
109+
RUNNER_SERVER_PORT=$(_pick_port)
110+
_write_config "$datadir" "$RUNNER_SERVER_PORT"
111+
112+
PGPASSWORD=password "$binary" -data-dir="$datadir" \
113+
--config="$datadir/runner-config.yaml" > "$logfile" 2>&1 &
114+
RUNNER_SERVER_PID=$!
115+
116+
local end=$((SECONDS + 20))
117+
while [ $SECONDS -lt $end ]; do
118+
if PGPASSWORD=password psql -U postgres -h localhost -p "$RUNNER_SERVER_PORT" \
119+
-c "SELECT 1;" postgres >/dev/null 2>&1; then
120+
return 0
121+
fi
122+
sleep 0.5
123+
done
124+
125+
echo "ERROR: server failed to start on port $RUNNER_SERVER_PORT" >&2
126+
cat "$logfile" >&2
127+
kill "$RUNNER_SERVER_PID" 2>/dev/null
128+
return 1
129+
}
130+
131+
stop_server() {
132+
if [ -n "$RUNNER_SERVER_PID" ]; then
133+
kill "$RUNNER_SERVER_PID" 2>/dev/null || true
134+
wait "$RUNNER_SERVER_PID" 2>/dev/null || true
135+
RUNNER_SERVER_PID=""
136+
RUNNER_SERVER_PORT=""
137+
fi
138+
}
139+
140+
RUNNER_SERVER_PID=""
141+
RUNNER_SERVER_PORT=""
142+
143+
# ---------------------------------------------------------------------------
144+
# Repository setup
145+
# ---------------------------------------------------------------------------
146+
147+
# setup_repo <label> [<binary>]
148+
# Creates a test repository under repos/<label>/ using the given binary
149+
# (defaults to doltgres from PATH). Sets REPO_DIR to the resulting directory.
150+
setup_repo() {
151+
local label="$1"
152+
local binary="${2:-doltgres}"
153+
REPO_DIR="$(pwd)/repos/${label}"
154+
mkdir -p "$REPO_DIR"
155+
./test_files/setup_repo.sh "$REPO_DIR" "$binary"
156+
}
157+
158+
# ---------------------------------------------------------------------------
159+
# Version list helpers
160+
# ---------------------------------------------------------------------------
161+
162+
list_backward_compatible_versions() {
163+
grep -v '^ *#' < test_files/backward_compatible_versions.txt
164+
}
165+
166+
list_forward_compatible_versions() {
167+
grep -v '^ *#' < test_files/forward_compatible_versions.txt
168+
}
169+
170+
# ---------------------------------------------------------------------------
171+
# Test runners
172+
# ---------------------------------------------------------------------------
173+
174+
test_backward_compatibility() {
175+
local ver="$1"
176+
local bin
177+
bin=$(download_release "$ver")
178+
179+
echo "=== Backward compat: creating repo with doltgresql ${ver} ==="
180+
setup_repo "$ver" "${bin}/doltgres"
181+
182+
echo "=== Backward compat: testing HEAD doltgresql against repo from ${ver} ==="
183+
DOLTGRES_TEST_BIN="$(which doltgres)" \
184+
REPO_DIR="$(pwd)/repos/${ver}" \
185+
bats --print-output-on-failure ./test_files/bats/compatibility.bats
186+
187+
DOLTGRES_TEST_BIN="$(which doltgres)" \
188+
REPO_DIR="$(pwd)/repos/${ver}" \
189+
bats --print-output-on-failure ./test_files/bats/types_compatibility.bats
190+
}
191+
192+
test_forward_compatibility() {
193+
if [ -z $1 ]; then
194+
return
195+
fi
196+
197+
local ver="$1"
198+
local bin
199+
bin=$(download_release "$ver")
200+
201+
echo "=== Forward compat: testing doltgresql ${ver} against repo from HEAD ==="
202+
# repos/HEAD was already created by the main flow (see _main).
203+
DOLTGRES_TEST_BIN="${bin}/doltgres" \
204+
REPO_DIR="$(pwd)/repos/HEAD" \
205+
bats --print-output-on-failure ./test_files/bats/compatibility.bats
206+
207+
DOLTGRES_TEST_BIN="${bin}/doltgres" \
208+
REPO_DIR="$(pwd)/repos/HEAD" \
209+
bats --print-output-on-failure ./test_files/bats/types_compatibility.bats
210+
}
211+
212+
test_bidirectional_compatibility() {
213+
if [ -z $1 ]; then
214+
return
215+
fi
216+
217+
local ver="$1"
218+
local bin
219+
bin=$(download_release "$ver")
220+
221+
local head_bin
222+
head_bin="$(which doltgres)"
223+
224+
# Forward direction: old = released version, new = HEAD
225+
local scratch_fwd="$(pwd)/repos/${ver}-bidir-forward"
226+
mkdir -p "$scratch_fwd"
227+
echo "=== Bidirectional (forward): old=${ver}, new=HEAD ==="
228+
DOLTGRES_LEGACY_BIN="${bin}/doltgres" \
229+
DOLTGRES_NEW_BIN="$head_bin" \
230+
REPO_DIR="$scratch_fwd" \
231+
bats --print-output-on-failure ./test_files/bats/bidirectional/bidirectional_compat.bats
232+
233+
# Reverse direction: old = HEAD, new = released version
234+
local scratch_rev="$(pwd)/repos/${ver}-bidir-reverse"
235+
mkdir -p "$scratch_rev"
236+
echo "=== Bidirectional (reverse): old=HEAD, new=${ver} ==="
237+
DOLTGRES_LEGACY_BIN="$head_bin" \
238+
DOLTGRES_NEW_BIN="${bin}/doltgres" \
239+
REPO_DIR="$scratch_rev" \
240+
bats --print-output-on-failure ./test_files/bats/bidirectional/bidirectional_compat.bats
241+
}
242+
243+
# ---------------------------------------------------------------------------
244+
# Cleanup
245+
# ---------------------------------------------------------------------------
246+
247+
cleanup() {
248+
stop_server
249+
rm -rf repos binaries
250+
}
251+
252+
# ---------------------------------------------------------------------------
253+
# Main
254+
# ---------------------------------------------------------------------------
255+
256+
_main() {
257+
PLATFORM_TUPLE=$(get_platform_tuple)
258+
259+
mkdir -p repos binaries
260+
trap cleanup EXIT
261+
262+
# --- Backward compatibility ---
263+
if [ -z "$DOLTGRES_SKIP_BACKWARD" ] && [ -s "test_files/backward_compatible_versions.txt" ]; then
264+
echo "=== Running backward compatibility tests ==="
265+
while IFS= read -r ver; do
266+
test_backward_compatibility "$ver"
267+
done < <(list_backward_compatible_versions)
268+
fi
269+
270+
# --- Create HEAD repo (used for forward compat and the sanity check) ---
271+
echo "=== Creating HEAD repo ==="
272+
setup_repo HEAD
273+
274+
# --- Forward compatibility ---
275+
if [ -z "$DOLTGRES_SKIP_FORWARD" ] && [ -s "test_files/forward_compatible_versions.txt" ]; then
276+
echo "=== Running forward compatibility tests ==="
277+
while IFS= read -r ver; do
278+
test_forward_compatibility "$ver"
279+
done < <(list_forward_compatible_versions)
280+
fi
281+
282+
# --- Bidirectional compatibility (uses forward_compatible_versions list) ---
283+
if [ -z "$DOLTGRES_SKIP_BIDIR" ] && [ -s "test_files/forward_compatible_versions.txt" ]; then
284+
echo "=== Running bidirectional compatibility tests ==="
285+
while IFS= read -r ver; do
286+
test_bidirectional_compatibility "$ver"
287+
done < <(list_forward_compatible_versions)
288+
fi
289+
290+
# --- Sanity check: HEAD against HEAD ---
291+
echo "=== Sanity check: HEAD doltgresql against HEAD repo ==="
292+
DOLTGRES_TEST_BIN="$(which doltgres)" \
293+
REPO_DIR="$(pwd)/repos/HEAD" \
294+
bats --print-output-on-failure ./test_files/bats/compatibility.bats
295+
296+
DOLTGRES_TEST_BIN="$(which doltgres)" \
297+
REPO_DIR="$(pwd)/repos/HEAD" \
298+
bats --print-output-on-failure ./test_files/bats/types_compatibility.bats
299+
}
300+
301+
_main

‎integration-tests/compatibility/test_files/backward_compatible_versions.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
# Doltgresql versions to test backward compatibility against.
2+
# Current HEAD must be able to read and write repos created by these older versions.
3+
# Format: one version tag per line, e.g. v0.8.0
4+
# Keep this list reasonably short; each entry downloads a binary and runs the full
5+
# test suite, so CI time grows linearly.
6+
v0.55.0
7+
v0.51.0

0 commit comments

Comments
 (0)