Skip to content

Commit 5ba27a7

Browse files
authored
Merge pull request #8 from sqliteai/dev
release version 0.9.110 - fix/schema-hash - fix/bind-column-value-parameters-also-if-null - feat/sync-filter
2 parents 58ce9a4 + ff30ae5 commit 5ba27a7

27 files changed

+3576
-121
lines changed

.claude/commands/test-sync-roundtrip-rls.md

Lines changed: 532 additions & 0 deletions
Large diffs are not rendered by default.

.github/workflows/main.yml

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -226,10 +226,41 @@ jobs:
226226
path: dist/${{ matrix.name == 'apple-xcframework' && 'CloudSync.*' || 'cloudsync.*'}}
227227
if-no-files-found: error
228228

229+
postgres-test:
230+
runs-on: ubuntu-22.04
231+
name: postgresql build + test
232+
timeout-minutes: 10
233+
234+
steps:
235+
236+
- uses: actions/checkout@v4.2.2
237+
238+
- name: build and start postgresql container
239+
run: make postgres-docker-rebuild
240+
241+
- name: wait for postgresql to be ready
242+
run: |
243+
for i in $(seq 1 30); do
244+
if docker exec cloudsync-postgres pg_isready -U postgres > /dev/null 2>&1; then
245+
echo "PostgreSQL is ready"
246+
exit 0
247+
fi
248+
sleep 2
249+
done
250+
echo "PostgreSQL failed to start within 60s"
251+
docker logs cloudsync-postgres
252+
exit 1
253+
254+
- name: run postgresql tests
255+
run: |
256+
docker exec cloudsync-postgres mkdir -p /tmp/cloudsync/test
257+
docker cp test/postgresql cloudsync-postgres:/tmp/cloudsync/test/postgresql
258+
docker exec cloudsync-postgres psql -U postgres -d postgres -f /tmp/cloudsync/test/postgresql/full_test.sql
259+
229260
release:
230261
runs-on: ubuntu-22.04
231262
name: release
232-
needs: build
263+
needs: [build, postgres-test]
233264
if: github.ref == 'refs/heads/main'
234265

235266
env:

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,4 +48,5 @@ jniLibs/
4848

4949
# System
5050
.DS_Store
51-
Thumbs.db
51+
Thumbs.db
52+
CLAUDE.md

docker/Makefile.postgresql

Lines changed: 15 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -137,32 +137,32 @@ PG_DOCKER_DB_PASSWORD ?= postgres
137137

138138
# Build Docker image with pre-installed extension
139139
postgres-docker-build:
140-
@echo "Building Docker image via docker-compose (rebuilt when sources change)..."
140+
@echo "Building Docker image via docker compose (rebuilt when sources change)..."
141141
# To force plaintext BuildKit logs, run: make postgres-docker-build DOCKER_BUILD_ARGS="--progress=plain"
142-
cd docker/postgresql && docker-compose build $(DOCKER_BUILD_ARGS)
142+
cd docker/postgresql && docker compose build $(DOCKER_BUILD_ARGS)
143143
@echo ""
144144
@echo "Docker image built successfully!"
145145

146146
# Build Docker image with AddressSanitizer enabled (override compose file)
147147
postgres-docker-build-asan:
148-
@echo "Building Docker image with ASAN via docker-compose..."
148+
@echo "Building Docker image with ASAN via docker compose..."
149149
# To force plaintext BuildKit logs, run: make postgres-docker-build-asan DOCKER_BUILD_ARGS=\"--progress=plain\"
150-
cd docker/postgresql && docker-compose -f docker-compose.debug.yml -f docker-compose.asan.yml build $(DOCKER_BUILD_ARGS)
150+
cd docker/postgresql && docker compose -f docker-compose.debug.yml -f docker-compose.asan.yml build $(DOCKER_BUILD_ARGS)
151151
@echo ""
152152
@echo "ASAN Docker image built successfully!"
153153

154154
# Build Docker image using docker-compose.debug.yml
155155
postgres-docker-debug-build:
156-
@echo "Building debug Docker image via docker-compose..."
156+
@echo "Building debug Docker image via docker compose..."
157157
# To force plaintext BuildKit logs, run: make postgres-docker-debug-build DOCKER_BUILD_ARGS=\"--progress=plain\"
158-
cd docker/postgresql && docker-compose -f docker-compose.debug.yml build $(DOCKER_BUILD_ARGS)
158+
cd docker/postgresql && docker compose -f docker-compose.debug.yml build $(DOCKER_BUILD_ARGS)
159159
@echo ""
160160
@echo "Debug Docker image built successfully!"
161161

162162
# Run PostgreSQL container with CloudSync
163163
postgres-docker-run:
164164
@echo "Starting PostgreSQL with CloudSync..."
165-
cd docker/postgresql && docker-compose up -d --build
165+
cd docker/postgresql && docker compose up -d --build
166166
@echo ""
167167
@echo "Container started successfully!"
168168
@echo ""
@@ -179,7 +179,7 @@ postgres-docker-run:
179179
# Run PostgreSQL container with CloudSync and AddressSanitizer enabled
180180
postgres-docker-run-asan:
181181
@echo "Starting PostgreSQL with CloudSync (ASAN enabled)..."
182-
cd docker/postgresql && docker-compose -f docker-compose.debug.yml -f docker-compose.asan.yml up -d --build
182+
cd docker/postgresql && docker compose -f docker-compose.debug.yml -f docker-compose.asan.yml up -d --build
183183
@echo ""
184184
@echo "Container started successfully!"
185185
@echo ""
@@ -196,7 +196,7 @@ postgres-docker-run-asan:
196196
# Run PostgreSQL container using docker-compose.debug.yml
197197
postgres-docker-debug-run:
198198
@echo "Starting PostgreSQL with CloudSync (debug compose)..."
199-
cd docker/postgresql && docker-compose -f docker-compose.debug.yml up -d --build
199+
cd docker/postgresql && docker compose -f docker-compose.debug.yml up -d --build
200200
@echo ""
201201
@echo "Container started successfully!"
202202
@echo ""
@@ -213,21 +213,21 @@ postgres-docker-debug-run:
213213
# Stop PostgreSQL container
214214
postgres-docker-stop:
215215
@echo "Stopping PostgreSQL container..."
216-
cd docker/postgresql && docker-compose down
216+
cd docker/postgresql && docker compose down
217217
@echo "Container stopped"
218218

219219
# Rebuild and restart container
220220
postgres-docker-rebuild: postgres-docker-build
221221
@echo "Rebuilding and restarting container..."
222-
cd docker/postgresql && docker-compose down
223-
cd docker/postgresql && docker-compose up -d --build
222+
cd docker/postgresql && docker compose down
223+
cd docker/postgresql && docker compose up -d --build
224224
@echo "Container restarted with new image"
225225

226226
# Rebuild and restart container using docker-compose.debug.yml
227227
postgres-docker-debug-rebuild: postgres-docker-debug-build
228228
@echo "Rebuilding and restarting debug container..."
229-
cd docker/postgresql && docker-compose -f docker-compose.debug.yml down
230-
cd docker/postgresql && docker-compose -f docker-compose.debug.yml up -d --build
229+
cd docker/postgresql && docker compose -f docker-compose.debug.yml down
230+
cd docker/postgresql && docker compose -f docker-compose.debug.yml up -d --build
231231
@echo "Debug container restarted with new image"
232232

233233
# Interactive shell in container
@@ -353,5 +353,5 @@ postgres-help:
353353
# Simple smoke test: rebuild image/container, create extension, and query version
354354
unittest-pg: postgres-docker-rebuild
355355
@echo "Running PostgreSQL extension smoke test..."
356-
cd docker/postgresql && docker-compose exec -T postgres psql -U postgres -d cloudsync_test -f /tmp/cloudsync/docker/postgresql/smoke_test.sql
356+
cd docker/postgresql && docker compose exec -T postgres psql -U postgres -d cloudsync_test -f /tmp/cloudsync/docker/postgresql/smoke_test.sql
357357
@echo "Smoke test completed."

src/cloudsync.c

Lines changed: 129 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -49,12 +49,12 @@
4949
#define CLOUDSYNC_INIT_NTABLES 64
5050
#define CLOUDSYNC_MIN_DB_VERSION 0
5151

52-
#define CLOUDSYNC_PAYLOAD_SKIP_SCHEMA_HASH_CHECK 1
5352
#define CLOUDSYNC_PAYLOAD_MINBUF_SIZE (512*1024)
5453
#define CLOUDSYNC_PAYLOAD_SIGNATURE 0x434C5359 /* 'C','L','S','Y' */
5554
#define CLOUDSYNC_PAYLOAD_VERSION_ORIGNAL 1
5655
#define CLOUDSYNC_PAYLOAD_VERSION_1 CLOUDSYNC_PAYLOAD_VERSION_ORIGNAL
5756
#define CLOUDSYNC_PAYLOAD_VERSION_2 2
57+
#define CLOUDSYNC_PAYLOAD_VERSION_LATEST CLOUDSYNC_PAYLOAD_VERSION_2
5858
#define CLOUDSYNC_PAYLOAD_MIN_VERSION_WITH_CHECKSUM CLOUDSYNC_PAYLOAD_VERSION_2
5959

6060
#ifndef MAX
@@ -63,10 +63,6 @@
6363

6464
#define DEBUG_DBERROR(_rc, _fn, _data) do {if (_rc != DBRES_OK) printf("Error in %s: %s\n", _fn, database_errmsg(_data));} while (0)
6565

66-
#if CLOUDSYNC_PAYLOAD_SKIP_SCHEMA_HASH_CHECK
67-
bool schema_hash_disabled = true;
68-
#endif
69-
7066
typedef enum {
7167
CLOUDSYNC_PK_INDEX_TBL = 0,
7268
CLOUDSYNC_PK_INDEX_PK = 1,
@@ -1208,18 +1204,20 @@ int merge_insert_col (cloudsync_context *data, cloudsync_table_context *table, c
12081204
return rc;
12091205
}
12101206

1211-
// bind value
1207+
// bind value (always bind all expected parameters for correct prepared statement handling)
12121208
if (col_value) {
12131209
rc = databasevm_bind_value(vm, table->npks+1, col_value);
12141210
if (rc == DBRES_OK) rc = databasevm_bind_value(vm, table->npks+2, col_value);
1215-
if (rc != DBRES_OK) {
1216-
cloudsync_set_dberror(data);
1217-
dbvm_reset(vm);
1218-
return rc;
1219-
}
1220-
1211+
} else {
1212+
rc = databasevm_bind_null(vm, table->npks+1);
1213+
if (rc == DBRES_OK) rc = databasevm_bind_null(vm, table->npks+2);
12211214
}
1222-
1215+
if (rc != DBRES_OK) {
1216+
cloudsync_set_dberror(data);
1217+
dbvm_reset(vm);
1218+
return rc;
1219+
}
1220+
12231221
// perform real operation and disable triggers
12241222

12251223
// in case of GOS we reused the table->col_merge_stmt statement
@@ -1794,13 +1792,104 @@ int cloudsync_commit_alter (cloudsync_context *data, const char *table_name) {
17941792
return rc;
17951793
}
17961794

1795+
// MARK: - Filter Rewrite -
1796+
1797+
// Replace bare column names in a filter expression with prefix-qualified names.
1798+
// E.g., filter="user_id = 42", prefix="NEW", columns=["user_id","id"] → "NEW.\"user_id\" = 42"
1799+
// Columns must be sorted by length descending by the caller to avoid partial matches.
1800+
// Skips content inside single-quoted string literals.
1801+
// Returns a newly allocated string (caller must free with cloudsync_memory_free), or NULL on error.
1802+
// Helper: check if an identifier token matches a column name.
1803+
static bool filter_is_column (const char *token, size_t token_len, char **columns, int ncols) {
1804+
for (int i = 0; i < ncols; ++i) {
1805+
if (strlen(columns[i]) == token_len && strncmp(token, columns[i], token_len) == 0)
1806+
return true;
1807+
}
1808+
return false;
1809+
}
1810+
1811+
// Helper: check if character is part of a SQL identifier.
1812+
static bool filter_is_ident_char (char c) {
1813+
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
1814+
(c >= '0' && c <= '9') || c == '_';
1815+
}
1816+
1817+
char *cloudsync_filter_add_row_prefix (const char *filter, const char *prefix, char **columns, int ncols) {
1818+
if (!filter || !prefix || !columns || ncols <= 0) return NULL;
1819+
1820+
size_t filter_len = strlen(filter);
1821+
size_t prefix_len = strlen(prefix);
1822+
1823+
// Each identifier match grows by at most (prefix_len + 3) bytes.
1824+
// Worst case: the entire filter is one repeated column reference separated by
1825+
// single characters, so up to (filter_len / 2) matches. Use a safe upper bound.
1826+
size_t max_growth = (filter_len / 2 + 1) * (prefix_len + 3);
1827+
size_t cap = filter_len + max_growth + 64;
1828+
char *result = (char *)cloudsync_memory_alloc(cap);
1829+
if (!result) return NULL;
1830+
size_t out = 0;
1831+
1832+
// Single pass: tokenize into identifiers, quoted strings, and everything else.
1833+
size_t i = 0;
1834+
while (i < filter_len) {
1835+
// Skip single-quoted string literals verbatim (handle '' escape)
1836+
if (filter[i] == '\'') {
1837+
result[out++] = filter[i++];
1838+
while (i < filter_len) {
1839+
if (filter[i] == '\'') {
1840+
result[out++] = filter[i++];
1841+
// '' is an escaped quote — keep going
1842+
if (i < filter_len && filter[i] == '\'') {
1843+
result[out++] = filter[i++];
1844+
continue;
1845+
}
1846+
break; // single ' ends the literal
1847+
}
1848+
result[out++] = filter[i++];
1849+
}
1850+
continue;
1851+
}
1852+
1853+
// Extract identifier token
1854+
if (filter_is_ident_char(filter[i])) {
1855+
size_t start = i;
1856+
while (i < filter_len && filter_is_ident_char(filter[i])) ++i;
1857+
size_t token_len = i - start;
1858+
1859+
if (filter_is_column(&filter[start], token_len, columns, ncols)) {
1860+
// Emit PREFIX."column_name"
1861+
memcpy(&result[out], prefix, prefix_len); out += prefix_len;
1862+
result[out++] = '.';
1863+
result[out++] = '"';
1864+
memcpy(&result[out], &filter[start], token_len); out += token_len;
1865+
result[out++] = '"';
1866+
} else {
1867+
// Not a column — copy as-is
1868+
memcpy(&result[out], &filter[start], token_len); out += token_len;
1869+
}
1870+
continue;
1871+
}
1872+
1873+
// Any other character — copy as-is
1874+
result[out++] = filter[i++];
1875+
}
1876+
1877+
result[out] = '\0';
1878+
return result;
1879+
}
1880+
17971881
int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name) {
17981882
cloudsync_table_context *table = table_lookup(data, table_name);
17991883
if (!table) return DBRES_ERROR;
1800-
1884+
18011885
dbvm_t *vm = NULL;
18021886
int64_t db_version = cloudsync_dbversion_next(data, CLOUDSYNC_VALUE_NOTSET);
18031887

1888+
// Read row-level filter from settings (if any)
1889+
char filter_buf[2048];
1890+
int frc = dbutils_table_settings_get_value(data, table_name, "*", "filter", filter_buf, sizeof(filter_buf));
1891+
const char *filter = (frc == DBRES_OK && filter_buf[0]) ? filter_buf : NULL;
1892+
18041893
const char *schema = table->schema ? table->schema : "";
18051894
char *sql = sql_build_pk_collist_query(schema, table_name);
18061895
char *pkclause_identifiers = NULL;
@@ -1810,18 +1899,22 @@ int cloudsync_refill_metatable (cloudsync_context *data, const char *table_name)
18101899
char *pkvalues_identifiers = (pkclause_identifiers) ? pkclause_identifiers : "rowid";
18111900

18121901
// Use database-specific query builder to handle type differences in composite PKs
1813-
sql = sql_build_insert_missing_pks_query(schema, table_name, pkvalues_identifiers, table->base_ref, table->meta_ref);
1902+
sql = sql_build_insert_missing_pks_query(schema, table_name, pkvalues_identifiers, table->base_ref, table->meta_ref, filter);
18141903
if (!sql) {rc = DBRES_NOMEM; goto finalize;}
18151904
rc = database_exec(data, sql);
18161905
cloudsync_memory_free(sql);
18171906
if (rc != DBRES_OK) goto finalize;
1818-
1907+
18191908
// fill missing colums
18201909
// for each non-pk column:
18211910
// The new query does 1 encode per source row and one indexed NOT-EXISTS probe.
1822-
// The old plan does many decodes per candidate and can’t use an index to rule out matches quickly—so it burns CPU and I/O.
1823-
1824-
sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL, pkvalues_identifiers, table->base_ref, table->meta_ref);
1911+
// The old plan does many decodes per candidate and can't use an index to rule out matches quickly—so it burns CPU and I/O.
1912+
1913+
if (filter) {
1914+
sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL_FILTERED, pkvalues_identifiers, table->base_ref, filter, table->meta_ref);
1915+
} else {
1916+
sql = cloudsync_memory_mprintf(SQL_CLOUDSYNC_SELECT_PKS_NOT_IN_SYNC_FOR_COL, pkvalues_identifiers, table->base_ref, table->meta_ref);
1917+
}
18251918
rc = databasevm_prepare(data, sql, (void **)&vm, DBFLAG_PERSISTENT);
18261919
cloudsync_memory_free(sql);
18271920
if (rc != DBRES_OK) goto finalize;
@@ -2263,15 +2356,17 @@ int cloudsync_payload_apply (cloudsync_context *data, const char *payload, int b
22632356
header.nrows = ntohl(header.nrows);
22642357
header.schema_hash = ntohll(header.schema_hash);
22652358

2266-
#if !CLOUDSYNC_PAYLOAD_SKIP_SCHEMA_HASH_CHECK
2267-
if (!data || header.schema_hash != data->schema_hash) {
2268-
if (!database_check_schema_hash(data, header.schema_hash)) {
2269-
char buffer[1024];
2270-
snprintf(buffer, sizeof(buffer), "Cannot apply the received payload because the schema hash is unknown %llu.", header.schema_hash);
2271-
return cloudsync_set_error(data, buffer, DBRES_MISUSE);
2359+
// compare schema_hash only if not disabled and if the received payload was created with the current header version
2360+
// to avoid schema hash mismatch when processed by a peer with a different extension version during software updates.
2361+
if (dbutils_settings_get_int64_value(data, CLOUDSYNC_KEY_SKIP_SCHEMA_HASH_CHECK) == 0 && header.version == CLOUDSYNC_PAYLOAD_VERSION_LATEST ) {
2362+
if (header.schema_hash != data->schema_hash) {
2363+
if (!database_check_schema_hash(data, header.schema_hash)) {
2364+
char buffer[1024];
2365+
snprintf(buffer, sizeof(buffer), "Cannot apply the received payload because the schema hash is unknown %llu.", header.schema_hash);
2366+
return cloudsync_set_error(data, buffer, DBRES_MISUSE);
2367+
}
22722368
}
22732369
}
2274-
#endif
22752370

22762371
// sanity check header
22772372
if ((header.signature != CLOUDSYNC_PAYLOAD_SIGNATURE) || (header.ncols == 0)) {
@@ -2444,8 +2539,8 @@ int cloudsync_payload_get (cloudsync_context *data, char **blob, int *blob_size,
24442539

24452540
// retrieve BLOB
24462541
char sql[1024];
2447-
snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes) "
2448-
"SELECT * FROM (SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload, max_db_version AS max_db_version, MAX(IIF(db_version = max_db_version, seq, NULL)) FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND (db_version>%d OR (db_version=%d AND seq>%d))) WHERE payload IS NOT NULL", *db_version, *db_version, *seq);
2542+
snprintf(sql, sizeof(sql), "WITH max_db_version AS (SELECT MAX(db_version) AS max_db_version FROM cloudsync_changes WHERE site_id=cloudsync_siteid()) "
2543+
"SELECT * FROM (SELECT cloudsync_payload_encode(tbl, pk, col_name, col_value, col_version, db_version, site_id, cl, seq) AS payload, max_db_version AS max_db_version, MAX(IIF(db_version = max_db_version, seq, 0)) FROM cloudsync_changes, max_db_version WHERE site_id=cloudsync_siteid() AND (db_version>%d OR (db_version=%d AND seq>%d))) WHERE payload IS NOT NULL", *db_version, *db_version, *seq);
24492544

24502545
int64_t len = 0;
24512546
int rc = database_select_blob_2int(data, sql, blob, &len, new_db_version, new_seq);
@@ -2723,8 +2818,13 @@ int cloudsync_init_table (cloudsync_context *data, const char *table_name, const
27232818
// sync algo with table (unused in this version)
27242819
// cloudsync_sync_table_key(data, table_name, "*", CLOUDSYNC_KEY_ALGO, crdt_algo_name(algo_new));
27252820

2821+
// read row-level filter from settings (if any)
2822+
char init_filter_buf[2048];
2823+
int init_frc = dbutils_table_settings_get_value(data, table_name, "*", "filter", init_filter_buf, sizeof(init_filter_buf));
2824+
const char *init_filter = (init_frc == DBRES_OK && init_filter_buf[0]) ? init_filter_buf : NULL;
2825+
27262826
// check triggers
2727-
rc = database_create_triggers(data, table_name, algo_new);
2827+
rc = database_create_triggers(data, table_name, algo_new, init_filter);
27282828
if (rc != DBRES_OK) return cloudsync_set_error(data, "An error occurred while creating triggers", DBRES_MISUSE);
27292829

27302830
// check meta-table

0 commit comments

Comments
 (0)