Merge remote-tracking branch 'origin/main' into serverless

khustup2 · khustup2 · commit e598424cc9cb · 2026-02-14T22:20:38.000Z
diff --git a/.github/workflows/pg-extension-build.yaml b/.github/workflows/pg-extension-build.yaml
@@ -8,12 +8,11 @@ on:
   workflow_dispatch:
     inputs:
       pg_version:
-        description: "PostgreSQL version to build (16, 17, 18, or all)"
+        description: "PostgreSQL version to build (17, 18, or all)"
         required: false
         default: "18"
         type: choice
         options:
-          - "16"
           - "17"
           - "18"
           - all
@@ -50,8 +49,8 @@ jobs:
         id: set-versions
         run: |-
           if [ "${PG_VERSION}" == "all" ]; then
-            echo "versions=[\"16\",\"17\",\"18\"]" >> "${GITHUB_OUTPUT}"
-            echo "versions-list=16,17,18" >> "${GITHUB_OUTPUT}"
+            echo "versions=[\"17\",\"18\"]" >> "${GITHUB_OUTPUT}"
+            echo "versions-list=17,18" >> "${GITHUB_OUTPUT}"
           else
             echo "versions=[\"${PG_VERSION}\"]" >> "${GITHUB_OUTPUT}"
             echo "versions-list=${PG_VERSION}" >> "${GITHUB_OUTPUT}"
diff --git a/cpp/CMakeLists.pg.cmake b/cpp/CMakeLists.pg.cmake
@@ -1,15 +1,10 @@
-option(BUILD_PG_16 "Build PostgreSQL 16 extension" OFF)
 option(BUILD_PG_17 "Build PostgreSQL 17 extension" OFF)
 option(BUILD_PG_18 "Build PostgreSQL 18 extension" ON)
 option(USE_DEEPLAKE_SHARED "Use shared library for deeplake_api (default: auto-detect)" OFF)
 
 set(PG_MODULE deeplake_pg)
 set(PG_VERSIONS)
 
-if(BUILD_PG_16)
-    list(APPEND PG_VERSIONS 16)
-endif()
-
 if(BUILD_PG_17)
     list(APPEND PG_VERSIONS 17)
 endif()
diff --git a/cpp/cmake/modules/FindPostgres.cmake b/cpp/cmake/modules/FindPostgres.cmake
@@ -3,14 +3,12 @@ include(ExternalProject)
 
 # Define PostgreSQL versions
 set(postgres_versions
-    "REL_16_0"
     "REL_17_0"
     "REL_18_0"
 )
 
 # Define corresponding SHA256 checksums for each version
 set(postgres_SHA256_CHECKSUMS
-    "37851d1fdae1f2cdd1d23bf9a4598b6c2f3f6792e18bc974d78ed780a28933bf"
     "16912fe4aef3c8f297b5da1b591741f132377c8b5e1b8e896e07fdd680d6bf34"
     "b155bd4a467b401ebe61b504643492aae2d0836981aa4a5a60f8668b94eadebc"
 )
@@ -47,6 +45,5 @@ foreach(postgres_version IN LISTS postgres_versions)
     )
 endforeach()
 
-set(postgres_INSTALL_DIR_REL_16_0 ${DEFAULT_PARENT_DIR}/.ext/postgres-REL_16_0/install)
 set(postgres_INSTALL_DIR_REL_17_0 ${DEFAULT_PARENT_DIR}/.ext/postgres-REL_17_0/install)
 set(postgres_INSTALL_DIR_REL_18_0 ${DEFAULT_PARENT_DIR}/.ext/postgres-REL_18_0/install)
diff --git a/cpp/deeplake_pg/extension_init.cpp b/cpp/deeplake_pg/extension_init.cpp
@@ -642,8 +642,6 @@ static void process_utility(PlannedStmt* pstmt,
                         }
                     }
                 }
-
-                // DROP SCHEMA propagation is handled by DDL WAL logging post-hook.
             }
         } else if (stmt->removeType == OBJECT_DATABASE) {
             const char* query = "SELECT nspname, relname "
diff --git a/cpp/deeplake_pg/pg_version_compat.h b/cpp/deeplake_pg/pg_version_compat.h
@@ -1,6 +1,5 @@
 #pragma once
 
-#define PG_VERSION_NUM_16 160000
 #define PG_VERSION_NUM_17 170000
 #define PG_VERSION_NUM_18 180000
 
diff --git a/cpp/deeplake_pg/sync_worker.cpp b/cpp/deeplake_pg/sync_worker.cpp
@@ -32,6 +32,10 @@ extern "C" {
 #include "table_storage.hpp"
 #include "utils.hpp"
 
+#include <async/promise.hpp>
+#include <deeplake_api/catalog_table.hpp>
+#include <icm/vector.hpp>
+
 #include <algorithm>
 #include <cstdio>
 #include <cstring>
@@ -550,7 +554,6 @@ PGDLLEXPORT void deeplake_sync_worker_main(Datum main_arg)
     std::unordered_map<std::string, int64_t> last_db_seqs;
     load_wal_checkpoints(last_db_seqs);
 
-
     while (!got_sigterm) {
         // Process pending interrupts (including ProcSignalBarrier from DROP DATABASE)
         CHECK_FOR_INTERRUPTS();
diff --git a/cpp/deeplake_pg/table_am.cpp b/cpp/deeplake_pg/table_am.cpp
@@ -313,7 +313,6 @@ bool deeplake_scan_analyze_next_tuple(
     return true;
 }
 
-#if PG_VERSION_NUM >= PG_VERSION_NUM_17
 bool deeplake_scan_analyze_next_block(TableScanDesc scan, ReadStream* stream)
 {
     DeeplakeScanData* scan_data = get_scan_data(scan);
@@ -349,7 +348,6 @@ bool deeplake_scan_analyze_next_block(TableScanDesc scan, ReadStream* stream)
 
     return true; // Indicate we have data to process
 }
-#endif
 
 double deeplake_index_build_range_scan(Relation heap_rel,
                                        Relation index_rel,
@@ -657,9 +655,7 @@ void deeplake_table_am_routine::initialize()
     routine.relation_size = deeplake_relation_size;
     routine.relation_estimate_size = deeplake_estimate_rel_size;
     routine.scan_analyze_next_tuple = deeplake_scan_analyze_next_tuple;
-#if PG_VERSION_NUM >= PG_VERSION_NUM_17
     routine.scan_analyze_next_block = deeplake_scan_analyze_next_block;
-#endif
 
     routine.parallelscan_initialize = parallelscan_initialize;
     routine.parallelscan_estimate = parallelscan_estimate;
diff --git a/cpp/deeplake_pg/table_storage.cpp b/cpp/deeplake_pg/table_storage.cpp
@@ -240,8 +240,6 @@ void table_storage::save_table_metadata(const pg::table_data& table_data)
         }
         return true;
     });
-
-    // Stateless sync uses DDL WAL replay from __wal_table.
 }
 
 void table_storage::load_table_metadata()
@@ -855,8 +853,6 @@ void table_storage::drop_table(const std::string& table_name)
         auto& table_data = get_table_data(table_name);
         auto creds = session_credentials::get_credentials();
 
-        // Stateless propagation is handled by DDL WAL logging in ProcessUtility hook.
-
         try {
             table_data.commit(); // Ensure all changes are committed before deletion
             table_version_tracker::drop_table(table_data.get_table_oid());
diff --git a/postgres/scripts/build_deb.sh b/postgres/scripts/build_deb.sh
@@ -42,10 +42,10 @@ usage() {
     echo -e "  repository:         Repository directory path"
     echo -e "  architecture:       amd64 or arm64"
     echo -e "  gpg-keyid:          GPG key ID for signing"
-    echo -e "  supported-versions: Comma-separated PostgreSQL versions (e.g., 16,17,18)"
+    echo -e "  supported-versions: Comma-separated PostgreSQL versions (e.g., 17,18)"
     echo -e "\nExamples:"
-    echo -e "  bash $0 4.4.4-1 /tmp/repo arm64 1F8B584DBEA11E9D 16,17,18"
-    echo -e "  bash $0 /tmp/repo arm64 1F8B584DBEA11E9D 16,17,18  # Auto-detect version"
+    echo -e "  bash $0 4.4.4-1 /tmp/repo arm64 1F8B584DBEA11E9D 17,18"
+    echo -e "  bash $0 /tmp/repo arm64 1F8B584DBEA11E9D 17,18  # Auto-detect version"
 }
 
 # Error handling function
diff --git a/postgres/scripts/install.sh b/postgres/scripts/install.sh
@@ -63,11 +63,11 @@ check_postgres() {
   if command_exists psql; then
     psql_version=$(psql -V | awk '{print $3}' | cut -d'.' -f1)
     case "$psql_version" in
-    14 | 15 | 16 | 17)
+    17 | 18)
       log "PostgreSQL version $psql_version detected."
       ;;
     *)
-      handle_error 1 "Unsupported PostgreSQL version: $psql_version. Only versions 14, 15, 16, and 17 are supported."
+      handle_error 1 "Unsupported PostgreSQL version: $psql_version. Only versions 17 and 18 are supported."
       ;;
     esac
   else
diff --git a/scripts/build_pg_ext.py b/scripts/build_pg_ext.py
@@ -11,8 +11,8 @@
 Usage: python3 scripts/build_pg_ext.py prod                                #Release build
 Usage: python3 scripts/build_pg_ext.py debug --deeplake-shared             #Debug build with shared deeplake_api linking
 Usage: python3 scripts/build_pg_ext.py debug --deeplake-static             #Debug build with static deeplake_api linking (force)
-Usage: python3 scripts/build_pg_ext.py dev --pg-versions 16,17,18          #Build for PostgreSQL 16, 17, and 18
-Usage: python3 scripts/build_pg_ext.py dev --pg-versions 16                #Build for PostgreSQL 16 only
+Usage: python3 scripts/build_pg_ext.py dev --pg-versions 17,18             #Build for PostgreSQL 17 and 18
+Usage: python3 scripts/build_pg_ext.py dev --pg-versions 17                #Build for PostgreSQL 17 only
 Usage: python3 scripts/build_pg_ext.py prod --pg-versions all              #Build for all supported PostgreSQL versions
 """
 
@@ -141,7 +141,7 @@ def run(mode: str, incremental: bool, deeplake_link_type: str = None, pg_version
 
             # Add PostgreSQL version options if specified
             if pg_versions is not None:
-                supported_versions = {16, 17, 18}
+                supported_versions = {17, 18}
                 # Set all versions to OFF by default
                 for ver in supported_versions:
                     if ver in pg_versions:
@@ -234,20 +234,20 @@ def write_mode(mode: str):
                 i += 1
             elif arg == "--pg-versions":
                 if i + 1 >= len(sys.argv):
-                    raise Exception("--pg-versions requires a value (e.g., '16,17,18' or 'all')")
+                    raise Exception("--pg-versions requires a value (e.g., '17,18' or 'all')")
                 versions_str = sys.argv[i + 1]
                 if versions_str == "all":
-                    pg_versions = [16, 17, 18]
+                    pg_versions = [17, 18]
                 else:
                     try:
                         pg_versions = [int(v.strip()) for v in versions_str.split(',')]
                         # Validate versions
-                        supported = {16, 17, 18}
+                        supported = {17, 18}
                         invalid = set(pg_versions) - supported
                         if invalid:
                             raise Exception(f"Invalid PostgreSQL versions: {invalid}. Supported: {supported}")
                     except ValueError:
-                        raise Exception(f"Invalid --pg-versions format: '{versions_str}'. Use comma-separated numbers (e.g., '16,17,18') or 'all'")
+                        raise Exception(f"Invalid --pg-versions format: '{versions_str}'. Use comma-separated numbers (e.g., '17,18') or 'all'")
                 i += 2
             else:
                 raise Exception(f"Invalid option '{arg}'. Use --deeplake-shared, --deeplake-static, or --pg-versions")

Original file line number	Diff line number	Diff line change
`@@ -3,14 +3,12 @@ include(ExternalProject)`
`3`	`3`
`4`	`4`	`# Define PostgreSQL versions`
`5`	`5`	`set(postgres_versions`
`6`		`- "REL_16_0"`
`7`	`6`	`"REL_17_0"`
`8`	`7`	`"REL_18_0"`
`9`	`8`	`)`
`10`	`9`
`11`	`10`	`# Define corresponding SHA256 checksums for each version`
`12`	`11`	`set(postgres_SHA256_CHECKSUMS`
`13`		`- "37851d1fdae1f2cdd1d23bf9a4598b6c2f3f6792e18bc974d78ed780a28933bf"`
`14`	`12`	`"16912fe4aef3c8f297b5da1b591741f132377c8b5e1b8e896e07fdd680d6bf34"`
`15`	`13`	`"b155bd4a467b401ebe61b504643492aae2d0836981aa4a5a60f8668b94eadebc"`
`16`	`14`	`)`
`@@ -47,6 +45,5 @@ foreach(postgres_version IN LISTS postgres_versions)`
`47`	`45`	`)`
`48`	`46`	`endforeach()`
`49`	`47`
`50`		`-set(postgres_INSTALL_DIR_REL_16_0 ${DEFAULT_PARENT_DIR}/.ext/postgres-REL_16_0/install)`
`51`	`48`	`set(postgres_INSTALL_DIR_REL_17_0 ${DEFAULT_PARENT_DIR}/.ext/postgres-REL_17_0/install)`
`52`	`49`	`set(postgres_INSTALL_DIR_REL_18_0 ${DEFAULT_PARENT_DIR}/.ext/postgres-REL_18_0/install)`
Original file line number	Diff line number	Diff line change
`@@ -642,8 +642,6 @@ static void process_utility(PlannedStmt* pstmt,`
`642`	`642`	`}`
`643`	`643`	`}`
`644`	`644`	`}`
`645`		`-`
`646`		`- // DROP SCHEMA propagation is handled by DDL WAL logging post-hook.`
`647`	`645`	`}`
`648`	`646`	`} else if (stmt->removeType == OBJECT_DATABASE) {`
`649`	`647`	`const char* query = "SELECT nspname, relname "`