Skip to content

Commit 041eb41

Browse files
TylerHilleryTylerHillery
committed
wip: add pg_duckdb extension
1 parent edee072 commit 041eb41

5 files changed

Lines changed: 204 additions & 2 deletions

File tree

‎ansible/files/postgresql_config/postgresql.conf.j2‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -687,7 +687,7 @@ default_text_search_config = 'pg_catalog.english'
687687
#local_preload_libraries = ''
688688
#session_preload_libraries = ''
689689

690-
shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault' # (change requires restart)
690+
shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, supabase_vault, pg_duckdb' # (change requires restart)
691691
jit_provider = 'llvmjit' # JIT library to use
692692

693693
# - Other Defaults -

‎nix/ext/duckdb-lib.nix‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
{
2+
lib,
3+
stdenv,
4+
fetchFromGitHub,
5+
cmake,
6+
ninja,
7+
openssl,
8+
python3,
9+
}:
10+
11+
stdenv.mkDerivation (finalAttrs: {
12+
pname = "duckdb";
13+
version = "1.4.3";
14+
15+
src = fetchFromGitHub {
16+
owner = "duckdb";
17+
repo = "duckdb";
18+
rev = "v${finalAttrs.version}";
19+
hash = "sha256-zYiyY/8mYCyKuSQYNxepGbZPVgdCgULLmhZlWAAW0QA=";
20+
};
21+
22+
outputs = [
23+
"out"
24+
"lib"
25+
"dev"
26+
];
27+
28+
# cmake installs the shared library and headers; nothing goes into $out
29+
# since BUILD_SHELL=OFF (no CLI binary). We create $out explicitly so
30+
# nixpkgs's multi-output setup hooks have a valid fallback for outputBin,
31+
# outputDoc, outputMan, outputInfo, etc.
32+
postInstall = ''
33+
mkdir -p $out
34+
'';
35+
36+
nativeBuildInputs = [
37+
cmake
38+
ninja
39+
# python3 is required by DuckDB's cmake build scripts for code generation
40+
python3
41+
];
42+
43+
buildInputs = [ openssl ];
44+
45+
cmakeFlags = [
46+
# Required by pg_duckdb so that DuckDB symbols are visible when loaded
47+
# by PostgreSQL's dlopen. Without this, pg_duckdb's .so cannot resolve
48+
# DuckDB symbols at runtime.
49+
"-DCXX_EXTRA=-fvisibility=default"
50+
(lib.cmakeBool "BUILD_SHELL" false)
51+
(lib.cmakeBool "BUILD_PYTHON" false)
52+
(lib.cmakeBool "BUILD_UNITTESTS" false)
53+
# Prevent cmake from trying to fetch anything from the internet
54+
(lib.cmakeBool "FETCHCONTENT_FULLY_DISCONNECTED" true)
55+
# Embed the version string so DuckDB doesn't report "unknown"
56+
(lib.cmakeFeature "OVERRIDE_GIT_DESCRIBE" "v${finalAttrs.version}-0-g0000000")
57+
];
58+
59+
# Skip the test suite — we just want the library
60+
doInstallCheck = false;
61+
doCheck = false;
62+
63+
meta = {
64+
description = "DuckDB shared library (for use by pg_duckdb)";
65+
homepage = "https://duckdb.org/";
66+
license = lib.licenses.mit;
67+
platforms = lib.platforms.unix;
68+
};
69+
})

‎nix/ext/pg_duckdb.nix‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
{
2+
lib,
3+
stdenv,
4+
fetchFromGitHub,
5+
postgresql,
6+
buildEnv,
7+
lz4,
8+
patchelf,
9+
duckdb-lib,
10+
latestOnly ? false,
11+
}:
12+
let
13+
pname = "pg_duckdb";
14+
version = "1.1.1";
15+
16+
drv = stdenv.mkDerivation {
17+
inherit pname version;
18+
19+
src = fetchFromGitHub {
20+
owner = "duckdb";
21+
repo = "pg_duckdb";
22+
rev = "v${version}";
23+
hash = "sha256-0cNfDZkd6x45xpWyPMfFoYAklE+4lAjO02SjV+V/dxU=";
24+
};
25+
26+
nativeBuildInputs = lib.optionals (!stdenv.isDarwin) [ patchelf ];
27+
28+
buildInputs = [
29+
postgresql
30+
duckdb-lib.lib
31+
duckdb-lib.dev
32+
lz4
33+
];
34+
35+
# No configure script — uses PGXS directly via Makefile.global
36+
dontConfigure = true;
37+
38+
postPatch = ''
39+
# The GitHub tarball does not include the third_party/duckdb git submodule.
40+
# pg_duckdb's Makefile expects DuckDB headers at:
41+
# third_party/duckdb/src/include
42+
# third_party/duckdb/third_party/re2
43+
# and the built library at:
44+
# third_party/duckdb/build/release/src/libduckdb<ext>
45+
# We satisfy all three using our pre-built duckdb-lib derivation.
46+
47+
mkdir -p third_party/duckdb/src
48+
ln -sf ${duckdb-lib.dev}/include third_party/duckdb/src/include
49+
50+
# pg_duckdb adds re2 as a system include for warning suppression only;
51+
# it does not include re2 headers directly.
52+
mkdir -p third_party/duckdb/third_party/re2
53+
54+
# Make the pre-built libduckdb visible at the path the Makefile's linker
55+
# flags expect: -L$(DUCKDB_BUILD_DIR)/src -lduckdb
56+
# (DUCKDB_BUILD_DIR = third_party/duckdb/build/release)
57+
mkdir -p third_party/duckdb/build/release/src
58+
ln -sf ${duckdb-lib.lib}/lib/libduckdb${postgresql.dlSuffix} \
59+
third_party/duckdb/build/release/src/libduckdb${postgresql.dlSuffix}
60+
61+
# The Makefile has two separate dependencies on .git/modules/third_party/duckdb/HEAD:
62+
# 1. $(FULL_DUCKDB_LIB) — recipe overridden to no-op below
63+
# 2. $(OBJS) — every .o depends on it directly (Makefile:98) to ensure
64+
# DuckDB headers are present before compilation.
65+
# Fake the sentinel so make considers it satisfied without running git.
66+
mkdir -p .git/modules/third_party/duckdb
67+
echo "ref: refs/heads/main" > .git/modules/third_party/duckdb/HEAD
68+
69+
# Override the FULL_DUCKDB_LIB build recipe with a no-op.
70+
# GNU Make uses the last-defined recipe when a target appears multiple times.
71+
# This prevents make from invoking cmake/ninja to build DuckDB from source;
72+
# the pre-built library is already in place via the symlink above.
73+
printf '\n# Nix override: skip DuckDB cmake build\n$(FULL_DUCKDB_LIB):\n\t@:\n' >> Makefile
74+
'';
75+
76+
makeFlags = [
77+
"PG_CONFIG=${postgresql}/bin/pg_config"
78+
];
79+
80+
installPhase = ''
81+
runHook preInstall
82+
mkdir -p $out/{lib,share/postgresql/extension}
83+
84+
install -Dm755 pg_duckdb${postgresql.dlSuffix} $out/lib/pg_duckdb${postgresql.dlSuffix}
85+
86+
# Fix rpath so pg_duckdb.so finds libduckdb in the Nix store at runtime.
87+
# PostgreSQL uses dlopen() to load extensions, so the rpath in the .so
88+
# file must point to an absolute path where libduckdb lives.
89+
${lib.optionalString (!stdenv.isDarwin) ''
90+
${patchelf}/bin/patchelf \
91+
--set-rpath "${duckdb-lib.lib}/lib:${postgresql}/lib" \
92+
$out/lib/pg_duckdb${postgresql.dlSuffix}
93+
''}
94+
${lib.optionalString stdenv.isDarwin ''
95+
install_name_tool \
96+
-add_rpath "${duckdb-lib.lib}/lib" \
97+
-add_rpath "${postgresql}/lib" \
98+
$out/lib/pg_duckdb${postgresql.dlSuffix}
99+
''}
100+
101+
cp pg_duckdb.control $out/share/postgresql/extension/
102+
cp sql/pg_duckdb--*.sql $out/share/postgresql/extension/
103+
104+
runHook postInstall
105+
'';
106+
107+
meta = {
108+
description = "DuckDB-powered analytical queries inside PostgreSQL";
109+
homepage = "https://github.com/duckdb/pg_duckdb";
110+
platforms = postgresql.meta.platforms;
111+
license = lib.licenses.mit;
112+
};
113+
};
114+
in
115+
buildEnv {
116+
name = pname;
117+
paths = [ drv ];
118+
pathsToLink = [
119+
"/lib"
120+
"/share/postgresql/extension"
121+
];
122+
123+
passthru = {
124+
inherit pname version latestOnly;
125+
};
126+
}

‎nix/packages/default.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
build-test-ami = pkgs.callPackage ./build-test-ami.nix { packer = self'.packages.packer; };
4141
cleanup-ami = pkgs.callPackage ./cleanup-ami.nix { };
4242
dbmate-tool = pkgs.callPackage ./dbmate-tool.nix { inherit (self.supabase) defaults; };
43+
duckdb-lib = pkgs.callPackage ../ext/duckdb-lib.nix { };
4344
docker-image-inputs = pkgs.callPackage ./docker-image-inputs.nix {
4445
psql_15_slim = self'.packages."psql_15_slim/bin";
4546
psql_17_slim = self'.packages."psql_17_slim/bin";

‎nix/packages/postgres.nix‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
../ext/wrappers/default.nix
5151
../ext/supautils.nix
5252
../ext/plv8
53+
../ext/pg_duckdb.nix
5354
];
5455

5556
#Where we import and build the orioledb extension, we add on our custom extensions
@@ -59,7 +60,11 @@
5960
x: x != ../ext/timescaledb.nix && x != ../ext/timescaledb-2.9.1.nix && x != ../ext/plv8
6061
) ourExtensions;
6162

62-
orioledbExtensions = orioleFilteredExtensions ++ [ ../ext/orioledb.nix ];
63+
# pg_duckdb is excluded from orioledb because orioledb's PostgreSQL fork
64+
# modifies the TableAM API in ways that are incompatible with pg_duckdb v1.1.1.
65+
orioledbExtensions = (builtins.filter (x: x != ../ext/pg_duckdb.nix) orioleFilteredExtensions) ++ [
66+
../ext/orioledb.nix
67+
];
6368
dbExtensions17 = orioleFilteredExtensions;
6469

6570
# CLI extensions - minimal set for Supabase CLI with migration support
@@ -137,6 +142,7 @@
137142
inherit postgresql latestOnly;
138143
switch-ext-version = extCallPackage ./switch-ext-version.nix { };
139144
overlayfs-on-package = extCallPackage ./overlayfs-on-package.nix { };
145+
duckdb-lib = pkgs.callPackage ../ext/duckdb-lib.nix { };
140146
}
141147
);
142148
in

0 commit comments

Comments
 (0)