Merge branch 'develop' into automate-flake-updates

Author: yvan-sraka

Dockerfile-orioledb-17

@@ -157,14 +157,6 @@ RUN sed -i 's/ timescaledb,//g;' "/etc/postgresql/postgresql.conf" && \
 RUN sed -i 's/\(shared_preload_libraries.*\)'\''\(.*\)$/\1, orioledb'\''\2/' "/etc/postgresql/postgresql.conf" && \
     echo "default_table_access_method = 'orioledb'" >> "/etc/postgresql/postgresql.conf"
 
-# OrioleDB rewind configuration
-# Enables time-based rewind capability for up to 20 minutes (1200 seconds)
-# Buffer size: 1280 buffers * 8KB = 10MB for transaction retention
-RUN echo "orioledb.enable_rewind = true" >> "/etc/postgresql/postgresql.conf" && \
-    echo "orioledb.rewind_max_time = 1200" >> "/etc/postgresql/postgresql.conf" && \
-    echo "orioledb.rewind_max_transactions = 100000" >> "/etc/postgresql/postgresql.conf" && \
-    echo "orioledb.rewind_buffers = 1280" >> "/etc/postgresql/postgresql.conf"
-
 # Include schema migrations
 COPY migrations/db /docker-entrypoint-initdb.d/
 COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql

ansible/tasks/setup-system.yml

@@ -167,10 +167,11 @@
         state: 'present'
 
     # postgres_exporter runs on port 9187 and postgresT occasionlly chooses it as random srcport
-    - name: Set net.ipv4.ip_local_reserved_ports=9187
+    # adminapi for 8085
+    - name: Set net.ipv4.ip_local_reserved_ports
       ansible.builtin.sysctl:
         name: 'net.ipv4.ip_local_reserved_ports'
-        value: 9187
+        value: '9187,8085'
         state: 'present'
 
 - name: Execute tasks when (debpkg_mode or nixpkg_mode)

ansible/tasks/stage2-setup-postgres.yml

@@ -52,38 +52,6 @@
             path: '/etc/postgresql/postgresql.conf'
             state: 'present'
 
-        - name: Enable OrioleDB rewind feature
-          ansible.builtin.lineinfile:
-            path: /etc/postgresql/postgresql.conf
-            line: "orioledb.enable_rewind = true"
-            state: present
-          when: is_psql_oriole and stage2_nix
-          become: yes
-
-        - name: Set OrioleDB rewind max time (20 minutes)
-          ansible.builtin.lineinfile:
-            path: /etc/postgresql/postgresql.conf
-            line: "orioledb.rewind_max_time = 1200"
-            state: present
-          when: is_psql_oriole and stage2_nix
-          become: yes
-
-        - name: Set OrioleDB rewind max transactions
-          ansible.builtin.lineinfile:
-            path: /etc/postgresql/postgresql.conf
-            line: "orioledb.rewind_max_transactions = 100000"
-            state: present
-          when: is_psql_oriole and stage2_nix
-          become: yes
-
-        - name: Set OrioleDB rewind buffers (1280 buffers = 10MB)
-          ansible.builtin.lineinfile:
-            path: /etc/postgresql/postgresql.conf
-            line: "orioledb.rewind_buffers = 1280"
-            state: present
-          when: is_psql_oriole and stage2_nix
-          become: yes
-
         - name: Add ORIOLEDB_ENABLED environment variable
           ansible.builtin.lineinfile:
             line: 'ORIOLEDB_ENABLED=true'

ansible/vars.yml

@@ -10,9 +10,9 @@ postgres_major:
 
 # Full version strings for each major version
 postgres_release:
-  postgresorioledb-17: "17.6.0.041-orioledb"
-  postgres17: "17.6.1.084"
-  postgres15: "15.14.1.084"
+  postgresorioledb-17: "17.6.0.044-orioledb"
+  postgres17: "17.6.1.087"
+  postgres15: "15.14.1.087"
 
 # Non Postgres Extensions
 pgbouncer_release: 1.25.1

nix/config.nix

@@ -56,8 +56,8 @@ in
         };
         orioledb = {
           "17" = {
-            version = "17_15";
-            hash = "sha256-1v3FGIN0UW+E4ilLwbsV3nXvef3n9O8bVmgyjRFEJsU=";
+            version = "17_16";
+            hash = "sha256-Xm9IUsvmlcayNQH8TCvHoIV23xkt/WQV0Oy4CiJkywc=";
           };
         };
       };

nix/devShells.nix

@@ -32,28 +32,28 @@
             export HISTFILE=.history
           '';
         };
+      docvenv = pkgs.python3.buildEnv.override {
+        extraLibs = self'.packages.docs.nativeBuildInputs;
+      };
     in
     {
       devShells = {
         default = pkgs.devshell.mkShell {
-          packages =
-            with pkgs;
-            [
-              coreutils
-              just
-              nix-update
-              #pg_prove
-              shellcheck
-              ansible
-              ansible-lint
-              aws-vault
-              packer
-              dbmate
-              nushell
-              pythonEnv
-              config.treefmt.build.wrapper
-            ]
-            ++ self'.packages.docs.nativeBuildInputs;
+          packages = with pkgs; [
+            coreutils
+            just
+            nix-update
+            #pg_prove
+            shellcheck
+            ansible
+            ansible-lint
+            aws-vault
+            packer
+            dbmate
+            nushell
+            pythonEnv
+            config.treefmt.build.wrapper
+          ];
           devshell.startup.pre-commit.text = config.pre-commit.installationScript;
           commands = [
             {
@@ -74,6 +74,12 @@
               command = "pre-commit run --all-files";
               category = "check";
             }
+            {
+              name = "serve-nix-doc";
+              help = "Spin up a server exposing the nix documentation";
+              command = "pushd $(git rev-parse --show-toplevel)/nix && ${docvenv}/bin/mkdocs serve -o";
+              category = "doc";
+            }
             {
               name = "watch";
               help = "Watch for file changes and run all checks";

nix/docs/README.md

@@ -18,14 +18,19 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md).
 - **[Receipt Files](./receipt-files.md)** - Understanding build receipts
 - **[Start Client/Server](./start-client-server.md)** - Running PostgreSQL client and server
 - **[Docker](./docker.md)** - Docker integration and usage
+- **[Docker Image Size Analyzer](./image-size-analyzer-usage.md)** - Tool to analyze the Docker image sizes
 - **[Use direnv](./use-direnv.md)** - Development environment with direnv
 - **[Pre-commit Hooks](./pre-commit-hooks.md)** - Automatic formatting and code checks before commits
 - **[Nix Formatter](./nix-formatter.md)** - Code formatting with treefmt
+- **[Create a New pgrx Extension](./creating-pgrx-extension.md)** - How to set up a new cargo pgrx extension
+- **[Updating pgrx Extensions](./updating-pgrx-extensions.md)** - How to upgrade the cargo pgrx extensions
+- **[Receipt Files](./receipt-files.md)** - Understand what receipt files are
 
 ## Package Management
 
 - **[Adding New Packages](./adding-new-package.md)** - How to add new PostgreSQL extensions
 - **[Update Extensions](./update-extension.md)** - How to update existing extensions
+- **[Update Nix Dependecies](./updating-dependencies.md)** - How to update the Nix dependencies
 - **[New Major PostgreSQL](./new-major-postgres.md)** - Adding support for new PostgreSQL versions
 - **[Nix Overlays](./nix-overlays.md)** - Understanding and using Nix overlays
 
@@ -34,6 +39,7 @@ learn how to play with `postgres` in the [build guide](./build-postgres.md).
 - **[Adding Tests](./adding-tests.md)** - How to add tests for extensions
 - **[Migration Tests](./migration-tests.md)** - Testing database migrations
 - **[Testing PG Upgrade Scripts](./testing-pg-upgrade-scripts.md)** - Testing PostgreSQL upgrades
+- **[Docker Image testing](./docker-testing.md)** - How to test the docker images against the pg_regress test suite.
 
 ## Reference
 

nix/docs/adding-tests.md

@@ -5,13 +5,13 @@ There are basically two types of tests you can add:
 - Migration tests.
 
 In all cases, a number of extensions may be installed into the database for
-use; you can see those in both [postgresql.conf.in](../tests/postgresql.conf.in)
-and [prime.sql](../tests/prime.sql) (extensions may be enabled in either place.)
+use; you can see those in both `/nix/tests/postgresql.conf.in`
+and `/nix/tests/prime.sql` (extensions may be enabled in either place.)
 
 ## pg\_regress tests
 
-pg\_regress tests are in [tests/sql](./../tests/sql/) with output in [tests/expected](./../tests/expected/).
-To create a new test, create a new SQL file in [tests/sql](./../tests/sql/)
+pg\_regress tests are in `/nix/tests/sql/` with output in `/nix/tests/expected/`.
+To create a new test, create a new SQL file in `/nix/tests/sql/`.
 
 Next, for each current major version of postgres, we run a "flake check" build one at a time.
 
@@ -29,8 +29,8 @@ Next, review the logs to identify where the test output was written
 
 ```
 postgres> CREATE EXTENSION IF NOT EXISTS index_advisor;
-postgres> CREATE EXTENSION  
-postgres> (using postmaster on localhost, port 5432)    
+postgres> CREATE EXTENSION
+postgres> (using postmaster on localhost, port 5432)
 postgres> ============== running regression test queries        ==============
 postgres> test new_test                     ... diff: /nix/store/5gk419ddz7mzzwhc9j6yj5i8lkw67pdl-tests/expected/new_test.out: No such file or directory
 postgres> diff command failed with status 512: diff  "/nix/store/5gk419ddz7mzzwhc9j6yj5i8lkw67pdl-tests/expected/new_test.out" "/nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output/results/new_test.out" > "/nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/regression_output/results/new_test.out.diff
@@ -44,15 +44,15 @@ cp -r /nix/store/2fbrvnnr7iz6yigyf0rb0vxnyqvrgxzp-postgres-15.6-check-harness/re
 
 Then you can review the contents of `regression_output/results/new_test.out` to see if it matches what you expected.
 
-If it does match your expectations, copy the file to [tests/expected](./../tests/expected/) and the test will pass on the next run.
+If it does match your expectations, copy the file to `/nix/tests/expected/` and the test will pass on the next run.
 
 If the output does not match your expectations, update the `<new_test>.sql` file, re-run with `nix flake check -L` and try again
 
 
 ## pgTAP tests
 
 These are super easy: simply add `.sql` files to the
-[tests/smoke](./../tests/smoke/) directory, then:
+`/nix/tests/smoke/` directory, then:
 
 ```
 nix flake check -L
@@ -100,7 +100,7 @@ extension authors), step 3 isn't guaranteed, so that's what the whole idea is
 designed to test.
 
 To add data into the database, modify the
-[data.sql](../nix/tests/migrations/data.sql) script and add whatever you want into
+`/nix/tests/migrations/data.sql` script and add whatever you want into
 it. This script gets loaded into the old version of the database at startup, and
 it's expected that the new version of the database can handle it.
 

nix/docs/development-workflow.md

@@ -20,13 +20,14 @@ nix run .#trigger-nix-build
 ```
 
 This will:
+
 - Trigger a GitHub Actions workflow
 - Build PostgreSQL and extensions
 - Run nix flake check tests (evaluation of nix code, pg_regress and migrations tests)
 - Cache the results in the Supabase Nix binary cache
 - Watch the workflow progress until completion
 
-The workflow will run on the branch you're currently on. 
+The workflow will run on the branch you're currently on.
 
 If you're on a feature different branch, you'll be prompted to confirm before proceeding.
 
@@ -46,6 +47,7 @@ aws-vault exec  <profile-name> -- nix run .#build-test-ami orioledb-17
 ```
 
 This will:
+
 - Build two AMI stages using Packer
 - Clean up temporary instances after AMI builds
 - Output the final AMI name (e.g., `supabase-postgres-abc123`)
@@ -62,13 +64,15 @@ nix run .#run-testinfra -- --aws-vault-profile <profile-name> --ami-name supabas
 ```
 
 This will:
+
 - Create a Python virtual environment
 - Install required Python packages
 - Create an EC2 instance from the AMI
 - Run the test suite
 - Automatically terminate the EC2 instance when done
 
 The script handles:
+
 - Setting up AWS credentials via aws-vault
 - Creating and managing the Python virtual environment
 - Running the tests
@@ -85,6 +89,7 @@ aws-vault exec <profile-name> -- nix run .#cleanup-ami supabase-postgres-abc123
 ```
 
 This will:
+
 - Deregister the AMI
 - Clean up any associated resources
 
@@ -109,26 +114,29 @@ This will:
 ### Environment Variables
 
 The following environment variables are used:
+
 - `AWS_VAULT`: AWS Vault profile name (default: staging)
 - `AWS_REGION`: AWS region (default: ap-southeast-1)
 - `AMI_NAME`: Name of the AMI to test
 
 ## Best Practices
 
 1. **Branch Management**
-   - Use feature branches for development
-   - Merge to develop for testing
-   - Use release branches for version-specific changes
+
+    - Use feature branches for development
+    - Merge to develop for testing
+    - Use release branches for version-specific changes
 
 2. **Resource Cleanup**
-   - Always run the cleanup step after testing
-   - Monitor AWS console for any lingering resources
-   - Use the cleanup-ami command when done with an AMI
+
+    - Always run the cleanup step after testing
+    - Monitor AWS console for any lingering resources
+    - Use the cleanup-ami command when done with an AMI
 
 3. **Testing**
-   - Run tests locally before pushing changes
-   - Verify AMI builds before running testinfra
-   - Check test output for any warnings or errors
+    - Run tests locally before pushing changes
+    - Verify AMI builds before running testinfra
+    - Check test output for any warnings or errors
 
 ## Additional Commands
 
@@ -138,4 +146,4 @@ nix run .#show-commands
 
 # Update README with latest command information
 nix run .#update-readme
-``` 
+```

nix/docs/documentation.md

@@ -7,8 +7,7 @@ We are using markdown for documentation in the `nix/docs` directory, which is re
 To generate the documentation locally, you can use the following command in a development shell:
 
 ```bash
-cd nix
-mkdocs serve
+serve-nix-doc
 ```
 
 This will start a local server at `http://localhost:8000` where you can view the documentation.